diff --git a/SMART-ORIGIN-README.md b/SMART-ORIGIN-README.md index eb8e3a384..5736f0127 100644 --- a/SMART-ORIGIN-README.md +++ b/SMART-ORIGIN-README.md @@ -7,26 +7,35 @@ This documentation concerns the **"Public Transport Access"** box code, which ap ```sh v6_api/.env.template #rename it into .env ``` + +Variables defined in the above file must be available at runtime for the scripts to execute correctly. +One can either source the env file, or update the compose configuration to include them in the api container runtime (this is the production approach). + If you haven't already, update the database with Alembic (this will create the missing tables and fields) : ```bash docker-compose exec api .build/venv/bin/alembic upgrade head ``` - - - ## "Show Nearby Stops" section This section is used to search for public transport stops around 'access' waypoints. The public transport data comes from an external API called Navitia, which is stored in the CamptoCamp database. **IF YOU DON'T HAVE ANY RESULT ON THE "Public Transports Access" section** : this means that your imported database does not contain Navitia data. -To have visible results, **you must launch the script "get_public_transports_from_France.sh" in the backend (see backend documentation)** +To have visible results, you must launch a `get_public_transports_*.sh` in the backend (see backend documentation). +There are different versions of these scripts: +- whose name is ending with "_France.sh", "_Rhone.sh", or "_Isere.sh": Navitia fetching scripts for France, Rhone, and Isere regions (resp.) that should be launched *outside containers* (from the host); +- whose name is ending with "_France.bm.sh", "_Rhone.bm.sh", or "_Isere.bm.sh": Same but those should be launched *within containers*; +- whose name is ending with "_distinct.sh": Navitia fetching scripts (there is also an Isere variant) that fetch **distincts** transport stops. Those make more requests than non-distinct scripts, but they avoid duplicated stops. Those scripts should be launched *outside containers* (from the host); +- whose name is ending with "_distinct.bm.sh": Same but those should be launched *within containers*; + +On the production setup hosted by camptocamp, only scripts aimed at running within containers (ending with ".bm.sh") should be used. +The result will always be fetched for the whole France area, and preferably using the distinct script. Warning ⚠️ : it takes a while (~3h, see other option below) ``` (on api_v6/ ) -sh get_public_transports_from_France.sh +sh get_public_transports_from_France_distinct.bm.sh ``` If you just want to work with the Isère department for local dev, you can run this script instead (~18 minutes): @@ -78,7 +87,9 @@ BACK-END : This section is used to plan a trip by calling the Navitia API. Unlike the previous section, we don't store the results in the database; we query Navitia directly by launching a query from the backend. -This section uses the calculated_duration attribute, **which is calculated with the calcul_duration_for_routes.sh script in the backend (see backend documentation)** +This section uses the calculated_duration attribute, **which is calculated with the `calcul_duration_for_routes.bm.sh` script in the backend (see backend documentation)** + +Note that the `calcul_duration_for_routes.bm.sh` is intended to run from within the container. A variant named `calcul_duration_for_routes.sh` can be used to launch the script from the host. If you need to update the calculated duration of itineraries, you can run this : @@ -92,7 +103,7 @@ LIMIT_MAX = 100000 2) Run the script ``` (on api_v6/ ) -sh calcul_duration_for_routes.sh +sh calcul_duration_for_routes.bm.sh ``` 3) Put the limit back to 100 ```python diff --git a/alembic_migration/versions/27bf1b7197a6_add_coverages.py b/alembic_migration/versions/27bf1b7197a6_add_coverages.py new file mode 100644 index 000000000..d068e1b83 --- /dev/null +++ b/alembic_migration/versions/27bf1b7197a6_add_coverages.py @@ -0,0 +1,39 @@ +"""Add coverages + +Revision ID: 335e0bc4df28 +Revises: 6b40cb9c7c3d +Create Date: 2025-11-18 14:15:26.377504 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '335e0bc4df28' +down_revision = '6b40cb9c7c3d' +branch_labels = None +depends_on = None + +def upgrade(): + coverage_type = sa.Enum('fr-idf', 'fr-ne', 'fr-nw', 'fr-se', 'fr-sw', name='coverage_type', schema='guidebook') + op.create_table('coverages', + sa.Column('coverage_type', coverage_type, nullable=True), + sa.Column('document_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['document_id'], ['guidebook.documents.document_id'], ), + sa.PrimaryKeyConstraint('document_id'), + schema='guidebook' + ) + op.create_table('coverages_archives', + sa.Column('coverage_type', coverage_type, nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['id'], ['guidebook.documents_archives.id'], ), + sa.PrimaryKeyConstraint('id'), + schema='guidebook' + ) + + +def downgrade(): + op.drop_table('coverages_archives', schema='guidebook') + op.drop_table('coverages', schema='guidebook') + sa.Enum('fr-idf', 'fr-ne', 'fr-nw', 'fr-se', 'fr-sw', name='coverage_type', schema='guidebook').drop(op.get_bind()) diff --git a/c2corg_api/__init__.py b/c2corg_api/__init__.py index 2d9e2cde8..f7168c038 100644 --- a/c2corg_api/__init__.py +++ b/c2corg_api/__init__.py @@ -101,26 +101,31 @@ def configure_anonymous(settings, config): config.registry.anonymous_user_id = account_id +def delete_waypoint_stopareas(connection, waypoint_id): + # Delete existing stopareas for waypoint + delete_relation_query = text( + """ + DELETE FROM guidebook.waypoints_stopareas + WHERE waypoint_id = :waypoint_id + """ + ) + + connection.execute( + delete_relation_query, + { + "waypoint_id": waypoint_id, + }, + ) + + @event.listens_for(DocumentGeometry, "after_insert") @event.listens_for(DocumentGeometry, "after_update") def process_new_waypoint(mapper, connection, geometry): """Processes a new waypoint to find its public transports after inserting it into documents_geometries.""" - log.debug("Entering process_new_waypoint callback") + # Check if document is a waypoint waypoint_id = geometry.document_id - max_distance_waypoint_to_stoparea = int( - os.getenv("MAX_DISTANCE_WAYPOINT_TO_STOPAREA") - ) - walking_speed = float(os.getenv("WALKING_SPEED")) - max_stop_area_for_1_waypoint = int(os.getenv("MAX_STOP_AREA_FOR_1_WAYPOINT")) # noqa: E501 - api_key = os.getenv("NAVITIA_API_KEY") - max_duration = int(max_distance_waypoint_to_stoparea / walking_speed) - - # Augmenter le nombre d'arrêts récupérés pour avoir plus de choix (comme dans le bash) # noqa: E501 - max_stop_area_fetched = max_stop_area_for_1_waypoint * 3 - - # Check if document is a waypoint document_type = connection.execute( text( """ @@ -134,6 +139,21 @@ def process_new_waypoint(mapper, connection, geometry): if document_type != "w": return + log.debug("Entering process_new_waypoint callback") + max_distance_waypoint_to_stoparea = int( + os.getenv("MAX_DISTANCE_WAYPOINT_TO_STOPAREA") + ) + walking_speed = float(os.getenv("WALKING_SPEED")) + max_stop_area_for_1_waypoint = int( + os.getenv("MAX_STOP_AREA_FOR_1_WAYPOINT") + ) + api_key = os.getenv("NAVITIA_API_KEY") + max_duration = int(max_distance_waypoint_to_stoparea / walking_speed) + + # increase number of retrieved stop areas, + # to get more choices (just like the bash script) + max_stop_area_fetched = max_stop_area_for_1_waypoint * 3 + waypoint_type = connection.execute( text( """ @@ -147,28 +167,31 @@ def process_new_waypoint(mapper, connection, geometry): if waypoint_type != "access": return - log.warning(f"Waypoint Navitia processing with ID: {waypoint_id}") + log.info("Waypoint Navitia processing with ID: %d", waypoint_id) # Get waypoint coordinates lon_lat = connection.execute( text( """ - SELECT ST_X(ST_Transform(geom, 4326)) || ',' || ST_Y(ST_Transform(geom, 4326)) + SELECT ST_X(ST_Transform(geom, 4326)) + || ',' + || ST_Y(ST_Transform(geom, 4326)) FROM guidebook.documents_geometries WHERE document_id = :waypoint_id - """ # noqa: E501 + """ ), {"waypoint_id": waypoint_id}, ).scalar() if not lon_lat: - log.warning(f"Coordinates not found for waypoint {waypoint_id}") + log.warning("Coordinates not found for waypoint %d", waypoint_id) return lon, lat = lon_lat.strip().split(",") - # Navitia request - récupérer plus d'arrêts pour filtrage - places_url = f"https://api.navitia.io/v1/coord/{lon};{lat}/places_nearby" + # Navitia request - get more stop areas to filter + places_url = "https://api.navitia.io/v1/coord/%s;%s/places_nearby" % ( + lon, lat) places_params = { "type[]": "stop_area", "count": max_stop_area_fetched, # Plus d'arrêts récupérés @@ -182,15 +205,17 @@ def process_new_waypoint(mapper, connection, geometry): places_data = places_response.json() if "places_nearby" not in places_data or not places_data["places_nearby"]: - log.warning(f"No Navitia stops found for the waypoint {waypoint_id}") + log.info("No Navitia stops found for the waypoint %d; \ + deleting previously registered stops", waypoint_id) + delete_waypoint_stopareas(connection, waypoint_id) return - # --- NOUVEAU : Filtrage par diversité de transport (comme dans bash) --- + # --- NEW : Filter by diversity of transports (just like bash script) --- selected_stops = [] known_transports = set() selected_count = 0 - # Traiter les arrêts dans l'ordre (déjà triés par distance par Navitia) + # Treat stopareas in order (already sorted by distance using Navitia) for place in places_data["places_nearby"]: if place.get("embedded_type") != "stop_area": continue @@ -200,62 +225,56 @@ def process_new_waypoint(mapper, connection, geometry): stop_id = place["id"] - # Récupérer les informations de l'arrêt pour connaître ses transports - stop_info_url = f"https://api.navitia.io/v1/places/{stop_id}" - stop_info_response = requests.get(stop_info_url, headers=navitia_headers) # noqa: E501 + # Get informations of stopareas to know its transports + stop_info_url = "https://api.navitia.io/v1/places/%d", stop_id + stop_info_response = requests.get( + stop_info_url, headers=navitia_headers + ) stop_info = stop_info_response.json() if "places" not in stop_info or not stop_info["places"]: continue - # Extraire les transports de cet arrêt + # Extract transports from its stoparea current_stop_transports = set() for line in stop_info["places"][0]["stop_area"].get("lines", []): mode = line.get("commercial_mode", {}).get("name", "") code = line.get("code", "") - transport_key = f"{mode} {code}" + transport_key = "%s %s", mode, code current_stop_transports.add(transport_key) - # Vérifier si cet arrêt apporte de nouveaux transports + # Check if this stoparea can bring new transports new_transport_found = bool(current_stop_transports - known_transports) - # Si l'arrêt apporte au moins un nouveau transport, le sélectionner + # If stoparea brings at least one new transport, select it if new_transport_found: place["stop_info"] = stop_info selected_stops.append(place) known_transports.update(current_stop_transports) selected_count += 1 - # Delete existing stopareas for waypoint - delete_relation_query = text( - """ - DELETE FROM guidebook.waypoints_stopareas - WHERE waypoint_id = :waypoint_id - """ + log.info( + "Selected %d stops out of %d for waypoint %d", + selected_count, + len(places_data['places_nearby']), waypoint_id ) - connection.execute( - delete_relation_query, - { - "waypoint_id": waypoint_id, - }, - ) - - log.warning(f"Selected {selected_count} stops out of {len(places_data['places_nearby'])} for waypoint {waypoint_id}") # noqa: E501 + log.info("Deleting previously registered stops") + delete_waypoint_stopareas(connection, waypoint_id) - # Traiter uniquement les arrêts sélectionnés + # Only treat selected stopareas for place in selected_stops: stop_id = place["id"] stop_name = place["name"] lat_stop = place["stop_area"]["coord"]["lat"] lon_stop = place["stop_area"]["coord"]["lon"] - # Get the travel time by walking - utiliser les mêmes paramètres que le bash # noqa: E501 + # Get the travel time by walking - use same parameters as bash script journey_url = "https://api.navitia.io/v1/journeys" journey_params = { - "to": f"{lon};{lat}", + "to": "%s;%s" % (lon, lat), "walking_speed": walking_speed, - "max_walking_direct_path_duration": max_duration, # Paramètre corrigé # noqa: E501 + "max_walking_direct_path_duration": max_duration, "direct_path_mode[]": "walking", "from": stop_id, "direct_path": "only_with_alternatives", @@ -276,7 +295,7 @@ def process_new_waypoint(mapper, connection, geometry): duration = journey_data["journeys"][0].get("duration", 0) # Convert to distance - distance_km = round((duration * walking_speed) / 1000, 2) # Arrondi à 2 décimales comme bash # noqa: E501 + distance_km = round((duration * walking_speed) / 1000, 2) # Check if stop already exists existing_stop_query = text( @@ -292,7 +311,6 @@ def process_new_waypoint(mapper, connection, geometry): if not existing_stop_id: stop_info = place["stop_info"] - # Traiter chaque ligne comme dans le bash for line in stop_info["places"][0]["stop_area"].get("lines", []): line_full_name = line.get("name", "") line_name = line.get("code", "") @@ -304,16 +322,24 @@ def process_new_waypoint(mapper, connection, geometry): """ WITH new_stoparea AS ( INSERT INTO guidebook.stopareas - (navitia_id, stoparea_name, line, operator, geom) - VALUES (:stop_id, :stop_name, :line, :operator, - ST_Transform(ST_SetSRID(ST_MakePoint(:lon_stop, :lat_stop), 4326), 3857)) + (navitia_id, stoparea_name, line, operator, geom) + VALUES (:stop_id, :stop_name, :line, :operator, + ST_Transform( + ST_SetSRID( + ST_MakePoint( + :lon_stop, + :lat_stop + ), 4326 + ),3857 + ) + ) RETURNING stoparea_id ) INSERT INTO guidebook.waypoints_stopareas (stoparea_id, waypoint_id, distance) SELECT stoparea_id, :waypoint_id, :distance_km FROM new_stoparea - """ # noqa: E501, W291 + """ ) connection.execute( @@ -321,7 +347,8 @@ def process_new_waypoint(mapper, connection, geometry): { "stop_id": stop_id, "stop_name": stop_name, - "line": f"{mode} {line_name} - {line_full_name}", + "line": "%s %s - %s" % + (mode, line_name, line_full_name), "operator": operator_name, "lon_stop": lon_stop, "lat_stop": lat_stop, @@ -348,7 +375,7 @@ def process_new_waypoint(mapper, connection, geometry): }, ) - log.warning(f"Traitement terminé pour le waypoint {waypoint_id}") + log.info("Traitement terminé pour le waypoint %d", waypoint_id) # pylint: disable=too-complex,too-many-branches,too-many-statements @@ -363,7 +390,7 @@ def calculate_route_duration(mapper, connection, route): jour du script bash. """ route_id = route.document_id - log.warn(f"Calculating duration for route ID: {route_id}") + log.info("Calculating duration for route ID: %d", route_id) # Récupération des activités et normalisation des dénivelés activities = route.activities if route.activities is not None else [] @@ -406,16 +433,22 @@ def _get_climbing_activities(): ] -def _calculate_min_duration_for_activities(route, activities, height_diff_up, height_diff_down, route_id): # noqa: E501 +def _calculate_min_duration_for_activities( + route, activities, height_diff_up, height_diff_down, route_id +): """Calcule la durée minimale parmi toutes les activités.""" min_duration = None climbing_activities = _get_climbing_activities() for activity in activities: if activity in climbing_activities: - dm = _calculate_climbing_duration(route, height_diff_up, height_diff_down, route_id, activity) # noqa: E501 + dm = _calculate_climbing_duration( + route, height_diff_up, height_diff_down, route_id, activity + ) else: - dm = _calculate_standard_duration(activity, route, height_diff_up, height_diff_down, route_id) # noqa: E501 + dm = _calculate_standard_duration( + activity, route, height_diff_up, height_diff_down, route_id + ) if dm is not None and (min_duration is None or dm < min_duration): min_duration = dm @@ -423,64 +456,82 @@ def _calculate_min_duration_for_activities(route, activities, height_diff_up, he return min_duration -def _calculate_climbing_duration(route, height_diff_up, height_diff_down, route_id, activity): # noqa: E501 - """Calcule la durée pour les activités de grimpe selon la logique du bash.""" # noqa: E501 - v_diff = 50.0 # Vitesse ascensionnelle pour les difficultés (m/h) +def _calculate_climbing_duration( + route, height_diff_up, height_diff_down, route_id, activity +): + """ + Compute duration for climbing activities according to bash script + """ + v_diff = 50.0 # Climbing ascent speed for difficulties (m/h) - h = float(route.route_length if route.route_length is not None else 0) / 1000 # km # noqa: E501 + h = float( + route.route_length if route.route_length is not None else 0 + ) / 1000 # km dp = float(height_diff_up if height_diff_up is not None else 0) # m dn = float(height_diff_down if height_diff_down is not None else 0) # m difficulties_height = getattr(route, "height_diff_difficulties", None) - # CAS 1: Le dénivelé des difficultés n'est pas renseigné + # CASE 1: difficulties height is not provided if difficulties_height is None or difficulties_height <= 0: - # On considère que tout l'itinéraire est grimpant et sans approche + # Consider the whole route as climbing with no approach if dp <= 0: - return None # Pas de données utilisables pour le calcul + return None # No usable data for calculation dm = dp / v_diff - log.warn(f"Calculated climbing route duration for route {route_id} (activity {activity}, no difficulties_height): {dm:.2f} hours") # noqa: E501 + log.info( + "Calculated climbing route duration for route %d \ + (activity %s, no difficulties_height): %0.2f hours", + route_id, activity, dm + ) return dm - # CAS 2: Le dénivelé des difficultés est renseigné + # CASE 2: difficulties height is provided d_diff = float(difficulties_height) - # Vérification de cohérence + # Consistency check if dp > 0 and d_diff > dp: - log.warn(f"Route {route_id}: Inconsistent difficulties_height ({d_diff}m) > height_diff_up ({dp}m). Returning NULL.") # noqa: E501 + log.info("Route %d: Inconsistent difficulties_height (%fm) > \ + height_diff_up ( %fm). Returning NULL.", route_id, d_diff, dp + ) return None - # Calcul du temps des difficultés + # Compute time for difficulties t_diff = d_diff / v_diff - # Calcul du dénivelé de l'approche + # Compute approach height d_app = max(dp - d_diff, 0) - # Calcul du temps d'approche + # Compute approach time if d_app > 0: t_app = _calculate_approach_time(h, d_app, dn) else: t_app = 0 - # Calcul final selon le cadrage: max(t_diff, t_app) + 0.5 * min(t_diff, t_app) # noqa: E501 + # Final calculation according to rule: + # max(t\_diff, t\_app) + 0.5 * min(t\_diff, t\_app) dm = max(t_diff, t_app) + 0.5 * min(t_diff, t_app) - log.warn(f"Calculated climbing route duration for route {route_id} (activity {activity}): {dm:.2f} hours (t_diff={t_diff:.2f}, t_app={t_app:.2f})") # noqa: E501 + log.info("Calculated climbing route duration for route \ + %d (activity %s): %.2f hours (t_diff=%.2f, t_app=%.2f)" % ( + route_id, activity, dm, t_diff, t_app) + ) return dm def _calculate_approach_time(h, d_app, dn): - """Calcule le temps d'approche pour la grimpe selon la formule DIN 33466.""" # noqa: E501 - # Paramètres pour l'approche (randonnée) - v = 5.0 # km/h (vitesse horizontale) - a = 300.0 # m/h (montée) - d = 500.0 # m/h (descente) - - dh_app = h / v # Composante horizontale de l'approche - dv_app = (d_app / a) + (dn / d) # Composante verticale de l'approche (montée + descente) # noqa: E501 - - # Appliquer la formule DIN 33466 pour le temps d'approche + """Calculate the approach time for climbing using the DIN 33466 formula.""" + # Parameters for the approach (hiking) + v = 5.0 # km/h (horizontal speed) + a = 300.0 # m/h (ascent rate) + d = 500.0 # m/h (descent rate) + + # Horizontal component of the approach + dh_app = h / v + # Vertical component of the approach (ascent + descent) + dv_app = (d_app / a) + (dn / d) + + # Apply the DIN 33466 formula for approach time if dh_app < dv_app: t_app = dv_app + (dh_app / 2) else: @@ -490,49 +541,64 @@ def _calculate_approach_time(h, d_app, dn): def _get_activity_parameters(activity): - """Retourne les paramètres de vitesse selon l'activité.""" + """Return speed parameters for the activity.""" parameters = { "hiking": (5.0, 300.0, 500.0), "snowshoeing": (4.5, 250.0, 400.0), "skitouring": (5.0, 300.0, 1500.0), "mountain_biking": (15.0, 250.0, 1000.0), } - return parameters.get(activity, (5.0, 300.0, 500.0)) # Valeurs par défaut + return parameters.get(activity, (5.0, 300.0, 500.0)) # default values -def _calculate_standard_duration(activity, route, height_diff_up, height_diff_down, route_id): # noqa: E501 - """Calcule la durée pour les activités standard (non grimpantes) selon DIN 33466.""" # noqa: E501 +def _calculate_standard_duration( + activity, route, height_diff_up, height_diff_down, route_id +): + """Calculate duration for standard (non-climbing) activities + according to DIN 33466.""" v, a, d = _get_activity_parameters(activity) - h = float(route.route_length if route.route_length is not None else 0) / 1000 # km # noqa: E501 + h = float( + route.route_length if route.route_length is not None else 0 + ) / 1000 # km dp = float(height_diff_up if height_diff_up is not None else 0) # m dn = float(height_diff_down if height_diff_down is not None else 0) # m - dh = h / v # durée basée sur la distance horizontale - dv = (dp / a) + (dn / d) # durée basée sur les dénivelés + dh = h / v # duration based on horizontal distance + dv = (dp / a) + (dn / d) # duration based on elevation changes - # Calcul de la durée finale en heures selon DIN 33466 + # Final duration in hours according to DIN 33466 if dh < dv: dm = dv + (dh / 2) else: dm = (dv / 2) + dh - log.warn(f"Calculated standard route duration for route {route_id} (activity {activity}): {dm:.2f} hours") # noqa: E501 + log.info( + "Calculated standard route duration for route %s \ + (activity %s): %.2f hours" + % (route_id, activity, dm)) return dm def _validate_and_convert_duration(min_duration, route_id): - """Valide la durée calculée et la convertit en jours.""" + """Validate the calculated duration and convert it to days.""" min_duration_hours = 0.5 # 30 minutes - max_duration_hours = 18.0 # 18 heures + max_duration_hours = 18.0 # 18 hours if ( min_duration is None or min_duration < min_duration_hours or min_duration > max_duration_hours ): - log.warn( - f"Route {route_id}: Calculated duration ({min_duration:.2f} hours if not None) is out of bounds (min={min_duration_hours}h, max={max_duration_hours}h) or NULL. Setting duration to NULL." # noqa: E501 + min_duration_str = "None" if min_duration is None else "%0.2f", + min_duration + log.info( + "Route %d: Calculated duration (min_duration=%s) is out of bounds \ + (min=%fh, max=%fh) or NULL. Setting duration to NULL." % + (route_id, + min_duration_str, + min_duration_hours, + max_duration_hours) ) return None @@ -540,7 +606,7 @@ def _validate_and_convert_duration(min_duration, route_id): def _update_route_duration(connection, route_id, calculated_duration_in_days): - """Met à jour la durée calculée dans la base de données.""" + """Update the calculated duration in the database.""" connection.execute( text( """ @@ -551,6 +617,7 @@ def _update_route_duration(connection, route_id, calculated_duration_in_days): ), {"duration": calculated_duration_in_days, "route_id": route_id}, ) - log.warn( - f"Route {route_id}: Database updated with calculated_duration = {calculated_duration_in_days} days." # noqa: E501 + log.info( + "Route %d: Database updated with calculated_duration = %f days.", + calculated_duration_in_days ) diff --git a/c2corg_api/models/__init__.py b/c2corg_api/models/__init__.py index f3f8d4b5c..e6cb04924 100644 --- a/c2corg_api/models/__init__.py +++ b/c2corg_api/models/__init__.py @@ -24,6 +24,7 @@ class BaseMixin(object): # all models, for which tables should be created, must be listed here: from c2corg_api.models import document # noqa +from c2corg_api.models import coverage # noqa from c2corg_api.models import waypoint # noqa from c2corg_api.models import route # noqa from c2corg_api.models import document_history # noqa @@ -59,6 +60,7 @@ class BaseMixin(object): topo_map.MAP_TYPE: topo_map.TopoMap, area.AREA_TYPE: area.Area, outing.OUTING_TYPE: outing.Outing, + coverage.COVERAGE_TYPE: coverage.Coverage, } document_locale_types = { @@ -72,4 +74,5 @@ class BaseMixin(object): topo_map.MAP_TYPE: document.DocumentLocale, area.AREA_TYPE: document.DocumentLocale, outing.OUTING_TYPE: outing.OutingLocale, + coverage.COVERAGE_TYPE: document.DocumentLocale, } diff --git a/c2corg_api/models/common/attributes.py b/c2corg_api/models/common/attributes.py index ecf4dafca..6f21de152 100644 --- a/c2corg_api/models/common/attributes.py +++ b/c2corg_api/models/common/attributes.py @@ -785,3 +785,11 @@ 'highline', 'waterline' ] + +coverage_types = [ + 'fr-idf', + 'fr-ne', + 'fr-nw', + 'fr-se', + 'fr-sw' +] diff --git a/c2corg_api/models/common/document_types.py b/c2corg_api/models/common/document_types.py index d7fb2a854..3425dc919 100644 --- a/c2corg_api/models/common/document_types.py +++ b/c2corg_api/models/common/document_types.py @@ -9,6 +9,7 @@ WAYPOINT_TYPE = 'w' BOOK_TYPE = 'b' XREPORT_TYPE = 'x' +COVERAGE_TYPE = 'v' ALL = [ AREA_TYPE, ARTICLE_TYPE, IMAGE_TYPE, MAP_TYPE, OUTING_TYPE, ROUTE_TYPE, diff --git a/c2corg_api/models/common/fields_coverage.py b/c2corg_api/models/common/fields_coverage.py new file mode 100644 index 000000000..4aaacfee9 --- /dev/null +++ b/c2corg_api/models/common/fields_coverage.py @@ -0,0 +1,21 @@ +DEFAULT_FIELDS = [ + 'coverage_type' + 'geometry.geom_detail' +] + +DEFAULT_REQUIRED = [ + 'coverage_type', + 'geometry', + 'geometry.geom_detail' +] + +LISTING_FIELDS = [ + 'coverage_type', + 'geometry.geom_detail' +] + +fields_coverage = { + 'fields': DEFAULT_FIELDS, + 'required': DEFAULT_REQUIRED, + 'listing': LISTING_FIELDS +} diff --git a/c2corg_api/models/coverage.py b/c2corg_api/models/coverage.py new file mode 100644 index 000000000..17004c7e3 --- /dev/null +++ b/c2corg_api/models/coverage.py @@ -0,0 +1,96 @@ +from c2corg_api.models import Base, schema +from c2corg_api.models.enums import coverage_types +from c2corg_api.models.document import ( + schema_document_locale, + ArchiveDocument, + Document, + get_geometry_schema_overrides, + schema_attributes) +from c2corg_api.models.schema_utils import get_update_schema, \ + get_create_schema, restrict_schema +from c2corg_api.models.utils import copy_attributes +from c2corg_api.models.common.fields_coverage import fields_coverage +from colanderalchemy import SQLAlchemySchemaNode +from sqlalchemy import ( + Column, + Integer, + ForeignKey +) +from c2corg_api.models.common import document_types + +COVERAGE_TYPE = document_types.COVERAGE_TYPE + + +class _CoverageMixin(object): + coverage_type = Column(coverage_types) + + +attributes = ['coverage_type'] + + +class Coverage(_CoverageMixin, Document): + __tablename__ = 'coverages' + + document_id = Column( + Integer, + ForeignKey(schema + '.documents.document_id'), primary_key=True) + + __mapper_args__ = { + 'polymorphic_identity': COVERAGE_TYPE, + 'inherit_condition': Document.document_id == document_id + } + + def to_archive(self): + coverage = ArchiveCoverage() + super(Coverage, self)._to_archive(coverage) + copy_attributes(self, coverage, attributes) + + return coverage + + def update(self, other): + super(Coverage, self).update(other) + copy_attributes(other, self, attributes) + + +schema_coverage_locale = schema_document_locale +schema_coverage_attributes = list(schema_attributes) + + +class ArchiveCoverage(_CoverageMixin, ArchiveDocument): + """ + """ + __tablename__ = 'coverages_archives' + + id = Column( + Integer, + ForeignKey(schema + '.documents_archives.id'), primary_key=True) + + __mapper_args__ = { + 'polymorphic_identity': COVERAGE_TYPE, + 'inherit_condition': ArchiveDocument.id == id + } + + __table_args__ = Base.__table_args__ + + +schema_coverage = SQLAlchemySchemaNode( + Coverage, + # whitelisted attributes + includes=schema_coverage_attributes + attributes, + overrides={ + 'document_id': { + 'missing': None + }, + 'version': { + 'missing': None + }, + 'locales': { + 'children': [schema_coverage_locale] + }, + 'geometry': get_geometry_schema_overrides(['POLYGON']) + }) + +schema_create_coverage = get_create_schema(schema_coverage) +schema_update_coverage = get_update_schema(schema_coverage) +schema_listing_coverage = restrict_schema( + schema_coverage, fields_coverage.get('listing')) diff --git a/c2corg_api/models/enums.py b/c2corg_api/models/enums.py index e0eb5a6ac..b57469310 100644 --- a/c2corg_api/models/enums.py +++ b/c2corg_api/models/enums.py @@ -159,3 +159,5 @@ def enum(name, types): 'snow_quality_ratings', attributes.snow_quality_ratings) snow_quantity_ratings = enum( 'snow_quantity_ratings', attributes.snow_quantity_ratings) +coverage_types = enum( + 'coverage_types', attributes.coverage_types) diff --git a/c2corg_api/models/schema_utils.py b/c2corg_api/models/schema_utils.py index 8f30dcd26..3871df4d6 100644 --- a/c2corg_api/models/schema_utils.py +++ b/c2corg_api/models/schema_utils.py @@ -89,6 +89,8 @@ class SchemaAssociations(MappingSchema): Sequence(), SchemaAssociationDoc(), missing=None) outings = SchemaNode( Sequence(), SchemaAssociationDoc(), missing=None) + coverages = SchemaNode( + Sequence(), SchemaAssociationDoc(), missing=None) def get_create_schema(document_schema): diff --git a/c2corg_api/scripts/migration/documents/coverage.py b/c2corg_api/scripts/migration/documents/coverage.py new file mode 100644 index 000000000..aa53aa0ed --- /dev/null +++ b/c2corg_api/scripts/migration/documents/coverage.py @@ -0,0 +1,90 @@ +from c2corg_api.models import enums +from c2corg_api.models.coverage import ArchiveCoverage, Coverage, COVERAGE_TYPE +from c2corg_api.models.document import DocumentLocale, ArchiveDocumentLocale, \ + DOCUMENT_TYPE +from c2corg_api.scripts.migration.documents.document import MigrateDocuments, \ + DEFAULT_QUALITY + + +class MigrateCoverages(MigrateDocuments): + + def get_name(self): + return 'coverages' + + def get_model_document(self, locales): + return DocumentLocale if locales else Coverage + + def get_model_archive_document(self, locales): + return ArchiveDocumentLocale if locales else ArchiveCoverage + + def get_document_geometry(self, document_in, version): + return dict( + document_id=document_in.id, + id=document_in.id, + version=version, + geom_detail=document_in.geom + ) + + def get_count_query(self): + return ( + 'select count(*) ' + 'from app_coverages_archives aa join coverages a on aa.id = a.id ' + 'where a.redirects_to is null;' + ) + + def get_query(self): + return ( + 'select ' + ' aa.id, aa.document_archive_id, aa.is_latest_version, ' + ' aa.is_protected, aa.redirects_to, ' + ' ST_Force2D(ST_SetSRID(aa.geom, 3857)) geom, ' + ' aa.coverage_type ' + 'from app_coverages_archives aa join coverages a on aa.id = a.id ' + 'where a.redirects_to is null ' + 'order by aa.id, aa.document_archive_id;' + ) + + def get_count_query_locales(self): + return ( + 'select count(*) ' + 'from app_coverages_i18n_archives aa ' + 'join coverages a on aa.id = a.id ' + 'where a.redirects_to is null;' + ) + + def get_query_locales(self): + return ( + 'select ' + ' aa.id, aa.document_i18n_archive_id, aa.is_latest_version, ' + ' aa.culture, aa.name, aa.description ' + 'from app_coverages_i18n_archives aa ' + 'join coverages a on aa.id = a.id ' + 'where a.redirects_to is null ' + 'order by aa.id, aa.culture, aa.document_i18n_archive_id;' + ) + + def get_document(self, document_in, version): + return dict( + document_id=document_in.id, + type=COVERAGE_TYPE, + version=version, + protected=document_in.is_protected, + redirects_to=document_in.redirects_to, + coverage_type=self.convert_type( + document_in.coverage_type, enums.coverage_types), + quality=DEFAULT_QUALITY + ) + + def get_document_locale(self, document_in, version): + description = self.convert_tags(document_in.description) + description, summary = self.extract_summary(description) + return dict( + document_id=document_in.id, + id=document_in.document_i18n_archive_id, + type=DOCUMENT_TYPE, + version=version, + lang=document_in.culture, + title=document_in.name, + description=description, + summary=summary + ) diff --git a/c2corg_api/scripts/migration/migrate.py b/c2corg_api/scripts/migration/migrate.py index 818a03295..9c8b2bdd3 100644 --- a/c2corg_api/scripts/migration/migrate.py +++ b/c2corg_api/scripts/migration/migrate.py @@ -6,6 +6,7 @@ MigrateAreaAssociations from c2corg_api.scripts.migration.climbing_site_routes import \ CreateClimbingSiteRoutes +from c2corg_api.scripts.migration.documents.coverage import MigrateCoverages from c2corg_api.scripts.migration.documents.xreports import MigrateXreports from c2corg_api.scripts.migration.documents.area import MigrateAreas from c2corg_api.scripts.migration.documents.associations import \ @@ -97,6 +98,7 @@ def main(argv=sys.argv): MigrateBooks(connection_source, session, batch_size).migrate() MigrateVersions(connection_source, session, batch_size).migrate() MigrateAssociations(connection_source, session, batch_size).migrate() + MigrateCoverages(connection_source, session, batch_size).migrate() CreateClimbingSiteRoutes(connection_source, session, batch_size).migrate() SetRouteTitlePrefix(connection_source, session, batch_size).migrate() SetDefaultGeometries(connection_source, session, batch_size).migrate() diff --git a/c2corg_api/search/__init__.py b/c2corg_api/search/__init__.py index 8b2186bc6..7b81cf5d1 100644 --- a/c2corg_api/search/__init__.py +++ b/c2corg_api/search/__init__.py @@ -9,6 +9,7 @@ from c2corg_api.models.user_profile import USERPROFILE_TYPE from c2corg_api.models.waypoint import WAYPOINT_TYPE from c2corg_api.models.xreport import XREPORT_TYPE +from c2corg_api.models.coverage import COVERAGE_TYPE from c2corg_api.search.mappings.area_mapping import SearchArea from c2corg_api.search.mappings.article_mapping import SearchArticle from c2corg_api.search.mappings.book_mapping import SearchBook @@ -19,6 +20,7 @@ from c2corg_api.search.mappings.user_mapping import SearchUser from c2corg_api.search.mappings.waypoint_mapping import SearchWaypoint from c2corg_api.search.mappings.xreport_mapping import SearchXreport +from c2corg_api.search.mappings.coverage_mapping import SearchCoverage from elasticsearch import Elasticsearch from elasticsearch_dsl import Search from elasticsearch_dsl.connections import connections @@ -114,5 +116,6 @@ def get_text_query_on_title(search_term, search_lang=None): ROUTE_TYPE: SearchRoute, MAP_TYPE: SearchTopoMap, USERPROFILE_TYPE: SearchUser, - WAYPOINT_TYPE: SearchWaypoint + WAYPOINT_TYPE: SearchWaypoint, + COVERAGE_TYPE: SearchCoverage, } diff --git a/c2corg_api/search/advanced_search.py b/c2corg_api/search/advanced_search.py index 3cf2dc450..9abd2c931 100644 --- a/c2corg_api/search/advanced_search.py +++ b/c2corg_api/search/advanced_search.py @@ -27,6 +27,73 @@ def search(url_params, meta_params, doc_type): return document_ids, total +def search_with_ids(url_params, meta_params, doc_type, id_chunk=None): + """Builds a query from the URL parameters and a list of ids + and return a tuple (document_ids, total) received from ElasticSearch. + """ + query = build_query(url_params, meta_params, doc_type) + search_dict = query.to_dict() + + # Inject a terms filter for the chunk of IDs if provided + if id_chunk: + id_chunk_str = [str(document_id) for document_id in id_chunk] + terms_filter = {"terms": {"_id": id_chunk_str}} + if "bool" not in search_dict.get("query", {}): + search_dict["query"] = {"bool": {"filter": []}} + elif "filter" not in search_dict["query"]["bool"]: + search_dict["query"]["bool"]["filter"] = [] + + search_dict["query"]["bool"]["filter"].append(terms_filter) + + search_dict["from"] = 0 + search_dict["size"] = len(id_chunk) + + query.update_from_dict(search_dict) + + response = query.execute() + document_ids = [int(doc.meta.id) for doc in response] + total = response.hits.total + + return document_ids, total + + +def get_all_filtered_docs( + params, + meta_params, + ids, + keep_order, + doc_type +): + """get all docs ids, taking into account ES filter in params""" + filtered_doc_ids = [] + total_hits = 0 + + # use elastic search to apply filters + # to documents of type ids + # do it by chunk of size 'limit' + for idx, id_chunk in enumerate(chunk_ids( + ids, + chunk_size=(len(ids) if keep_order else 100) + ), start=1): + doc_ids, hits = search_with_ids( + params, + meta_params, + doc_type=doc_type, + id_chunk=id_chunk + ) + filtered_doc_ids.extend(doc_ids) + total_hits += hits + + return filtered_doc_ids, total_hits + + +def chunk_ids(ids_set, chunk_size=100): + """Yield successive chunks of IDs from a set/list.""" + ids_list = list(ids_set) + for i in range(0, len(ids_list), chunk_size): + yield ids_list[i:i + chunk_size] + + def contains_search_params(url_params): """Checks if the url query string contains other parameters than meta-data parameters (like offset, limit, preferred language). diff --git a/c2corg_api/search/mappings/coverage_mapping.py b/c2corg_api/search/mappings/coverage_mapping.py new file mode 100644 index 000000000..2e3a1af04 --- /dev/null +++ b/c2corg_api/search/mappings/coverage_mapping.py @@ -0,0 +1,29 @@ +from c2corg_api.models.coverage import COVERAGE_TYPE, Coverage +from c2corg_api.search.mapping import SearchDocument, BaseMeta +from c2corg_api.search.mapping_types import QueryableMixin, QEnum + + +class SearchCoverage(SearchDocument): + class Meta(BaseMeta): + doc_type = COVERAGE_TYPE + + coverage_type = QEnum('ctyp', model_field=Coverage.coverage_type) + + FIELDS = ['coverage_type'] + + @staticmethod + def to_search_document(document, index): + search_document = SearchDocument.to_search_document( + document, index, include_areas=False) + + if document.redirects_to: + return search_document + + SearchDocument.copy_fields( + search_document, document, SearchCoverage.FIELDS) + + return search_document + + +SearchCoverage.queryable_fields = \ + QueryableMixin.get_queryable_fields(SearchCoverage) diff --git a/c2corg_api/search/utils.py b/c2corg_api/search/utils.py index e06b27a6c..6660599e9 100644 --- a/c2corg_api/search/utils.py +++ b/c2corg_api/search/utils.py @@ -1,4 +1,6 @@ +import logging import re +log = logging.getLogger(__name__) BBCODE_TAGS = [ 'b', 'i', 'u', 's', 'q', 'c', 'sup', 'ind', 'url', 'email', 'acr(onym)?', diff --git a/c2corg_api/views/coverage.py b/c2corg_api/views/coverage.py new file mode 100644 index 000000000..407edcd35 --- /dev/null +++ b/c2corg_api/views/coverage.py @@ -0,0 +1,142 @@ + +from c2corg_api.views.validation import validate_associations, \ + validate_pagination, validate_preferred_lang_param +from c2corg_api.views import cors_policy, restricted_json_view +from cornice.resource import resource, view +from cornice.validators import colander_body_validator +from shapely.geometry import Point, shape +from c2corg_api.views.document import DocumentRest, make_validator_create, \ + make_validator_update +from c2corg_api.views.validation import validate_cook_param, validate_id, \ + validate_lang_param +from c2corg_api.views.document_schemas import coverage_documents_config +from c2corg_api.models.utils import wkb_to_shape +import functools +import json +import logging +from c2corg_api.models import DBSession +from c2corg_api.models.common.fields_coverage import fields_coverage +from c2corg_api.models.coverage import COVERAGE_TYPE, Coverage, \ + schema_coverage, schema_create_coverage, schema_update_coverage + + +log = logging.getLogger(__name__) + +validate_coverage_create = make_validator_create( + fields_coverage.get('required')) +validate_coverage_update = make_validator_update( + fields_coverage.get('required')) +validate_associations_create = functools.partial( + validate_associations, COVERAGE_TYPE, True) +validate_associations_update = functools.partial( + validate_associations, COVERAGE_TYPE, False) + + +@resource(collection_path='/coverages', path='/coverages/{id}', + cors_policy=cors_policy) +class CoverageRest(DocumentRest): + + def __init__(self, request, context=None): + self.request = request + + @view(validators=[validate_pagination, validate_preferred_lang_param]) + def collection_get(self): + return self._collection_get(COVERAGE_TYPE, coverage_documents_config) + + @view(validators=[validate_id, validate_lang_param, validate_cook_param]) + def get(self): + return self._get( + coverage_documents_config, schema_coverage, include_areas=False) + + @restricted_json_view( + schema=schema_create_coverage, + validators=[ + colander_body_validator, + validate_coverage_create, + validate_associations_create]) + def collection_post(self): + return self._collection_post(schema_coverage, allow_anonymous=False) + + @restricted_json_view( + schema=schema_update_coverage, + validators=[ + colander_body_validator, + validate_id, + validate_coverage_update, + validate_associations_update]) + def put(self): + return self._put(Coverage, schema_coverage) + + +@resource(path='/getcoverage', cors_policy=cors_policy) +class WaypointCoverageRest(DocumentRest): + + def __init__(self, request, context=None): + self.request = request + + @view(validators=[]) + def get(self): + """Returns the coverage from a longitude and a latitude""" + + lon = float(self.request.GET['lon']) + lat = float(self.request.GET['lat']) + + return get_coverage(lon, lat) + + +@resource(path='/getpolygoncoverage', cors_policy=cors_policy) +class PolygonCoverage(DocumentRest): + + def __init__(self, request, context=None): + self.request = request + + @view(validators=[]) + def post(self): + """Returns the coverages from a geom_detail type polygon + (geom_detail has to be EPSG 4326 since isochrone is 4326)""" + geom_detail = json.loads( + (json.loads(self.request.body)['geom_detail'])) + polygon = shape(geom_detail) + return get_coverages(polygon) + + +def get_coverage(lon, lat): + """get the coverage that contains a point(lon, lat)""" + pt = Point(lon, lat) + + coverage_found = None + + coverages = DBSession.query(Coverage).all() + + for coverage in coverages: + geom = coverage.geometry.geom_detail + + # convert WKB → Shapely polygon + poly = wkb_to_shape(geom) + + if poly.contains(pt): + coverage_found = coverage + break + + if (coverage_found): + return coverage_found.coverage_type + else: + return None + + +def get_coverages(polygon): + """get all the coverages that intersects a polygon""" + coverage_found = [] + + coverages = DBSession.query(Coverage).all() + + for coverage in coverages: + geom = coverage.geometry.geom_detail + + # convert WKB → Shapely polygon + poly = wkb_to_shape(geom) + + if poly.contains(polygon) or poly.intersects(polygon): + coverage_found.append(coverage.coverage_type) + + return coverage_found diff --git a/c2corg_api/views/document_delete.py b/c2corg_api/views/document_delete.py index 6d557f6df..107b42710 100644 --- a/c2corg_api/views/document_delete.py +++ b/c2corg_api/views/document_delete.py @@ -1,3 +1,4 @@ +from c2corg_api.models.coverage import COVERAGE_TYPE, ArchiveCoverage, Coverage from c2corg_api.security.acl import ACLDefault from c2corg_api.models import DBSession, article, image from c2corg_api.models.area_association import AreaAssociation @@ -352,6 +353,8 @@ def _get_models(document_type): return Article, None, ArchiveArticle, None if document_type == BOOK_TYPE: return Book, None, ArchiveBook, None + if document_type == COVERAGE_TYPE: + return Coverage, None, ArchiveCoverage, None if document_type == XREPORT_TYPE: return Xreport, XreportLocale, ArchiveXreport, ArchiveXreportLocale assert False diff --git a/c2corg_api/views/document_schemas.py b/c2corg_api/views/document_schemas.py index db08b12b9..90a842736 100644 --- a/c2corg_api/views/document_schemas.py +++ b/c2corg_api/views/document_schemas.py @@ -2,6 +2,8 @@ from c2corg_api.models.article import ARTICLE_TYPE, Article, \ schema_listing_article from c2corg_api.models.book import BOOK_TYPE, Book, schema_listing_book +from c2corg_api.models.coverage import COVERAGE_TYPE, Coverage, \ + schema_listing_coverage from c2corg_api.models.image import IMAGE_TYPE, Image, schema_listing_image from c2corg_api.models.outing import OUTING_TYPE, Outing, schema_outing from c2corg_api.models.xreport import XREPORT_TYPE, Xreport, \ @@ -27,6 +29,7 @@ from c2corg_api.models.common.fields_image import fields_image from c2corg_api.models.common.fields_topo_map import fields_topo_map from c2corg_api.models.common.fields_user_profile import fields_user_profile +from c2corg_api.models.common.fields_coverage import fields_coverage from functools import lru_cache @@ -253,6 +256,10 @@ def adapt_waypoint_schema_for_type(waypoint_type, field_list_type): WAYPOINT_TYPE, Waypoint, schema_waypoint, clazz_locale=WaypointLocale, fields=fields_waypoint, adapt_schema=waypoint_listing_schema_adaptor) +# coverages +coverage_documents_config = GetDocumentsConfig( + COVERAGE_TYPE, Coverage, schema_listing_coverage, + listing_fields=fields_coverage['listing']) document_configs = { WAYPOINT_TYPE: waypoint_documents_config, @@ -264,5 +271,6 @@ def adapt_waypoint_schema_for_type(waypoint_type, field_list_type): XREPORT_TYPE: xreport_documents_config, MAP_TYPE: topo_map_documents_config, ARTICLE_TYPE: article_documents_config, - USERPROFILE_TYPE: user_profile_documents_config + USERPROFILE_TYPE: user_profile_documents_config, + COVERAGE_TYPE: coverage_documents_config } diff --git a/c2corg_api/views/navitia.py b/c2corg_api/views/navitia.py index aaa44f7bd..642d81b84 100644 --- a/c2corg_api/views/navitia.py +++ b/c2corg_api/views/navitia.py @@ -1,8 +1,42 @@ +import json +import logging import os import requests -from pyramid.httpexceptions import HTTPBadRequest, HTTPInternalServerError # noqa: E501 +import redis +import uuid +import time +import threading +import ast +from c2corg_api.models import DBSession +from c2corg_api.models.area import Area +from c2corg_api.models.utils import wkb_to_shape +from c2corg_api.models.waypoint import Waypoint, schema_waypoint +from c2corg_api.views.coverage import get_coverage +from c2corg_api.views.waypoint import build_reachable_waypoints_query +from c2corg_api.views.route import build_reachable_route_query_with_waypoints +from shapely.geometry import Point +from pyramid.httpexceptions import HTTPBadRequest, HTTPInternalServerError +from pyramid.response import Response from cornice.resource import resource, view -from c2corg_api.views import cors_policy +from c2corg_api.views import cors_policy, to_json_dict +from c2corg_api.models.route import schema_route +from c2corg_api.models.area import schema_listing_area +from shapely.geometry import shape +from pyproj import Transformer + + +log = logging.getLogger(__name__) + +# When editing these constants, make sure to edit them in the front too +# (itinevert-service) +MAX_ROUTE_THRESHOLD = 50 +MAX_TRIP_DURATION = 240 +MIN_TRIP_DURATION = 20 + +# redis to store job's value (progress, result, error...) +REDIS_HOST = "redis" +REDIS_PORT = 6379 +REDIS_DB = 0 def validate_navitia_params(request, **kwargs): @@ -14,7 +48,7 @@ def validate_navitia_params(request, **kwargs): request.errors.add( 'querystring', param, - f'Paramètre {param} requis') + 'Parameter %s required' % param) @resource(path='/navitia/journeys', cors_policy=cors_policy) @@ -34,74 +68,850 @@ def get(self): - datetime: date and hour (format ISO 8601) - datetime_represents: 'departure' or 'arrival' """ + # Retrieve navitia API Key from env var + api_key = os.getenv("NAVITIA_API_KEY") + if not api_key: + raise HTTPInternalServerError( + "Navitia API config is missing") + + # Build journey api parameters + params = { + "from": self.request.params.get("from"), + "to": self.request.params.get("to"), + "datetime": self.request.params.get("datetime"), + "datetime_represents": self.request.params.get( + "datetime_represents" + ), + } + + # Add optional params + optional_params = [ + "max_duration_to_pt", + "walking_speed", + "bike_speed", + "bss_speed", + "car_speed", + "forbidden_uris", + "allowed_id", + "first_section_mode", + "last_section_mode", + "max_walking_duration_to_pt", + "max_nb_transfers", + "min_nb_journeys", + "max_bike_duration_to_pt", + "max_bss_duration_to_pt", + "max_car_duration_to_pt", + "timeframe_duration", + "max_walking_direct_path_duration", + "wheelchair", + "traveler_type", + "data_freshness", + ] + + for param in optional_params: + if param in self.request.params: + params[param] = self.request.params.get(param) + + return navitia_get( + "https://api.navitia.io/v1/journeys", + params=params, + headers={"Authorization": api_key}, + ) + + +def validate_journey_reachable_params(request, **kwargs): + """Validates the required parameters for the journey reachable doc route""" + required_params = ['from', 'datetime', 'datetime_represents', + 'walking_speed', 'max_walking_duration_to_pt'] + + for param in required_params: + if param not in request.params: + request.errors.add( + 'querystring', + param, + 'Parameter %s required' % param) + + +@resource(path='/navitia/journeyreachableroutes/start', + cors_policy=cors_policy) +class StartNavitiaJourneyReachableRoutesRest: + def __init__(self, request, context=None): + self.request = request + + @view(validators=[validate_journey_reachable_params]) + def get(self): + """ + start job to retrieve journey reachable routes + returns job id + """ + return start_job_background( + compute_journey_reachable_routes, + self.request + ) + + +@resource(path='/navitia/journeyreachablewaypoints/start', + cors_policy=cors_policy) +class StartNavitiaJourneyReachableWaypointsRest: + def __init__(self, request, context=None): + self.request = request + + @view(validators=[validate_journey_reachable_params]) + def get(self): + """ + start job to retrieve journey reachable waypoints + returns job id + """ + return start_job_background( + compute_journey_reachable_waypoints, + self.request + ) + + +@resource(path='/navitia/journeyreachableroutes/result/{job_id}', + cors_policy=cors_policy) +class NavitiaJourneyReachableRoutesResultRest: + def __init__(self, request, context=None): + self.request = request + + @view() + def get(self): + """ + get the result of the job : get journey reachable routes + returns the result + """ + r = redis_client() + job_id = self.request.matchdict.get("job_id") + return read_result_from_redis(r, job_id) + + +@resource(path='/navitia/journeyreachablewaypoints/result/{job_id}', + cors_policy=cors_policy) +class NavitiaJourneyReachableWaypointsResultRest: + def __init__(self, request, context=None): + self.request = request + + @view() + def get(self): + """ + get the result of the job : get journey reachable waypoints + returns the result + """ + r = redis_client() + job_id = self.request.matchdict.get("job_id") + return read_result_from_redis(r, job_id) + + +@resource(path='/navitia/journeyreachableroutes/progress/{job_id}', + cors_policy=cors_policy) +class NavitiaJourneyReachableRoutesProgressRest: + def __init__(self, request, context=None): + self.request = request + + @view() + def get(self): + """ + monitor progress of job id for journey reachable routes + """ + r = redis_client() + job_id = self.request.matchdict.get("job_id") + return Response(app_iter=progress_stream(r, job_id), + content_type="text/event-stream") + + +@resource(path='/navitia/journeyreachablewaypoints/progress/{job_id}', + cors_policy=cors_policy) +class NavitiaJourneyReachableWaypointsProgressRest: + def __init__(self, request, context=None): + self.request = request + + @view() + def get(self): + """ + monitor progress of job id for journey reachable waypoints + """ + r = redis_client() + job_id = self.request.matchdict.get("job_id") + return Response(app_iter=progress_stream(r, job_id), + content_type="text/event-stream") + + +def compute_journey_reachable_routes(job_id, request): + """ + Get all waypoints matching filters in params, that are reachable + (means there exists a Navitia journey for at least one of + their waypoints of type access). + + NOTE : the number of routes after applying filters, + has to be < MAX_ROUTE_THRESHOLD, + to reduce number of queries towards Navitia journey API + + the result can be found inside redis + """ + r = redis_client() + try: + meta_params = extract_meta_params(request) + journey_params = extract_journey_params(request) + query, count = build_reachable_route_query_with_waypoints( + request.GET, + meta_params + ) + if query is None: + results = [] + else: + results = query.all() + + if len(results) > MAX_ROUTE_THRESHOLD: + raise HTTPBadRequest( + "Couldn't proceed with computation : Too much routes found.") + + areas_map = collect_areas_from_results(results, 1) + wp_objects = collect_waypoints_from_results(results) + + total = len(wp_objects) + log.info("Number of NAVITIA journey queries : %d", total) + r.set("job:%s:total" % job_id, total) + + count = found = not_found = 0 + navitia_wp_map = {} + + for wp in wp_objects: + result = is_wp_journey_reachable( + to_json_dict(wp, schema_waypoint), journey_params) + navitia_wp_map[wp.document_id] = result + count += 1 + if result: + found += 1 + else: + not_found += 1 + _store_job_progress(r, job_id, count, found, not_found) + + routes = [] + for route, areas, waypoints in results: + journey_exists = any(navitia_wp_map.get( + wp.get("document_id")) for wp in waypoints) + if not journey_exists: + continue + json_areas = [] + for area in (areas or []): + area_obj = areas_map.get(area.get("document_id")) + if area_obj: + json_areas.append(to_json_dict( + area_obj, schema_listing_area)) + route.areas = json_areas + routes.append(to_json_dict(route, schema_route, True)) + + r.set("job:%s:result" % job_id, json.dumps( + {'documents': routes, 'total': len(routes)})) + r.set("job:%s:status" % job_id, "done") + except Exception as exc: + log.exception(str(exc)) + r.set("job:%s:status" % job_id, "error") + r.set("job:%s:error" % job_id, str(exc)) + + +def compute_journey_reachable_waypoints(job_id, request): + """ + Get all routes matching filters in params, that are reachable + (means there exists a Navitia journey for at least one of + their waypoints of type access). + + NOTE : the waypoints have to be filtered by one area (and not more) + to reduce number of request towards Navitia Journey API + + the result can be found inside redis + """ + r = redis_client() + try: + meta_params = extract_meta_params(request) + journey_params = extract_journey_params(request) + + # Ensure areas filter is provided and normalized + areas_param = None + try: + areas_param = request.GET['a'] + if isinstance(areas_param, str): + areas_list = areas_param.split(",") + else: + areas_list = list(areas_param) + except Exception: + areas_list = None + + if areas_list is None: + raise HTTPBadRequest('Missing filter : area is required') + if len(areas_list) > 1: + raise HTTPBadRequest('Only one filtering area is allowed') + + query, count = build_reachable_waypoints_query( + request.GET, + meta_params + ) + results = query.all() + + areas_map = collect_areas_from_results(results, 1) + + total = len(results) + log.info("Number of NAVITIA journey queries : %d", total) + r.set("job:%s:total" % job_id, total) + + count = found = not_found = 0 + waypoints = [] + + for waypoint, areas in results: + count += 1 + r.publish("job:%s:events" % job_id, "not_found:%d" % not_found) + reachable = is_wp_journey_reachable(to_json_dict( + waypoint, schema_waypoint), journey_params) + if reachable: + found += 1 + json_areas = [] + for area in (areas or []): + area_obj = areas_map.get(area.get("document_id")) + if area_obj: + json_areas.append(to_json_dict( + area_obj, schema_listing_area)) + waypoint.areas = json_areas + waypoints.append(to_json_dict(waypoint, schema_waypoint, True)) + else: + not_found += 1 + _store_job_progress(r, job_id, count, found, not_found) + + r.set("job:%s:result" % job_id, json.dumps( + {'documents': waypoints, 'total': len(waypoints)})) + r.set("job:%s:status" % job_id, "done") + except Exception as exc: + log.exception("Error computing reachable waypoints") + r.set("job:%s:status" % job_id, "error") + r.set("job:%s:error" % job_id, str(exc)) + + +def validate_isochrone_reachable_params(request, **kwargs): + """Validates the required parameters + for the isochrone reachable doc route""" + required_params = ['from', 'datetime', + 'datetime_represents', 'boundary_duration'] + + for param in required_params: + if param not in request.params: + request.errors.add( + 'querystring', + param, + 'Parameter %s required' % param) + + +@resource(path='/navitia/isochronesreachableroutes', cors_policy=cors_policy) +class NavitiaIsochronesReachableRoutesRest: + def __init__(self, request, context=None): + self.request = request + + @view(validators=[validate_isochrone_reachable_params]) + def get(self): + """ + Get all routes matching filters in params, that have at least + one waypoint of type access that is inside the isochron. + The isochron is created by querying navitia api + with specific parameters, see validate_isochrone_reachable_params func + """ try: - # Récupération de la clé API depuis les variables d'environnement - api_key = os.getenv('NAVITIA_API_KEY') - if not api_key: - raise HTTPInternalServerError( - 'Configuration API Navitia manquante') - - # Construction des paramètres - params = { - 'from': self.request.params.get('from'), - 'to': self.request.params.get('to'), - 'datetime': self.request.params.get('datetime'), - 'datetime_represents': self.request.params.get('datetime_represents') # noqa: E501 + meta_params = extract_meta_params(self.request) + + isochrone_params = extract_isochrone_params(self.request) + + query, count = build_reachable_route_query_with_waypoints( + self.request.GET, + meta_params + ) + + if query is None: + results = [] + else: + results = query.all() + + areas_map = collect_areas_from_results(results, 1) + + response = get_navitia_isochrone(isochrone_params) + + routes = [] + geojson = "" + # if isochrone found + if (len(response["isochrones"]) > 0): + geojson = response["isochrones"][0]["geojson"] + isochrone_geom = shape(geojson) + + wp_objects = collect_waypoints_from_results(results) + + navitia_wp_map = {wp.document_id: is_wp_in_isochrone( + to_json_dict(wp, schema_waypoint), isochrone_geom + ) for wp in wp_objects} + + for route, areas, waypoints in results: + # check if a journey exists for route + # (at least one wp has a journey associated) + one_wp_in_isochrone = False + for wp in waypoints: + wp_id = wp.get("document_id") + one_wp_in_isochrone |= navitia_wp_map.get(wp_id) + + if one_wp_in_isochrone: + json_areas = [] + + if areas is None: + areas = [] + + for area in areas: + area_obj = areas_map.get(area.get("document_id")) + if area_obj: + json_areas.append(to_json_dict( + area_obj, schema_listing_area)) + + # assign JSON areas to the waypoint + route.areas = json_areas + route_dict = to_json_dict(route, schema_route, True) + routes.append(route_dict) + + return { + 'documents': routes, + 'total': len(routes), + 'isochron_geom': geojson } + except Exception as e: + return json.dumps(str(e)) + + +@resource( + path='/navitia/isochronesreachablewaypoints', + cors_policy=cors_policy +) +class NavitiaIsochronesReachableWaypointsRest: + def __init__(self, request, context=None): + self.request = request + + @view(validators=[validate_isochrone_reachable_params]) + def get(self): + """ + Get all waypoints matching filters in params, + that are inside the isochron. + The isochron is created by querying navitia api + with specific parameters, see validate_isochrone_reachable_params func + """ + try: - # Ajout des paramètres optionnels s'ils sont présents - optional_params = [ - 'max_duration_to_pt', - 'walking_speed', - 'bike_speed', - 'bss_speed', - 'car_speed', - 'forbidden_uris', - 'allowed_id', - 'first_section_mode', - 'last_section_mode', - 'max_walking_duration_to_pt', - 'max_nb_transfers', - 'min_nb_journeys', - 'max_bike_duration_to_pt', - 'max_bss_duration_to_pt', - 'max_car_duration_to_pt', - 'timeframe_duration', - 'max_walking_direct_path_duration', - 'wheelchair', - 'traveler_type', - 'data_freshness'] - - for param in optional_params: - if param in self.request.params: - params[param] = self.request.params.get(param) - - # Appel à l'API Navitia - response = requests.get( - 'https://api.navitia.io/v1/journeys', - params=params, - headers={'Authorization': api_key}, - timeout=30 + meta_params = extract_meta_params(self.request) + + isochrone_params = extract_isochrone_params(self.request) + + query, count = build_reachable_waypoints_query( + self.request.GET, meta_params ) - # Vérification du statut de la réponse - if response.status_code == 401: - raise HTTPInternalServerError( - 'Authentication error with Navitia API') - elif response.status_code == 400: - raise HTTPBadRequest('Invalid parameters for Navitia API') - elif response.status_code == 404: - return {} - elif not response.ok: - raise HTTPInternalServerError(f'Navitia API error: {response.status_code}') # noqa: E501 - - # Retour des données JSON - return response.json() - - except requests.exceptions.Timeout: - raise HTTPInternalServerError( - 'Timeout when calling the Navitia API') - except requests.exceptions.RequestException as e: - raise HTTPInternalServerError(f'Network error: {str(e)}') + results = query.all() + + # manage areas for waypoints + areas_map = collect_areas_from_results(results, 1) + + response = get_navitia_isochrone(isochrone_params) + + waypoints = [] + geojson = "" + # if isochrone found + if (len(response["isochrones"]) > 0): + geojson = response["isochrones"][0]["geojson"] + isochrone_geom = shape(geojson) + + for waypoint, areas in results: + # check if wp is in isochrone + if is_wp_in_isochrone( + to_json_dict(waypoint, schema_waypoint), isochrone_geom + ): + json_areas = [] + if areas is None: + areas = [] + + for area in areas: + area_obj = areas_map.get(area.get("document_id")) + if area_obj: + json_areas.append(to_json_dict( + area_obj, schema_listing_area)) + + # assign JSON areas to the waypoint + waypoint.areas = json_areas + wp_dict = to_json_dict(waypoint, schema_waypoint, True) + waypoints.append(wp_dict) + + return { + 'documents': waypoints, + 'total': len(waypoints), + "isochron_geom": geojson + } except Exception as e: - raise HTTPInternalServerError(f'Internal error: {str(e)}') + return json.dumps(str(e)) + + +@resource(path='/navitia/areainisochrone', cors_policy=cors_policy) +class AreaInIsochroneRest: + def __init__(self, request, context=None): + self.request = request + + @view(validators=[]) + def post(self): + """ + returns all areas that are inside + or that intersects an isochrone geometry + + make sure the geom_detail in body is epsg:3857 + """ + polygon = shape(json.loads(json.loads( + self.request.body)['geom_detail'])) + + query = ( + DBSession.query(Area).filter(Area.area_type == 'range') + ) + + results = query.all() + + areas = [] + + for area in results: + if (polygon.intersects(wkb_to_shape(area.geometry.geom_detail))): + areas.append(area.document_id) + + return areas + + +def is_wp_journey_reachable(waypoint, journey_params): + """ + Query the navitia Journey api and returns true + if the waypoint is reachable (at least one journey has been found) + NOTE : the journey's departure time has to be + the same day as the datetime's day in journey_params + """ + # enhance journey params with the 'to' parameter, + # from the waypoint geometry. + geom = shape(json.loads(waypoint.get("geometry").get("geom"))) + + src_epsg = 3857 + transformer = Transformer.from_crs( + "EPSG:%d" % src_epsg, "EPSG:4326", always_xy=True) + lon, lat = transformer.transform(geom.x, geom.y) + + journey_params['to'] = "%s;%s" % (lon, lat) + + destination_coverage = get_coverage(lon, lat) + + api_key = os.getenv("NAVITIA_API_KEY") + if not api_key: + raise HTTPInternalServerError("Navitia API config is missing") + + if destination_coverage: + url = "https://api.navitia.io/v1/coverage/%s/journeys" % \ + destination_coverage + else: + url = "https://api.navitia.io/v1/journeys" + + json_response = navitia_get( + url, + params=journey_params, + headers={"Authorization": api_key}, + ) + + # no_origin / no_destination + if json_response is None: + return False + + # Business logic only + for journey in json_response.get("journeys", []): + journey_day = int(journey["departure_date_time"][6:8]) + param_day = int(journey_params["datetime"][6:8]) + if journey_day == param_day: + return True + + return False + + +def get_navitia_isochrone(isochrone_params): + """ + Query the navitia Isochrones api, and returns the isochrone object + """ + lon = isochrone_params.get("from").split(";")[0] + lat = isochrone_params.get("from").split(";")[1] + source_coverage = get_coverage(lon, lat) + + # Retrieve navitia API Key from env var + api_key = os.getenv('NAVITIA_API_KEY') + if not api_key: + raise HTTPInternalServerError("Navitia API config is missing") + + if not source_coverage: + raise HTTPInternalServerError("Coverage not found for source") + + return navitia_get( + "https://api.navitia.io/v1/coverage/%s/isochrones" % source_coverage, + params=isochrone_params, + headers={"Authorization": api_key}, + ) + + +def is_wp_in_isochrone(waypoint, isochrone_geom): + """ + Returns true if waypoints is contained in isochrone_geom + """ + # get lon & lat + geom = shape(json.loads(waypoint.get("geometry").get("geom"))) + + src_epsg = 3857 + transformer = Transformer.from_crs( + "EPSG:%d" % src_epsg, "EPSG:4326", always_xy=True) + lon, lat = transformer.transform(geom.x, geom.y) + pt = Point(lon, lat) + + return isochrone_geom.contains(pt) + + +def extract_meta_params(request): + """ + Extract meta parameters such as offset, limit and lang + """ + v = request.validated + return { + 'offset': 0, + 'limit': 2000, + 'lang': v.get('lang'), + } + + +def extract_journey_params(request): + """ + Extract parameters for journey query + """ + return { + 'from': request.params.get('from'), + 'datetime': request.params.get('datetime'), + 'datetime_represents': request.params.get('datetime_represents'), + 'walking_speed': request.params.get('walking_speed'), + 'max_walking_duration_to_pt': request.params.get( + 'max_walking_duration_to_pt' + ), + 'to': '' + } + + +def extract_isochrone_params(request): + """ + Extract parameters for isochrone query + + NOTE : the boundary duration is bounded by constants + MAX_TRIP_DURATION and MIN_TRIP_DURATION + if the boundary duration goes beyond limits, + it is set to the limit it goes past. + """ + params = { + 'from': request.params.get('from'), + 'datetime': request.params.get('datetime'), + 'boundary_duration[]': request.params.get('boundary_duration'), + 'datetime_represents': request.params.get('datetime_represents') + } + # normalize boundary + bd = params['boundary_duration[]'] + if len(bd.split(",")) == 1: + duration = int(bd) + params['boundary_duration[]'] = max( + min( + duration, + MAX_TRIP_DURATION * 60 + ), + MIN_TRIP_DURATION * 60 + ) + return params + + +def collect_areas_from_results(results, area_index): + """ + Extract all area document_ids from results, load Area objects from DB, + and return {document_id: Area}. + """ + area_ids = set() + + for row in results: + areas = row[area_index] + + if not areas: + continue + + for area in areas: + doc_id = area.get("document_id") + if doc_id: + area_ids.add(doc_id) + + area_objects = DBSession.query(Area).filter( + Area.document_id.in_(area_ids) + ).all() + + return {a.document_id: a for a in area_objects} + + +def collect_waypoints_from_results(results): + """ + Extract all waypoint document_ids from results, + load Waypoint objects from DB, + and return {document_id: Waypoint}. + """ + wp_ids = set() + + for route, areas, waypoints in results: + if not waypoints: + continue + + for wp in waypoints: + doc_id = wp.get("document_id") + if doc_id: + wp_ids.add(doc_id) + + wp_objects = DBSession.query(Waypoint).filter( + Waypoint.document_id.in_(wp_ids) + ).all() + + return {wp for wp in wp_objects} + + +def handle_navitia_response(response): + """ + Handles Navitia HTTP errors and returns parsed JSON on success. + Raises HTTP* exceptions otherwise. + """ + + if response.status_code == 401: + raise HTTPInternalServerError("Authentication error with Navitia API") + + elif response.status_code == 400: + raise HTTPBadRequest("Invalid parameters for Navitia API") + + elif response.status_code == 404: + error_id = response.json().get("error", {}).get("id") + + # Allowed "no journey found" cases + if error_id in ("no_destination", "no_origin"): + return None + + # Raise with the payload as the detail + raise HTTPInternalServerError( + response.json().get("error", {})) + + elif not response.ok: + raise HTTPInternalServerError("Navitia API error: %d ", + response.status_code) + + return response.json() + + +def navitia_get(url, *, params=None, headers=None, timeout=30): + try: + response = requests.get( + url, + params=params, + headers=headers, + timeout=timeout, + ) + return handle_navitia_response(response) + + except requests.exceptions.Timeout: + raise HTTPInternalServerError("Timeout when calling the Navitia API") + + except requests.exceptions.RequestException as e: + raise HTTPInternalServerError(str(e)) + + except Exception as e: + raise HTTPInternalServerError(str(e)) + + +def redis_client(): + """ fast way to get redis client """ + return redis.Redis(host=REDIS_HOST, port=REDIS_PORT, db=REDIS_DB) + + +def start_job_background(target, request): + """ start a job in the background, + target is the query function to execute in bg + request is the request to pass to the function""" + job_id = str(uuid.uuid4()) + r = redis_client() + r.set("job:%s:progress" % job_id, 0) + r.set("job:%s:status" % job_id, "running") + threading.Thread(target=target, args=( + job_id, request), daemon=True).start() + return {"job_id": job_id} + + +def get_job_status(r, job_id): + """ returns the ongoing job status """ + status = r.get("job:%s:status" % job_id) + if status is None: + return None, {"error": "unknown_job_id"} + status = status.decode() + return status, status + + +def read_result_from_redis(r, job_id): + """ returns the result from redis """ + status = r.get("job:%s:status" % job_id) + if status is None: + return {"error": "unknown_job_id"} + status = status.decode() + if status == "running": + return {"status": "running"} + if status == "error": + error_msg = r.get("job:%s:error" % job_id) + return { + "status": "error", + "message": error_msg.decode() if error_msg else "unknown error" + } + if status == "done": + data = r.get("job:%s:result" % job_id) + if not data: + return {"status": "error", "message": "missing_result"} + return {"status": "done", "result": json.loads(data)} + return {"error": "unknown_status", "status": status} + + +def progress_stream(r, job_id, poll_interval=0.5): + """ yield the job progress """ + while True: + raw_progress = r.get("job:%s:progress" % job_id) + raw_found = r.get("job:%s:found" % job_id) + raw_not_found = r.get("job:%s:not_found" % job_id) + raw_total = r.get("job:%s:total" % job_id) + + progress = int(raw_progress) if raw_progress is not None else 0 + found = int(raw_found) if raw_found is not None else 0 + not_found = int(raw_not_found) if raw_not_found is not None else 0 + total = int(raw_total) if raw_total is not None else 0 + + payload = {"progress": progress, "total": total, + "found": found, "not_found": not_found} + yield ("data: %s\n\n" % json.dumps(payload)).encode("utf-8") + + status = r.get("job:%s:status" % job_id) + if status and status.decode() == "done": + yield b"event: done\ndata: done\n\n" + break + elif status and status.decode() == "error": + payload = r.get("job:%s:error" % job_id) + json_payload = json.dumps(ast.literal_eval(payload.decode())) + yield ("event: error\ndata: %s\n\n" % json_payload).encode("utf-8") + break + + time.sleep(poll_interval) + + +def _store_job_progress(r, job_id, count, found, not_found): + """ + store job progress which is : + progress : the number of queries done + found : the number of successful queries + not_found: the number of unsuccessful queries + """ + r.set("job:%s:progress" % job_id, count) + r.set("job:%s:found" % job_id, found) + r.set("job:%s:not_found" % job_id, not_found) + r.publish("job:%s:events" % job_id, "progress:%d" % count) + r.publish("job:%s:events" % job_id, "found:%d" % found) + r.publish("job:%s:events" % job_id, "not_found:%d" % not_found) diff --git a/c2corg_api/views/route.py b/c2corg_api/views/route.py index 2ab7f9cb3..2ceeaf94f 100644 --- a/c2corg_api/views/route.py +++ b/c2corg_api/views/route.py @@ -1,6 +1,7 @@ from itertools import combinations import functools +from sqlalchemy import case from c2corg_api.models import DBSession from c2corg_api.models.association import Association @@ -15,6 +16,7 @@ route_schema_adaptor, outing_documents_config from c2corg_api.views.document_version import DocumentVersionRest from c2corg_api.models.utils import get_mid_point +from c2corg_api.search.advanced_search import get_all_filtered_docs from cornice.resource import resource, view from cornice.validators import colander_body_validator @@ -34,6 +36,16 @@ from sqlalchemy.orm import load_only from sqlalchemy.sql.expression import text, or_, column, union +from operator import and_ +from c2corg_api.models.area import Area +from c2corg_api.models.area_association import AreaAssociation +from c2corg_api.views import to_json_dict +from c2corg_api.views.document import (LIMIT_DEFAULT) +from c2corg_api.models.waypoint_stoparea import (WaypointStoparea) +from c2corg_api.models.area import schema_listing_area +from sqlalchemy import func, literal_column + + validate_route_create = make_validator_create( fields_route, 'activities', activities) validate_route_update = make_validator_update( @@ -202,6 +214,77 @@ def get(self): update_all_pt_rating(waypoint_extrapolation) +@resource(path='/reachableroutes', cors_policy=cors_policy) +class ReachableRouteRest(DocumentRest): + + def __init__(self, request, context=None): + self.request = request + + @view(validators=[validate_pagination, validate_preferred_lang_param]) + def get(self): + """Returns a list of object {documents: Route[], total: Integer} -> + documents: routes reachable within offset and limit + total: number of documents returned by query, + without offset and limit""" + + validated = self.request.validated + + meta_params = { + 'offset': validated.get('offset', 0), + 'limit': validated.get('limit', LIMIT_DEFAULT), + 'lang': validated.get('lang') + } + + query, count = build_reachable_route_query( + self.request.GET, + meta_params + ) + + if query is None: + results = [] + else: + # route, areas in results + results = ( + query + .limit(meta_params['limit']) + .offset(meta_params['offset']) + .all() + ) + + areas_id = set() + for route, areas in results: + if areas is None: + continue + for area in areas: + area_id = area.get("document_id") + if area_id is not None: + areas_id.add(area_id) + + areas_objects = DBSession.query(Area).filter( + Area.document_id.in_(areas_id)).all() + + areas_map = {area.document_id: area for area in areas_objects} + + routes = [] + for route, areas in results: + json_areas = [] + if areas is None: + areas = [] + + for area in areas: + area_obj = areas_map.get(area.get("document_id")) + if area_obj: + json_areas.append(to_json_dict( + area_obj, schema_listing_area)) + + # assign JSON areas to the route + route.areas = json_areas + wp_dict = to_json_dict(route, schema_route, True) + routes.append(wp_dict) + + return {'documents': routes, 'total': count} + + def set_default_geometry(linked_waypoints, route, user_id): """When creating a new route, set the default geometry to the middle point of a given track, if not to the geometry of the associated main waypoint @@ -499,7 +582,7 @@ def worst_rating(rating1, rating2): # Return the best starting rating if it's not a crossing if not (route_types and bool( set(["traverse", "raid", "expedition"]) & set(route_types) - )): + )): return best_starting_rating # If no ending point is provided @@ -513,3 +596,183 @@ def worst_rating(rating1, rating2): # Return the worst of the two ratings return worst_rating(best_starting_rating, best_ending_rating) + + +def build_reachable_route_query(params, meta_params): + """build the query based on params and meta params. + this includes every filters on route, + as well as offset + limit, sort, bbox... + returns a list of routes reachable + (can be accessible by public transports), filtered with params + """ + all_filtered_routes_reachable_ids, \ + total_hits = get_all_filtered_docs( + params, + meta_params, + get_routes_reachable_ids(), + True, + ROUTE_TYPE + ) + + if total_hits == 0: + return None, 0 + + # The order of ids within all_filtered_routes_reachable_ids order matters + # since a sort may have been applied by ES + + ordering_case = case( + { + doc_id: idx for idx, + doc_id in enumerate(all_filtered_routes_reachable_ids) + }, + value=Route.document_id + ) + + # Querying database with the ids from ES, mainting the order with the case + query = ( + DBSession. + query(Route, + func.jsonb_agg(func.distinct( + func.jsonb_build_object( + literal_column( + "'document_id'"), Area.document_id + ))).label("areas")). + filter(Route.document_id.in_( + all_filtered_routes_reachable_ids)). + join(Association, or_( + Route.document_id == Association.child_document_id, + Route.document_id == Association.parent_document_id + )). + join( + AreaAssociation, + AreaAssociation.document_id == Route.document_id + ). + join( + Area, + Area.document_id == AreaAssociation.area_id + ). + join(Waypoint, and_( + or_( + Waypoint.document_id == Association.child_document_id, + Waypoint.document_id == Association.parent_document_id + ), + Waypoint.waypoint_type == 'access' + )). + join( + WaypointStoparea, + WaypointStoparea.waypoint_id == Waypoint.document_id + ). + group_by(Route). + order_by(ordering_case) + ) + + return query, total_hits + + +def build_reachable_route_query_with_waypoints(params, meta_params): + """build the query based on params and meta params. + this includes every filters on route, + as well as offset + limit, sort, bbox... + returns a list of routes reachable + (accessible by common transports), filtered with params + """ + + all_filtered_routes_reachable_ids, \ + total_hits = get_all_filtered_docs( + params, + meta_params, + get_routes_reachable_ids(), + True, + ROUTE_TYPE + ) + + if total_hits == 0: + return None, 0 + + # The order of ids within all_filtered_routes_reachable_ids order matters + # since a sort may have been applied by ES + + ordering_case = case( + { + doc_id: idx for idx, + doc_id in enumerate(all_filtered_routes_reachable_ids) + }, + value=Route.document_id + ) + + # then query database with the ids from ES + query = ( + DBSession. + query(Route, + func.jsonb_agg(func.distinct( + func.jsonb_build_object( + literal_column( + "'document_id'"), Area.document_id + ))).label("areas"), + func.jsonb_agg(func.distinct( + func.jsonb_build_object( + literal_column( + "'document_id'"), Waypoint.document_id + ))).label("waypoints")). + select_from(Association). + join(Route, and_( + or_( + Route.document_id == Association.child_document_id, + Route.document_id == Association.parent_document_id + ), Route.document_id.in_(all_filtered_routes_reachable_ids) + )). + join( + AreaAssociation, + AreaAssociation.document_id == Route.document_id + ). + join( + Area, + Area.document_id == AreaAssociation.area_id + ). + join(Waypoint, and_( + or_( + Waypoint.document_id == Association.child_document_id, + Waypoint.document_id == Association.parent_document_id + ), + Waypoint.waypoint_type == 'access' + )). + join( + WaypointStoparea, + WaypointStoparea.waypoint_id == Waypoint.document_id + ). + group_by(Route). + order_by(ordering_case) + ) + + return query, total_hits + + +def get_routes_reachable_ids(): + """get all routes reachable ids""" + # get all routes reachable (join with waypoint stop area) + all_routes_reachable = ( + DBSession.query(Route). + join(Association, or_( + Association.child_document_id == Route.document_id, + Association.parent_document_id == Route.document_id + )). + join(Waypoint, and_( + or_( + Waypoint.document_id == Association.child_document_id, + Waypoint.document_id == Association.parent_document_id + ), + Waypoint.waypoint_type == 'access' + )). + join( + WaypointStoparea, + WaypointStoparea.waypoint_id == Waypoint.document_id + ) + .distinct() + .all() + ) + + # extract their ids + all_routes_reachable_ids = set( + [r.document_id for r in all_routes_reachable]) + + return all_routes_reachable_ids diff --git a/c2corg_api/views/waypoint.py b/c2corg_api/views/waypoint.py index a1d4ce97a..8572626d4 100644 --- a/c2corg_api/views/waypoint.py +++ b/c2corg_api/views/waypoint.py @@ -1,4 +1,6 @@ import functools +import logging +from sqlalchemy import case from c2corg_api.models import DBSession from c2corg_api.models.association import Association @@ -16,6 +18,8 @@ from cornice.resource import resource, view from cornice.validators import colander_body_validator +from c2corg_api.search.advanced_search import get_all_filtered_docs + from c2corg_api.models.waypoint import ( Waypoint, schema_waypoint, schema_update_waypoint, ArchiveWaypoint, ArchiveWaypointLocale, WAYPOINT_TYPE, @@ -34,6 +38,17 @@ from sqlalchemy.orm.util import aliased from sqlalchemy.sql.elements import literal_column from sqlalchemy.sql.expression import and_, text, union, column +from c2corg_api.models.area import Area +from c2corg_api.models.area_association import AreaAssociation +from c2corg_api.views import to_json_dict +from c2corg_api.views.document import ( + LIMIT_DEFAULT) +from c2corg_api.models.waypoint_stoparea import ( + WaypointStoparea) +from c2corg_api.models.area import schema_listing_area +from sqlalchemy import func + +log = logging.getLogger(__name__) # the number of routes that are included for waypoints NUM_ROUTES = 400 @@ -287,7 +302,7 @@ def set_recent_outings(waypoint, lang): join( with_query_waypoints, with_query_waypoints.c.document_id == t_route_wp.parent_document_id - ). \ + ). \ distinct(). \ count() @@ -369,9 +384,9 @@ def _get_select_children(waypoint): cte('waypoint_grandchildren') return union( - select_waypoint.select(), - select_waypoint_children.select(), - select_waypoint_grandchildren.select()). \ + select_waypoint.select(), + select_waypoint_children.select(), + select_waypoint_grandchildren.select()). \ cte('select_all_waypoints') @@ -393,6 +408,75 @@ def get(self): return self._get_document_info(waypoint_documents_config) +@resource(path='/reachablewaypoints', cors_policy=cors_policy) +class ReachableWaypointRest(DocumentRest): + + def __init__(self, request, context=None): + self.request = request + + @view(validators=[validate_pagination, validate_preferred_lang_param]) + def get(self): + """Returns a list of object {documents: Waypoint[], total: Integer} -> + documents: waypoints reachable within offset and limit + total: number of documents returned by query, + without offset and limit""" + validated = self.request.validated + + meta_params = { + 'offset': validated.get('offset', 0), + 'limit': validated.get('limit', LIMIT_DEFAULT), + 'lang': validated.get('lang') + } + + query, count = build_reachable_waypoints_query( + self.request.GET, + meta_params + ) + + if query is None: + results = [] + else: + results = ( + query + .limit(meta_params['limit']) + .offset(meta_params['offset']) + .all() + ) + + areas_id = set() + for wp, areas in results: + if areas is None: + continue + for area in areas: + area_id = area.get("document_id") + if area_id is not None: + areas_id.add(area_id) + + areas_objects = DBSession.query(Area).filter( + Area.document_id.in_(areas_id)).all() + + areas_map = {area.document_id: area for area in areas_objects} + + waypoints = [] + for wp, areas in results: + json_areas = [] + if areas is None: + areas = [] + + for area in areas: + area_obj = areas_map.get(area.get("document_id")) + if area_obj: + json_areas.append(to_json_dict( + area_obj, schema_listing_area)) + + # assign JSON areas to the waypoint + wp.areas = json_areas + wp_dict = to_json_dict(wp, schema_waypoint, True) + waypoints.append(wp_dict) + + return {'documents': waypoints, 'total': count} + + def update_linked_routes(waypoint, update_types, user_id): update_linked_route_titles(waypoint, update_types, user_id) update_linked_routes_public_transportation_rating(waypoint, update_types) @@ -459,3 +543,79 @@ def update_linked_routes_public_transportation_rating(waypoint, update_types): for route in routes: update_pt_rating(route) + + +def build_reachable_waypoints_query(params, meta_params): + """build the query based on params and meta params. + this includes every filters on waypoints, + as well as offset + limit, sort, bbox... + returns a list of waypoints reachable + (can be accessible by public transports), filtered with params + """ + all_filtered_waypoints_reachable_ids, \ + total_hits = get_all_filtered_docs( + params, + meta_params, + get_waypoints_reachable_ids(), + True, + WAYPOINT_TYPE + ) + + if total_hits == 0: + return None, 0 + + # The order of ids within all_filtered_waypoints_reachable_ids + # order matters since a sort may have been applied by ES + + ordering_case = case( + { + doc_id: idx for idx, + doc_id in enumerate(all_filtered_waypoints_reachable_ids) + }, + value=Waypoint.document_id + ) + + # then query database with the ids from ES, maintaining order with the case + query = ( + DBSession. + query(Waypoint, + func.jsonb_agg(func.distinct( + func.jsonb_build_object( + literal_column( + "'document_id'"), Area.document_id + ))).label("areas")). + filter(Waypoint.document_id.in_( + all_filtered_waypoints_reachable_ids)). + join( + AreaAssociation, + AreaAssociation.document_id == Waypoint.document_id + ). + join( + Area, + Area.document_id == AreaAssociation.area_id + ). + group_by(Waypoint). + order_by(ordering_case) + ) + + return query, total_hits + + +def get_waypoints_reachable_ids(): + """get all waypoints reachable ids""" + # get all waypoints reachable (join with waypoint stop area) + all_routes_reachable = ( + DBSession.query(Waypoint). + join( + WaypointStoparea, + WaypointStoparea.waypoint_id == Waypoint.document_id + ) + .distinct() + .all() + ) + + # extract their ids + all_waypoints_reachable_ids = set( + [r.document_id for r in all_routes_reachable]) + + return all_waypoints_reachable_ids diff --git a/c2corg_api/views/waypoint_stoparea.py b/c2corg_api/views/waypoint_stoparea.py index 83a2980e9..e446d451a 100644 --- a/c2corg_api/views/waypoint_stoparea.py +++ b/c2corg_api/views/waypoint_stoparea.py @@ -48,8 +48,9 @@ def __init__(self, request, context=None): @view(validators=[validate_waypoint_id]) def get(self): - """Returns all stopareas associated with a waypoint, with their full attributes and distance.""" # noqa: E501 - waypoint_id = self.request.matchdict['waypoint_id'] # noqa: E501 + """Returns all stopareas associated with a waypoint, + with their full attributes and distance.""" + waypoint_id = self.request.matchdict['waypoint_id'] query = ( DBSession.query( @@ -58,7 +59,8 @@ def get(self): func.ST_X(Stoparea.geom).label('x'), func.ST_Y(Stoparea.geom).label('y') ) - .join(WaypointStoparea, Stoparea.stoparea_id == WaypointStoparea.stoparea_id) # noqa: E501 + .join(WaypointStoparea, + Stoparea.stoparea_id == WaypointStoparea.stoparea_id) .filter(WaypointStoparea.waypoint_id == waypoint_id) .all() ) @@ -82,7 +84,8 @@ def __init__(self, request, context=None): @view(validators=[validate_waypoint_id]) def get(self): - """Returns true if the waypoint has at least one stoparea associated with it, false otherwise.""" # noqa: E501 + """Returns true if the waypoint has at least one stoparea + associated with it, false otherwise.""" waypoint_id = self.request.matchdict['waypoint_id'] has_stopareas = DBSession.query( diff --git a/update_navitia_coverage.sh b/update_navitia_coverage.sh new file mode 100755 index 000000000..e1a88327b --- /dev/null +++ b/update_navitia_coverage.sh @@ -0,0 +1,186 @@ +#!/bin/bash +# shellcheck disable=SC2001 + +# this scripts is meant to be executed whenever navitia france coverages are updated. +# the goal of this script is to save in DB france coverages, with their id and geometry. +# there are as many Navitia request as there are coverages in France. +# +# The script takes in parameter +# Username 'user123' +# Password 'password123' (make sure to escape special characters) +# Base API URL 'http://localhost' +# API Port '6543' +# And is meant to be used by moderators, as regular users can't delete documents. + +# First, a token is retrieved using username and password in parameter. +# Then, existing coverages are deleted +# Navitia request are made towards /coverages/{region_id} route to get all coverages. +# For each coverage found, a POST on Base_API_URL:API_PORT/coverages is made to insert in database. + +# NOTE: the geometry returned by Navitia for the coverages are in WGS384. + +if [ -f ./.env ]; then + # Load .env data + export $(grep -v '^#' ./.env | xargs) +else + echo ".env file not found!" + exit 1 +fi + +# Function to display usage +usage() { + echo "Usage: $0 " + exit 1 +} + +# Check if exactly 4 arguments are provided +if [ "$#" -ne 4 ]; then + usage +fi + +# Check if jq is installed +if ! command -v jq &> /dev/null; then + echo "jq could not be found. Please install it." + exit 1 +fi + +# Assign arguments to variables +username="$1" +password="$2" +base_api_url="$3" +api_port="$4" + +LOG_FILE="log-navitia-coverage.txt" +NAVITIA_REQUEST_COUNT=0 + +COVERAGE_API_URL="$base_api_url:$api_port/coverages" + +echo "Start time :" > "$LOG_FILE" +echo $(date +"%Y-%m-%d-%H-%M-%S") >> "$LOG_FILE" + +login_body=$(jq -n \ + --arg username "$username" \ + --arg password "$password" \ + '{ + username: $username, + password: $password, + discourse: true, + }') + +# log in to execute script +loginResponse=$(curl -s -X POST "$base_api_url:$api_port/users/login" \ + -H "Content-Type: application/json" \ + -d "$login_body") + +roles=$(echo "$loginResponse" | jq -r '.roles') +JWTToken=$(echo "$loginResponse" | jq -r '.token') + +coverages=$(curl -s -X GET "$COVERAGE_API_URL" \ + -H "Content-Type: application/json") + +numberOfCoverage=$(echo "$coverages" | jq -r '.total') + +if [ "$numberOfCoverage" != "0" ]; then + # check if logged user is a moderator + found=false + for role in "${roles[@]}"; do + if [[ "$role" == "moderator" ]]; then + found=true + break + fi + done + if ! $found; then + echo "Error : User should be a moderator to delete existing coverages" + exit 1 + fi + + # remove old coverages + echo "$coverages" | jq -c '.documents[]' | while IFS= read -r coverage; do + coverage_doc_id=$(echo "$coverage" | jq -r '.document_id') + + deleteResponse=$(curl -X POST -v -H "Content-Type: application/json" -H "Authorization: JWT token=\"$JWTToken\"" "$base_api_url:$api_port/documents/delete/${coverage_doc_id}") + + status=$(echo "$deleteResponse" | jq -r '.status') + + # if we can't delete coverage, then we stop the script + if [ $status = "error" ]; then + exit 1 + fi + done +fi + +# This define how much navitia request will be made executing this script +regions=('fr-idf' 'fr-ne' 'fr-nw' 'fr-se' 'fr-sw') + +responses=() + +# Loop over Navitia regions in France +for region_id in "${regions[@]}"; do + # Fetch the response from the Navitia API + response=$(curl -s -H "Authorization: $NAVITIA_API_KEY" \ + "https://api.navitia.io/v1/coverage/${region_id}") + ((NAVITIA_REQUEST_COUNT++)) + + # Extract the coverage type + coverage_type=$(echo "$response" | jq -r '.regions[0].id') + + # Extract the shape (WKT string) + shape=$(echo "$response" | jq -r '.regions[0].shape') + + # remove 'MULTIPOLGYON' from shape + coordinate_list=${shape//"MULTIPOLYGON"/} + + # remove ( + coordinate_list=${coordinate_list//"("/} + + # remove ) + coordinate_list=${coordinate_list//")"/} + + coordinates=() + coordinates+="[[" + + # get a list of all coordinates (separated by comma) + while IFS=',' read -ra coo; do + for i in "${coo[@]}"; do + # get lon & lat + lon_lat=($i) + # fix subcoordinates + operator concatenate not exist + subcoordinates="[${lon_lat[0]},${lon_lat[1]}]" + coordinates+="${subcoordinates}," + done + done <<< "$coordinate_list" + + # remove last comma + coordinates=${coordinates%?} + + coordinates+="]]" + + type="Polygon" + + geom_detail="{\"type\": \"Polygon\", \"coordinates\": $coordinates}" + + # no coverages yet, so we insert + if [ "$numberOfCoverage" = "0" ]; then + echo "inserting coverages" + # Build JSON payload + payload=$(jq -n \ + --arg coverage_type "$coverage_type" \ + --arg geom_detail "$geom_detail" \ + '{ + coverage_type: $coverage_type, + geometry: { + geom: null, + geom_detail: $geom_detail + } + }') + + # Send the POST request to create a coverage in DB + responses+=$(curl -X POST -v -H "Content-Type: application/json" -H "Authorization: JWT token=\"$JWTToken\"" -d "$payload" "$COVERAGE_API_URL") + fi +done + +# Log final progress +echo "Completed. Total Navitia API requests: $NAVITIA_REQUEST_COUNT" >> $LOG_FILE + +echo "Stop time :" >> $LOG_FILE +echo $(date +"%Y-%m-%d-%H-%M-%S") >> $LOG_FILE \ No newline at end of file