diff --git a/CHANGELOG.md b/CHANGELOG.md index a4960ebe..4185d8c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] +- Support HCAT mapping CSV files without crop_code +- Split Germany BB and NDS in block dataset and crop fields +- Add HCAT to datasets where possible +- Updated years & variants for at_crop, be_vlg, es_an, es_cl, es_pv, ie, pt, se - Extend create_stac, include include fiboa data - Publish command; skip hidden files, generate better texts - Fix to vecorel: converter.license and provider should be string diff --git a/fiboa_cli/datasets/ai4sf.py b/fiboa_cli/datasets/ai4sf.py index 56d6e949..f7569cec 100644 --- a/fiboa_cli/datasets/ai4sf.py +++ b/fiboa_cli/datasets/ai4sf.py @@ -111,7 +111,7 @@ def migrate(self, gdf): gdf["fiboa_id"] = ( gdf["id"].astype(str).str.zfill(2) + "_" + gdf.index.astype(str).str.zfill(5) ) - return gdf + return super().migrate(gdf) missing_schemas = { "properties": { diff --git a/fiboa_cli/datasets/at.py b/fiboa_cli/datasets/at.py index 06d73e0e..4ebbc3e2 100644 --- a/fiboa_cli/datasets/at.py +++ b/fiboa_cli/datasets/at.py @@ -1,44 +1,43 @@ from vecorel_cli.conversion.admin import AdminConverterMixin -from fiboa_cli.conversion.fiboa_converter import FiboaBaseConverter +from ..conversion.fiboa_converter import FiboaBaseConverter +from .commons.ec import AddHCATMixin -class Converter(AdminConverterMixin, FiboaBaseConverter): - sources = { - "https://inspire.lfrz.gv.at/009501/ds/inspire_referenzen_2021_polygon.gpkg.zip": [ - "INSPIRE_REFERENZEN_2021_POLYGON.gpkg" - ] +class Converter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter): + variants = { + "2025": "https://inspire.lfrz.gv.at/009501/ds/inspire_schlaege_2025-1_polygon.gpkg.zip", + "2024": "https://inspire.lfrz.gv.at/009501/ds/inspire_schlaege_2024-2_polygon.gpkg.zip", + "2023": "https://inspire.lfrz.gv.at/009501/ds/inspire_schlaege_2023-2_polygon.gpkg.zip", + "2022": "https://inspire.lfrz.gv.at/009501/ds/inspire_schlaege_2022_polygon.gpkg.zip", + "2021": "https://inspire.lfrz.gv.at/009501/ds/inspire_schlaege_2021_polygon.gpkg.zip", + "2020": "https://inspire.lfrz.gv.at/009501/ds/inspire_schlaege_2020_polygon.gpkg.zip", + "2019": "https://inspire.lfrz.gv.at/009501/ds/inspire_schlaege_2019_polygon.gpkg.zip", + "2018": "https://inspire.lfrz.gv.at/009501/ds/inspire_schlaege_2018_polygon.gpkg.zip", } + id = "at" country = "AT" short_name = "Austria" title = "Field boundaries for Austria" description = """ -**Field boundaries for Austria - INVEKOS Referenzen Österreich 2021.** - -The layer includes all reference parcels ("Referenzparzellen") defined by the paying agency Agrarmarkt Austria and recorded landscape elements (landscape element layers) within the meaning of Art. 5 of Regulation (EU) No. 640/2014 and Regulation of the competent federal ministry with horizontal rules for the area of the Common Agricultural Policy (Horizontal CAP Regulation) StF: Federal Law Gazette II No. 100/2015. +**Crop Field boundaries for Austria - INVEKOS Schläge Österreich 2025.** -Reference parcel: is the physical block that can be clearly delimited from the outside (e.g. forest, roads, water bodies) and is formed by contiguous agricultural areas that are recognizable in nature. +This layer includes all field uses recorded by the applicants, which serve as the basis for the funding process. A field +is a contiguous area of a piece of land that is cultivated for a growing season with only one crop (field use type) and +uniform management requirements or as a landscape element type in accordance with Annex 1 of the regulation of the responsible +Federal Ministry with horizontal rules for the area of the Common Agricultural Policy (Horizontal CAP Regulation) +StF: BGBl. II No. 100/2015 or is simply maintained in good agricultural and ecological condition in accordance with +Art. 94 of Regulation (EU) No. 1306/2013 and is digitized in the GIS as a polygon or as a point. """ provider = "Agrarmarkt Austria " license = "CC-BY-4.0" columns = { + "GEO_ID": "id", "geometry": "geometry", - "RFL_ID": "id", - "REF_ART": "ref_art", - "BRUTTOFLAECHE_HA": "metrics:area", - "INSPIRE_ID": "inspire:id", - "REF_ART_BEZEICHNUNG": "ref_art_bezeichnung", - "REFERENZ_KENNUNG": "referenz_kennung", - "FART_ID": "fart_id", - "GEO_DATERF": "determination:datetime", - } - extensions = {"https://fiboa.org/inspire-extension/v0.3.0/schema.yaml"} - missing_schemas = { - "properties": { - "ref_art": {"type": "string"}, - "ref_art_bezeichnung": {"type": "string"}, - "referenz_kennung": {"type": "uint64"}, - "fart_id": {"type": "uint32"}, - } + "SNAR_CODE": "crop:code", + "SNAR_BEZEICHNUNG": "crop:name", + "SL_FLAECHE_BRUTTO_HA": "metrics:area", + "GEOM_DATE_CREATED": "determination:datetime", } + ec_mapping_csv = "https://fiboa.org/code/at/at.csv" diff --git a/fiboa_cli/datasets/at_block.py b/fiboa_cli/datasets/at_block.py new file mode 100644 index 00000000..9620e909 --- /dev/null +++ b/fiboa_cli/datasets/at_block.py @@ -0,0 +1,44 @@ +from vecorel_cli.conversion.admin import AdminConverterMixin + +from fiboa_cli.conversion.fiboa_converter import FiboaBaseConverter + + +class Converter(AdminConverterMixin, FiboaBaseConverter): + sources = { + "https://inspire.lfrz.gv.at/009501/ds/inspire_referenzen_2021_polygon.gpkg.zip": [ + "INSPIRE_REFERENZEN_2021_POLYGON.gpkg" + ] + } + id = "at_block" + country = "AT" + short_name = "Austria" + title = "Field boundaries for Austria" + description = """ +**Field boundaries for Austria - INVEKOS Referenzen Österreich 2021.** + +The layer includes all reference parcels ("Referenzparzellen") defined by the paying agency Agrarmarkt Austria and recorded landscape elements (landscape element layers) within the meaning of Art. 5 of Regulation (EU) No. 640/2014 and Regulation of the competent federal ministry with horizontal rules for the area of the Common Agricultural Policy (Horizontal CAP Regulation) StF: Federal Law Gazette II No. 100/2015. + +Reference parcel: is the physical block that can be clearly delimited from the outside (e.g. forest, roads, water bodies) and is formed by contiguous agricultural areas that are recognizable in nature. + """ + provider = "Agrarmarkt Austria " + license = "CC-BY-4.0" + columns = { + "geometry": "geometry", + "RFL_ID": "id", + "REF_ART": "ref_art", + "BRUTTOFLAECHE_HA": "metrics:area", + "INSPIRE_ID": "inspire:id", + "REF_ART_BEZEICHNUNG": "ref_art_bezeichnung", + "REFERENZ_KENNUNG": "referenz_kennung", + "FART_ID": "fart_id", + "GEO_DATERF": "determination:datetime", + } + extensions = {"https://fiboa.org/inspire-extension/v0.3.0/schema.yaml"} + missing_schemas = { + "properties": { + "ref_art": {"type": "string"}, + "ref_art_bezeichnung": {"type": "string"}, + "referenz_kennung": {"type": "uint64"}, + "fart_id": {"type": "uint32"}, + } + } diff --git a/fiboa_cli/datasets/at_crop.py b/fiboa_cli/datasets/at_crop.py deleted file mode 100644 index 64ff6a89..00000000 --- a/fiboa_cli/datasets/at_crop.py +++ /dev/null @@ -1,46 +0,0 @@ -from vecorel_cli.conversion.admin import AdminConverterMixin - -from ..conversion.fiboa_converter import FiboaBaseConverter -from .commons.ec import AddHCATMixin - - -class Converter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter): - variants = { - "2024": "https://inspire.lfrz.gv.at/009501/ds/inspire_schlaege_2024-2_polygon.gpkg.zip", - "2023": "https://inspire.lfrz.gv.at/009501/ds/inspire_schlaege_2023-2_polygon.gpkg.zip", - "2022": "https://inspire.lfrz.gv.at/009501/ds/inspire_schlaege_2022_polygon.gpkg.zip", - "2021": "https://inspire.lfrz.gv.at/009501/ds/inspire_schlaege_2021_polygon.gpkg.zip", - "2020": "https://inspire.lfrz.gv.at/009501/ds/inspire_schlaege_2020_polygon.gpkg.zip", - "2019": "https://inspire.lfrz.gv.at/009501/ds/inspire_schlaege_2019_polygon.gpkg.zip", - "2018": "https://inspire.lfrz.gv.at/009501/ds/inspire_schlaege_2018_polygon.gpkg.zip", - } - - id = "at_crop" - country = "AT" - short_name = "Austria" - title = "Field boundaries for Austria" - description = """ -**Crop Field boundaries for Austria - INVEKOS Schläge Österreich 2021.** - -This layer includes all field uses recorded by the applicants, which serve as the basis for the funding process. A field -is a contiguous area of a piece of land that is cultivated for a growing season with only one crop (field use type) and -uniform management requirements or as a landscape element type in accordance with Annex 1 of the regulation of the responsible -Federal Ministry with horizontal rules for the area of the Common Agricultural Policy (Horizontal CAP Regulation) -StF: BGBl. II No. 100/2015 or is simply maintained in good agricultural and ecological condition in accordance with -Art. 94 of Regulation (EU) No. 1306/2013 and is digitized in the GIS as a polygon or as a point. - """ - provider = "Agrarmarkt Austria " - license = "CC-BY-4.0" - columns = { - "GEO_ID": "id", - "INSPIRE_ID": "inspire:id", - "geometry": "geometry", - "SNAR_CODE": "crop:code", - "SNAR_BEZEICHNUNG": "crop:name", - "SL_FLAECHE_BRUTTO_HA": "metrics:area", - "GEOM_DATE_CREATED": "determination:datetime", - } - ec_mapping_csv = "at_2021.csv" - extensions = { - "https://fiboa.org/inspire-extension/v0.3.0/schema.yaml", - } diff --git a/fiboa_cli/datasets/be_vlg.py b/fiboa_cli/datasets/be_vlg.py index 22f9bba9..4d4c253a 100644 --- a/fiboa_cli/datasets/be_vlg.py +++ b/fiboa_cli/datasets/be_vlg.py @@ -10,7 +10,8 @@ class Converter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter): variants = { str(k): {PREFIX + v: [v.replace("_GPKG.zip", ".gpkg")]} for k, v in ( - (2024, "Landbouwgebruikspercelen_2024_-_Voorlopig_(extractie_15-10-2024)_GPKG.zip"), + (2025, "Landbouwgebruikspercelen_2025_-_Voorlopig_(extractie_02-06-2025)_GPKG.zip"), + (2024, "Landbouwgebruikspercelen_2024_-_Definitief_(extractie_27-03-2025)_GPKG.zip"), (2023, "Landbouwgebruikspercelen_2023_-_Definitief_(extractie_28-03-2024)_GPKG.zip"), (2022, "Landbouwgebruikspercelen_2022_-_Definitief_(extractie_26-06-2023)_GPKG.zip"), (2021, "Landbouwgebruikspercelen_2021_-_Definitief_(extractie_15-03-2022)_GPKG.zip"), @@ -35,7 +36,6 @@ class Converter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter): columns = { "geometry": "geometry", - "BT_BRON": "source", "BT_OMSCH": "typology", "GRAF_OPP": "metrics:area", "REF_ID": "id", @@ -47,4 +47,4 @@ class Converter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter): } ec_mapping_csv = "be_vlg_2021.csv" - missing_schemas = {"properties": {"source": {"type": "string"}, "typology": {"type": "string"}}} + missing_schemas = {"properties": {"typology": {"type": "string"}}} diff --git a/fiboa_cli/datasets/be_wal.py b/fiboa_cli/datasets/be_wal.py index 3ce07c39..99eb957e 100644 --- a/fiboa_cli/datasets/be_wal.py +++ b/fiboa_cli/datasets/be_wal.py @@ -41,19 +41,20 @@ class Converter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter): column_additions = { "determination:datetime": "2022-01-01T00:00:00Z", } - column_migrations = { - "crop_code": lambda col: col.str.extract(r"\.(\d+)$", expand=False), - "crop_name": lambda col: col.str.strip(), - } index_as_id = True def layer_filter(self, layer: str, uri: str) -> bool: return layer == "ExistingLandUseObject" + column_migrations = { + "crop_code": lambda col: col.str.extract(r"\.(\d+)$", expand=False), + "crop_name": lambda col: col.str.strip(), + } + def file_migration( self, gdf: gpd.GeoDataFrame, path: str, uri: str, layer: str = None ) -> gpd.GeoDataFrame: - return gml_assure_columns( + gdf = gml_assure_columns( gdf, path, uri, @@ -61,3 +62,4 @@ def file_migration( crop_name={"ElementPath": "specificLandUse@title", "Type": "String", "Width": 255}, crop_code={"ElementPath": "specificLandUse@href", "Type": "String", "Width": 255}, ) + return gdf diff --git a/fiboa_cli/datasets/br_conab.py b/fiboa_cli/datasets/br_conab.py index 904d99b2..2298946a 100644 --- a/fiboa_cli/datasets/br_conab.py +++ b/fiboa_cli/datasets/br_conab.py @@ -93,7 +93,7 @@ def migrate(self, gdf): gdf.loc[gdf["area_ha"] == 0, "area_ha"] = None gdf["cd_mun"] = gdf["cd_mun"].combine_first(gdf["CD_MUN"]).apply(fformat) gdf["nm_mun"] = gdf["nm_mun"].combine_first(gdf["NM_MUN"]).combine_first(gdf["NM_MUNIC"]) - return gdf + return super().migrate(gdf) def get_data(self, paths, **kwargs): # Set invalid geometries to None in Cafe/MG/CAFE-MG_Safra_2017.zip diff --git a/fiboa_cli/datasets/ch.py b/fiboa_cli/datasets/ch.py index 34eea68e..65c0156b 100644 --- a/fiboa_cli/datasets/ch.py +++ b/fiboa_cli/datasets/ch.py @@ -2,9 +2,10 @@ from vecorel_cli.conversion.admin import AdminConverterMixin from ..conversion.fiboa_converter import FiboaBaseConverter +from .commons.hcat import AddHCATMixin -class Converter(AdminConverterMixin, FiboaBaseConverter): +class Converter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter): sources = None data_access = """ Data must be obtained from the Swiss open data portal at https://www.geodienste.ch/services/lwb_nutzungsflaechen . @@ -32,7 +33,7 @@ class Converter(AdminConverterMixin, FiboaBaseConverter): "id": "id", "flaeche_m2": "metrics:area", "kanton": "admin:subdivision_code", - "nutzung": "crop_name", + "nutzung": "crop:name", "bezugsjahr": "determination:datetime", } column_filters = { @@ -43,8 +44,4 @@ class Converter(AdminConverterMixin, FiboaBaseConverter): column_migrations = { "bezugsjahr": lambda col: pd.to_datetime(col, format="%Y"), } - missing_schemas = { - "properties": { - "crop_name": {"type": "string"}, - } - } + ec_mapping_csv = "https://fiboa.org/code/ch/ch.csv" diff --git a/fiboa_cli/datasets/commons/hcat.py b/fiboa_cli/datasets/commons/hcat.py index 0df5e929..5a94d219 100644 --- a/fiboa_cli/datasets/commons/hcat.py +++ b/fiboa_cli/datasets/commons/hcat.py @@ -3,6 +3,8 @@ from typing import Optional import geopandas as gpd +import numpy as np +import pandas as pd from vecorel_cli.vecorel.util import load_file HCAT_EXTENSION = "https://fiboa.org/hcat-extension/v0.3.0/schema.yaml" @@ -17,7 +19,7 @@ class AddHCATMixin: ec_mapping_csv: Optional[str] = None # TODO rename to hcat_mapping_csv mapping_file = None - ec_mapping: Optional[dict] = None # TODO rename to hcat_mapping + ec_mapping: Optional[list[dict]] = None # TODO rename to hcat_mapping hcat_columns = { "hcat:name_en": "hcat:name_en", @@ -38,11 +40,11 @@ def convert(self, *args, **kwargs): ) return super().convert(*args, **kwargs) - def get_code_column(self, gdf): + def get_code_column(self, gdf, code="crop:code"): try: - attribute = next(k for k, v in self.columns.items() if v == "crop:code") + attribute = next(k for k, v in self.columns.items() if v == code) except StopIteration: - raise Exception(f"Misssing crop:code column in converter {self.__class__.__name__}") + raise Exception(f"Misssing {code} column in converter {self.__class__.__name__}") col = gdf[attribute] # Should be corrected in original parser return col if col.dtype == "object" else col.astype(str) @@ -56,15 +58,42 @@ def add_hcat(self, gdf): if self.ec_mapping is None: self.ec_mapping = load_ec_mapping(self.ec_mapping_csv, url=self.mapping_file) - crop_code_col = self.get_code_column(gdf) + + from_code = "original_code" + if from_code not in self.ec_mapping[0]: + # Some code lists have no code, only a crop_name + from_code = "original_name" + crop_code_col = self.get_code_column(gdf, "crop:name") + else: + crop_code_col = self.get_code_column(gdf) def map_to(attribute): - return {e["original_code"]: e[attribute] for e in self.ec_mapping} + return {e[from_code]: e[attribute] or None for e in self.ec_mapping} + col = None for k, v in zip( self.hcat_columns.keys(), ("translated_name", "HCAT3_name", "HCAT3_code") ): - gdf[k] = crop_code_col.map(map_to(v)) + if v in self.ec_mapping[0]: + col = crop_code_col.map(map_to(v)) + gdf[k] = col + assert np.unique(col[~col.isna()]).size > 1, "No HCAT crops mapped" + + if col is not None and col.isna().any(): + index = [ + k for k, v in self.columns.items() if v.startswith("crop:") and k in gdf.columns + ] + missing = gdf[col.isna()][index].drop_duplicates() + missing.reset_index(drop=True, inplace=True) + with pd.option_context( + "display.max_colwidth", + None, + "display.max_columns", + None, + "display.max_rows", + None, + ): + self.info(f"Missing codes in HCAT mapping:\n{missing}") if "crop:code_list" not in gdf.columns: gdf["crop:code_list"] = ( @@ -72,12 +101,14 @@ def map_to(attribute): ) return gdf - def migrate(self, gdf) -> gpd.GeoDataFrame: - gdf = super().migrate(gdf) + def post_migrate(self, gdf) -> gpd.GeoDataFrame: + gdf = super().post_migrate(gdf) return self.add_hcat(gdf) def ec_url(csv_file): + if csv_file.startswith("https://"): + return csv_file return f"https://raw.githubusercontent.com/maja601/EuroCrops/refs/heads/main/csvs/country_mappings/{csv_file}" diff --git a/fiboa_cli/datasets/cz.py b/fiboa_cli/datasets/cz.py index d728ff8d..ae54ac8d 100644 --- a/fiboa_cli/datasets/cz.py +++ b/fiboa_cli/datasets/cz.py @@ -2,10 +2,10 @@ from vecorel_cli.conversion.admin import AdminConverterMixin from ..conversion.fiboa_converter import FiboaBaseConverter -from .commons.ec import ec_url +from .commons.hcat import AddHCATMixin -class Converter(AdminConverterMixin, FiboaBaseConverter): +class Converter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter): sources = "https://mze.gov.cz/public/app/eagriapp/Files/geoprostor_zadosti23_2024-08-01_202409261243_epsg4258.zip" id = "cz" short_name = "Czech" @@ -24,8 +24,7 @@ class Converter(AdminConverterMixin, FiboaBaseConverter): # 'OKRES_NAZE': 'admin:subdivision_code', } column_migrations = {"DATUM_REP": lambda col: pd.to_datetime(col, format="%d.%m.%Y")} - extensions = {"https://fiboa.org/crop-extension/v0.2.0/schema.yaml"} - column_additions = {"crop:code_list": ec_url("cz_2023.csv")} + ec_mapping_csv = "cz_2023.csv" missing_schemas = { "properties": { "block_id": {"type": "string"}, diff --git a/fiboa_cli/datasets/data-files/ie_2023.csv b/fiboa_cli/datasets/data-files/ie_2023.csv new file mode 100644 index 00000000..ebffeffe --- /dev/null +++ b/fiboa_cli/datasets/data-files/ie_2023.csv @@ -0,0 +1,178 @@ +original_name,HCAT3_name,HCAT3_code +Access Road / Roadways,not_known_and_other,3399000000 +Agroforestry (<400 trees per hectare),tree_wood_forest,3306000000 +Alfalfa,alfalfa_lucerne,3301090301 +Apples,apples,3303010200 +Arable Habitat,arable_crops,3301000000 +Arable Silage (Grass),pasture_meadow_grassland_grass,3302000000 +Arable Silage (No Grass),other_arable_land_crops,3301990000 +Artichoke,artichoke,3301270000 +Asparagus,asparagus,3301200000 +Baby Leaf Spinach,spinach,3301310800 +Barley - Spring,spring_barley,3301010402 +Barley - Winter,winter_barley,3301010401 +Basil,basil,3301061207 +Beans - Spring,beans,3301020100 +Beans - Winter,beans,3301020100 +Beetroot,beetroot_beets,3301290200 +Blackberries,blackberry,3303020200 +Blackcurrants,blackcurrant_cassis,3303020300 +Blueberries,blueberry,3303020400 +Bog,not_known_and_other,3399000000 +Borage,borage,3301061209 +Broccoli - Spring,broccoli,3301210202 +Broccoli - Winter,broccoli,3301210202 +Brussel sprouts,brussels_sprouts,3301210203 +Building,not_known_and_other,3399000000 +Cabbage - Spring,brassica_oleracea_cabbage,3301210200 +Cabbage - Winter,brassica_oleracea_cabbage,3301210200 +Calabrese,broccoli,3301210202 +Camelina,camelina,3301061500 +Carrots,carrots_daucus,3301290300 +Cauliflowers - Spring,cauliflower,3301210204 +Cauliflowers - Winter,cauliflower,3301210204 +Celeriac,celeriac,3301250100 +Celery - Spring,celery,3301250000 +Celery - Winter,celery,3301250000 +Cherries,cherry_cherries,3303010400 +Christmas Trees,other_tree_wood_forest,3306990000 +Clover,clover,3301090303 +Copse,tree_wood_forest,3306000000 +Coriander,coriander,3301061215 +Courgettes,zucchini_courgette,3301140600 +Cucumbers,cucumber_pickle,3301140100 +Daffodils,narcissus_daffodil,3301083300 +Designated Habitat,not_known_and_other,3399000000 +Environmental Management Of Arable Fallow,fallow_land_not_crop,3301110000 +Extensively Grazed Pasture,pasture_meadow_grassland_grass,3302000000 +Fallow,fallow_land_not_crop,3301110000 +Farm Road,not_known_and_other,3399000000 +Farmyard,not_known_and_other,3399000000 +Fennel,fennel,3301170000 +Flax,flax_linseed,3301060700 +Fodder Beet,mangelwurzel_fodder_beet,3301290400 +Foliage,tree_wood_forest,3306000000 +Forage Rape,rapeseed_rape,3301060400 +Forestry,tree_wood_forest,3306000000 +Forestry 2022,tree_wood_forest,3306000000 +Forestry 2023,tree_wood_forest,3306000000 +Forestry Eligible,tree_wood_forest,3306000000 +Forestry ESB Corridor,tree_wood_forest,3306000000 +Forestry ESB Corridor Eligible,tree_wood_forest,3306000000 +Forestry ESB Corridor Ineligible,tree_wood_forest,3306000000 +Forestry Ineligible,tree_wood_forest,3306000000 +Forestry Pre-2009,tree_wood_forest,3306000000 +Forestry Setaside,tree_wood_forest,3306000000 +Gardens,kitchen_gardens,3301120000 +Garlic,garlic,3301220200 +Glasshouse,greenhouse_foil_film,3305000000 +Gooseberries,gooseberry_gooseberries_cranberries,3303020700 +Grapes,vineyards_wine_vine_rebland_grapes,3303060000 +Grass Seed,temporary_grass,3301090100 +Grass Year 1,temporary_grass,3301090100 +Grass Year 1 (MSS Eco-Scheme),temporary_grass,3301090100 +Grass Year 1 (MSS Measure),temporary_grass,3301090100 +Grass Year 1 (MSS),temporary_grass,3301090100 +Grass Year 2,temporary_grass,3301090100 +Grass Year 2 (MSS),temporary_grass,3301090100 +Grass Year 3,temporary_grass,3301090100 +Grass Year 3 (MSS),temporary_grass,3301090100 +Grass Year 4,temporary_grass,3301090100 +Grass Year 4 (MSS),temporary_grass,3301090100 +Grass Year 5,temporary_grass,3301090100 +Grass Year 5 (MSS),temporary_grass,3301090100 +Grassmeal,temporary_grass,3301090100 +Habitat,not_known_and_other,3399000000 +Hemp for Food Use,hemp_cannabis,3301061000 +Hemp for Industrial Use,hemp_cannabis,3301061000 +Inactive,not_known_and_other,3399000000 +Invalid Crop,not_known_and_other,3399000000 +Kale,kale,3301210208 +Lake / Waterway / Pond,not_known_and_other,3399000000 +Leeks - Spring,leek,3301220300 +Leeks - Winter,leek,3301220300 +Lettuce,salads_lettuce_leaf_vegetables,3301310000 +Linnet Habitat,not_known_and_other,3399000000 +Linseed,flax_linseed,3301060700 +Loganberries,berries_berry_species,3303020000 +Low Input Grassland,pasture_meadow_grassland_grass,3302000000 +Low Input Peat Grassland,pasture_meadow_grassland_grass,3302000000 +Low Input Permanent Pasture,pasture_meadow_grassland_grass,3302000000 +Lucerne,alfalfa_lucerne,3301090301 +Lupins,sweet_lupins,3301020700 +Maize,grain_maize_corn_popcorn,3301010600 +Management of Environmental Fallow (horticulture),fallow_land_not_crop,3301110000 +Management of intensive grassland next to a watercourse,pasture_meadow_grassland_grass,3302000000 +Millet,millet_sorghum,3301010900 +Mint,mints_peppermint,3301061222 +Miscanthus Sinensis,miscanthus_silvergrass,3301083000 +Mixed Cropping,arable_crops,3301000000 +Mustard,mustard,3301210100 +Nursery,nurseries_nursery,3303070000 +Oats - Spring,spring_oats,3301010502 +Oats - Winter,winter_oats,3301010501 +Oilseed Rape - Spring,spring_rapeseed_rape,3301060402 +Oilseed Rape - Winter,winter_rapeseed_rape,3301060401 +Onions,onions,3301220400 +Orchard,orchards_fruits,3303010000 +Other cut flower / bulb crops,unspecified_flowers_ornamental_plants,3301089800 +Pak Choi,bok_choy_pak_choi,3301210201 +Parsley,parsly,3301061227 +Parsnips,parsnips,3301290500 +Pears,pears,3303011200 +Peas,peas,3301020600 +Peppers,capsicum,3301300000 +Permanent Pasture,pasture_meadow_grassland_grass,3302000000 +Permanent Pasture (MSS Eco-Scheme 2023),pasture_meadow_grassland_grass,3302000000 +Permanent Pasture (MSS Measure),pasture_meadow_grassland_grass,3302000000 +Permanent Pasture (MSS),pasture_meadow_grassland_grass,3302000000 +Perpetual Spinach,spinach,3301310800 +Planted Buffer Zone,not_known_and_other,3399000000 +Plums,plums,3303011300 +Potatoes - Early,potatoes,3301030000 +Potatoes - Maincrop,potatoes,3301030000 +Potatoes - Seed,potatoes,3301030000 +Protein/Cereal Mix 50/50,legumes_dried_pulses_protein_crops,3301020000 +Pumpkins,pumpkin_squash_gourd,3301140400 +Quarry,not_known_and_other,3399000000 +Quinoa,quinoa,3301150300 +Raspberries,raspberry_raspberries,3303021000 +Recreational Area,not_known_and_other,3399000000 +Red Clover,clover,3301090303 +Redcurrants,redcurrant,3303021100 +Reed Canary Grass,poaceae_grasses,3301090200 +Rhubarb,rhubarb,3301230000 +Riparian Buffer Zone - Arable,not_known_and_other,3399000000 +Riparian Buffer Zone - Grassland,pasture_meadow_grassland_grass,3302000000 +Riparian Zone,not_known_and_other,3399000000 +Rocket,rocket_arugula,3301310600 +Rocky Outcrop,not_known_and_other,3399000000 +Rosemary,rosemary,3301061230 +Rye,rye,3301010300 +Scallions,scallion,3301220500 +Scrub,shrubberries_shrubs,3303080000 +Shallot,shallot,3301220600 +Short Rotation Coppice,tree_wood_forest,3306000000 +Soya Bean,soy_soybeans,3301160000 +Squash,pumpkin_squash_gourd,3301140400 +Strawberries,strawberries,3301130000 +Sugar Beet,sugar_beet,3301290700 +Sunflower,sunflower,3301060500 +Swede,swede_rutabaga,3301210500 +Sweetcorn,grain_maize_corn_popcorn,3301010600 +Thyme,thyme,3301061237 +Tomatoes,tomato,3301280000 +Traditional Hay Meadow,pasture_meadow_grassland_grass,3302000000 +Tree belts for ammonia capture from farmyards,tree_wood_forest,3306000000 +Triticale - Spring,spring_triticale,3301010802 +Triticale - Winter,winter_triticale,3301010801 +Tulips,tulips,3301084900 +Turnips,turnips,3301290800 +Unknown,not_known_and_other,3399000000 +Vetch,vetches,3301090305 +Wheat - Spring,spring_common_soft_wheat,3301010102 +Wheat - Winter,winter_common_soft_wheat,3301010101 +Wild Bird Cover,not_known_and_other,3399000000 +Willow,willows_osiers,3306080000 +Winter Bird Food Plot,not_known_and_other,3399000000 +Woodland,tree_wood_forest,3306000000 diff --git a/fiboa_cli/datasets/data-files/jecam_crop.csv b/fiboa_cli/datasets/data-files/jecam_crop.csv new file mode 100644 index 00000000..f2080a2a --- /dev/null +++ b/fiboa_cli/datasets/data-files/jecam_crop.csv @@ -0,0 +1,106 @@ +crop_name +Agricultural bare soil +Albizia gummifera +Annual crop +Apple tree +Asparagus +Avocado tree +Banana +Barley +Bean +Beet +Bissap +Brede +Cabbage +Cape mahogany +Carrot +Cash woody crop +Cashew tree +Cashew +Cassava +Casava +Cauliflower and brocoli +Cereals +Citrus tree +Citrus +Coffee +Cordia Africana +Cotton +Coton +Cowpea +Croton +Cucumber +Cucurbit +Cyprus +Eggplant +Eucalyptus +Fallow +Fibre crop +Ficus lutea +Fodder +Forest plantation +Fruit crop +Fruit-bearing vegetable +Gabon tulip tree +Goat tree +Gombo +Goyava +Grasses and other fodder crop +Grevillea +Groundnut +Hibiscus +Jatropha +Leafy or stem vegetable +Macadamia tree +Maize +Mango tree +Mango +Market gardening +Mid fallow +Millet +Mixed annual crops +Mixed Cereals +Napier grass +NULL +Oat +Oilseed crop +Old fallow +Onion +Orange tree +Other crop +Papaya tree +Palm +Pea +Peach tree +Pear tree +Pine +Pineapple +Potato +Ravintsara +Rice +Root +Root, bulb or tuberous vegetable +Root/tuber crop with high starch or inulin content +Rubber +Sanio +Sapodilla tree +Sesame +Sorghum +Soybean +Sugarcane +Sunflower +Sweet potato +Taro +Tea +Tomato +Vegetable and root +Vegetables +Vineyard +Watermelon +Wattle tree +Weakly vegetated agricultural +Wheat +Wild radish +Woodlot +Young fallow +Zucchini diff --git a/fiboa_cli/datasets/de_bb.py b/fiboa_cli/datasets/de_bb.py index 0ae8cc37..8b2eff82 100644 --- a/fiboa_cli/datasets/de_bb.py +++ b/fiboa_cli/datasets/de_bb.py @@ -1,42 +1,30 @@ from vecorel_cli.conversion.admin import AdminConverterMixin from ..conversion.fiboa_converter import FiboaBaseConverter +from .commons.hcat import AddHCATMixin -class Converter(AdminConverterMixin, FiboaBaseConverter): - sources = {"https://data.geobasis-bb.de/geofachdaten/Landwirtschaft/dfbk.zip": ["DFBK_FB.shp"]} +class Converter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter): + sources = "https://data.geobasis-bb.de/geofachdaten/Landwirtschaft/antrag.zip" id = "de_bb" admin_subdivision_code = "BB" # TODO Berlin is also in here, check each row short_name = "Germany, Berlin/Brandenburg" title = "Field boundaries for Berlin / Brandenburg, Germany" - description = """A field block (German: "Feldblock") is a contiguous agricultural area surrounded by permanent boundaries, which is cultivated by one or more farmers with one or more crops, is fully or partially set aside or is fully or partially taken out of production.""" + description = """A Crop Field (German: "Schlaege") is a contiguous agricultural area surrounded by permanent boundaries, which is cultivated with a single crop.""" license = "DL-DE-BY-2.0" provider = "Land Brandenburg " - extensions = {"https://fiboa.org/flik-extension/v0.2.0/schema.yaml"} + ec_mapping_csv = "de.csv" columns = { "geometry": "geometry", - "FB_ID": ["flik", "id"], - "FGUE_JAHR": "fgue_jahr", - "FL_BRUTTO_": "metrics:area", - "FL_NETTO_H": "net_area", - "GUELTVON_F": "determination:datetime", - "GUELTBIS_F": "expiry_datetime", - "KREIS_NR": "kreis_nr", - "TK10_BLATT": "tk10", - "HBN_KAT": "hbn", - "SHAPE_LEN": "metrics:perimeter", + "ref_ident": "farmer_id", + "groesse": "metrics:area", + "guelt_von": "determination:datetime", + "code_bez": "crop:name", + "code": "crop:code", } missing_schemas = { "properties": { - "hbn": {"type": "string"}, - "fgue_jahr": {"type": "string"}, - "net_area": {"type": "float", "exclusiveMinimum": 0}, - "expiry_datetime": {"type": "date-time"}, - "kreis_nr": {"type": "uint16"}, - "tk10": {"type": "string"}, + "farmer_id": {"type": "string"}, } } - - def layer_filter(self, layer: str, uri: str) -> bool: - return layer == "DFBK_FB" diff --git a/fiboa_cli/datasets/de_bb_block.py b/fiboa_cli/datasets/de_bb_block.py new file mode 100644 index 00000000..6000dbba --- /dev/null +++ b/fiboa_cli/datasets/de_bb_block.py @@ -0,0 +1,42 @@ +from vecorel_cli.conversion.admin import AdminConverterMixin + +from ..conversion.fiboa_converter import FiboaBaseConverter + + +class Converter(AdminConverterMixin, FiboaBaseConverter): + sources = {"https://data.geobasis-bb.de/geofachdaten/Landwirtschaft/dfbk.zip": ["DFBK_FB.shp"]} + id = "de_bb_block" + admin_subdivision_code = "BB" + short_name = "Germany, Berlin/Brandenburg" + title = "Field boundaries for Berlin / Brandenburg, Germany" + description = """A field block (German: "Feldblock") is a contiguous agricultural area surrounded by permanent boundaries, which is cultivated by one or more farmers with one or more crops, is fully or partially set aside or is fully or partially taken out of production.""" + license = "DL-DE-BY-2.0" + provider = "Land Brandenburg " + extensions = {"https://fiboa.org/flik-extension/v0.2.0/schema.yaml"} + + columns = { + "geometry": "geometry", + "FB_ID": ("flik", "id"), + "FGUE_JAHR": "fgue_jahr", + "FL_BRUTTO": "metrics:area", + "FL_NETTO": "net_area", + "GUELTVON_F": "determination:datetime", + "GUELTBIS_F": "expiry_datetime", + "KREIS_NR": "kreis_nr", + "TK10_BLATT": "tk10", + "HBN_KAT": "hbn", + "SHAPE_LEN": "metrics:perimeter", + } + missing_schemas = { + "properties": { + "hbn": {"type": "string"}, + "fgue_jahr": {"type": "string"}, + "net_area": {"type": "float", "exclusiveMinimum": 0}, + "expiry_datetime": {"type": "date-time"}, + "kreis_nr": {"type": "uint16"}, + "tk10": {"type": "string"}, + } + } + + def layer_filter(self, layer: str, uri: str) -> bool: + return layer == "DFBK_FB" diff --git a/fiboa_cli/datasets/de_mv.py b/fiboa_cli/datasets/de_mv.py index aef59828..ce62ff95 100644 --- a/fiboa_cli/datasets/de_mv.py +++ b/fiboa_cli/datasets/de_mv.py @@ -1,6 +1,7 @@ from vecorel_cli.conversion.admin import AdminConverterMixin from ..conversion.fiboa_converter import FiboaBaseConverter +from .commons.hcat import CROP_EXTENSION class Converter(AdminConverterMixin, FiboaBaseConverter): @@ -15,14 +16,16 @@ class Converter(AdminConverterMixin, FiboaBaseConverter): provider = "Ministerium für Landwirtschaft und Umwelt M-V " license = "No restrictions apply " - extensions = {"https://fiboa.org/flik-extension/v0.2.0/schema.yaml"} + extensions = {"https://fiboa.org/flik-extension/v0.2.0/schema.yaml", CROP_EXTENSION} + column_additions = {"crop:code_list": "https://fiboa.org/code/de/de_mv.csv"} + + # ec_mapping_csv = "de.csv" columns = { "geometry": "geometry", "fbid": ("id", "flik"), # make flik id a dedicated column to align with NRW etc. "dgl_jahr": "dgl_jahr", - # TODO implement crop:code extension - "bodennutzu": "bodennutzu", # Bodennutzungsart + "bodennutzu": "crop:code", # Bodennutzungsart "bez_kreis": "bez_kreis", # Kreisbezeichnung "groesse_p": "metrics:area", # Produktive Fläche des FB in Hektar (Nettofläche) "perimeter": "metrics:perimeter", # Polygonumfang @@ -52,33 +55,21 @@ class Converter(AdminConverterMixin, FiboaBaseConverter): missing_schemas = { "properties": { "dgl_jahr": {"type": "int16"}, - "bodennutzu": { - "type": "string", - "enum": [ - "AAF", # Aufforstung auf Ackerfläche - "AF", # Ackerfläche - "AGL", # Aufforstung auf Grünland - "AÖD", # Aufforstung auf Ödland - "DK", # Dauerkultur - "FO", # Forst - "GL", # Grünland - ], - }, "bez_kreis": {"type": "string"}, "erwind": { "type": "string", "enum": [ - "CC0", # nicht relevant für Cross Compliance - "CC1", # Erosionsgefährdung nach Direktzahlungen-Verpflichtungenverordnung + "0", # nicht relevant für Cross Compliance + "1", # Erosionsgefährdung nach Direktzahlungen-Verpflichtungenverordnung "-", # keine Angabe ], }, "erwater": { "type": "string", "enum": [ - "CC0", # nicht relevant für Cross Compliance - "CC1", # Erosionsgefährdung nach Direktzahlungen-Verpflichtungenverordnung (15-27,5 t/ha/a Bodenabtrag durch Wasser) - "CC2", # hohe Erosionsgefährdung nach Direktzahlungen-Verpflichtungenverordnung (>27,5 t/ha/a Bodenabtrag durch Wasser) + "0", # nicht relevant für Cross Compliance + "1", # Erosionsgefährdung nach Direktzahlungen-Verpflichtungenverordnung (15-27,5 t/ha/a Bodenabtrag durch Wasser) + "2", # hohe Erosionsgefährdung nach Direktzahlungen-Verpflichtungenverordnung (>27,5 t/ha/a Bodenabtrag durch Wasser) "-", # keine Angabe ], }, diff --git a/fiboa_cli/datasets/de_nds.py b/fiboa_cli/datasets/de_nds.py index 23b3eedd..732a6d60 100644 --- a/fiboa_cli/datasets/de_nds.py +++ b/fiboa_cli/datasets/de_nds.py @@ -1,33 +1,62 @@ +import pandas as pd from vecorel_cli.conversion.admin import AdminConverterMixin from ..conversion.fiboa_converter import FiboaBaseConverter +from .commons.hcat import AddHCATMixin -class Converter(AdminConverterMixin, FiboaBaseConverter): - sources = "https://sla.niedersachsen.de/mapbender_sla/download/FB_NDS.zip" +class Converter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter): + """ + https://sla.niedersachsen.de/agrarfoerderung/schlaginfo/ (see download) + The zip contains: + - Schlaege = UD_25_S.shp + - TeilLandschaftElemente = UD_25_TLE.shp + - TeilSchlaege = UD_25_TS.shp + """ + + variants = { + f"{year}": { + f"https://sla.niedersachsen.de/mapbender_sla/download/schlaege_aktuell_{year}.zip": [ + f"UD_{year % 100}_S.shp" + ] + } + for year in range(2025, 2020, -1) + } id = "de_nds" admin_subdivision_code = "NI" short_name = "Germany, Lower Saxony/Bremen/Hamburg" - title = "Field boundaries for Lower Saxony / Bremen / Hamburg, Germany" - description = """A field block (German: "Feldblock") is a contiguous agricultural area surrounded by permanent boundaries, which is cultivated by one or more farmers with one or more crops, is fully or partially set aside or is fully or partially taken out of production.""" + title = "Crop Fields for Lower Saxony / Bremen / Hamburg, Germany" + description = """A Crop Field (German: "Schlaege") is a contiguous agricultural area surrounded by permanent boundaries, which is cultivated with a single crop.""" provider = "ML/SLA Niedersachsen " attribution = "© ML/SLA Niedersachsen (2024), DL-DE-BY-2.0 (www.govdata.de/DL-DE-BY-2.0), Daten bearbeitet" license = "DL-DE-BY-2.0" extensions = {"https://fiboa.org/flik-extension/v0.2.0/schema.yaml"} + + # https://www.sla.niedersachsen.de/download/141235/Verzeichnis_Nutzungscodes.xlsx + ec_mapping_csv = "de.csv" columns = { "geometry": "geometry", - "FLIK": ["id", "flik"], - "STAND": "determination:datetime", - "ANT_JAHR": "ant_jahr", - "BNK": "bnk", - "BNK_TXT": "bnk_txt", - "FLAECHE": "metrics:area", - "SHAPE_Leng": "metrics:perimeter", + "FLIK": "flik", + "SCHLAGNR": "subfield_id", + "NC_FESTG": "crop:code", + "ANTRAGSJAH": "determination:datetime", + "AKTUELLEFL": "metrics:area", } missing_schemas = { "properties": { - "ant_jahr": {"type": "int16"}, - "bnk": {"type": "string"}, - "bnk_txt": {"type": "string"}, + "subfield_id": {"type": "int64"}, } } + column_migrations = {"ANTRAGSJAH": lambda col: pd.to_datetime(col, format="%Y")} + + def migrate(self, gdf): + if "NC_FESTG" not in gdf.columns: + code = {"KULTURARTF", "KULTURCODE", "KC_FESTG"} & set(gdf.columns) + del self.columns["NC_FESTG"] + self.columns[code.pop()] = "crop:code" + + if "AKTUELLEFL" not in gdf.columns: + del self.columns["AKTUELLEFL"] + self.columns["AKT_FL"] = "metrics:area" + + return super().migrate(gdf) diff --git a/fiboa_cli/datasets/de_nds_block.py b/fiboa_cli/datasets/de_nds_block.py new file mode 100644 index 00000000..7621434a --- /dev/null +++ b/fiboa_cli/datasets/de_nds_block.py @@ -0,0 +1,33 @@ +from vecorel_cli.conversion.admin import AdminConverterMixin + +from ..conversion.fiboa_converter import FiboaBaseConverter + + +class Converter(AdminConverterMixin, FiboaBaseConverter): + sources = "https://sla.niedersachsen.de/mapbender_sla/download/FB_NDS.zip" + id = "de_nds_block" + admin_subdivision_code = "NI" + short_name = "Germany, Lower Saxony/Bremen/Hamburg" + title = "Field boundaries for Lower Saxony / Bremen / Hamburg, Germany" + description = """A field block (German: "Feldblock") is a contiguous agricultural area surrounded by permanent boundaries, which is cultivated by one or more farmers with one or more crops, is fully or partially set aside or is fully or partially taken out of production.""" + provider = "ML/SLA Niedersachsen " + attribution = "© ML/SLA Niedersachsen (2024), DL-DE-BY-2.0 (www.govdata.de/DL-DE-BY-2.0), Daten bearbeitet" + license = "DL-DE-BY-2.0" + extensions = {"https://fiboa.org/flik-extension/v0.2.0/schema.yaml"} + columns = { + "geometry": "geometry", + "FLIK": ("id", "flik"), + "STAND": "determination:datetime", + "ANT_JAHR": "ant_jahr", + "BNK": "bnk", + "BNK_TXT": "bnk_txt", + "FLAECHE": "metrics:area", + "SHAPE_Leng": "metrics:perimeter", + } + missing_schemas = { + "properties": { + "ant_jahr": {"type": "int16"}, + "bnk": {"type": "string"}, + "bnk_txt": {"type": "string"}, + } + } diff --git a/fiboa_cli/datasets/de_nrw.py b/fiboa_cli/datasets/de_nrw.py index e4913685..58f95c86 100644 --- a/fiboa_cli/datasets/de_nrw.py +++ b/fiboa_cli/datasets/de_nrw.py @@ -5,7 +5,7 @@ class Converter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter): - sources = "https://www.opengeodata.nrw.de/produkte/umwelt_klima/bodennutzung/landwirtschaft/LFK-AKTI_EPSG25832_Shape.zip" + sources = "https://www.opengeodata.nrw.de/produkte/umwelt_klima/bodennutzung/landwirtschaft/LWK-TSCHLAG_EPSG25832_Shape.zip" id = "de_nrw" admin_subdivision_code = "NW" short_name = "Germany, North Rhine-Westphalia" @@ -21,10 +21,9 @@ class Converter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter): columns = { "geometry": "geometry", "ID": "id", - "INSPIRE_ID": "inspire:id", "FLIK": "flik", - "GUELT_VON": "determination:datetime", - "NUTZ_CODE": "crop:code", - "NUTZ_TXT": "crop:name", + "BEGINLIFES": "determination:datetime", + "CODE": "crop:code", + "CODE_TXT": "crop:name", "AREA_HA": "metrics:area", } diff --git a/fiboa_cli/datasets/de_sax.py b/fiboa_cli/datasets/de_sax.py index 055f47a9..5412f708 100644 --- a/fiboa_cli/datasets/de_sax.py +++ b/fiboa_cli/datasets/de_sax.py @@ -21,7 +21,7 @@ class Converter(AdminConverterMixin, FiboaBaseConverter): extensions = {"https://fiboa.org/flik-extension/v0.2.0/schema.yaml"} columns = { "geometry": "geometry", - "FB_FLIK": ["id", "flik"], + "FB_FLIK": ("id", "flik"), "JAHR": "determination:datetime", "FB_A_FLAE": "metrics:area", "FB_BN_KAT": "FB_BN_KAT", diff --git a/fiboa_cli/datasets/de_sh.py b/fiboa_cli/datasets/de_sh.py index a3fc84b8..dcc2818c 100644 --- a/fiboa_cli/datasets/de_sh.py +++ b/fiboa_cli/datasets/de_sh.py @@ -16,7 +16,7 @@ class Converter(AdminConverterMixin, FiboaBaseConverter): columns = { "geometry": "geometry", "fachguelti": "determination:datetime", - "FLIK": ["flik", "id"], + "FLIK": ("flik", "id"), "Flaeche": "metrics:area", "HBN": "hbn", } diff --git a/fiboa_cli/datasets/de_sl.py b/fiboa_cli/datasets/de_sl.py index 0eb3b489..5ab3185e 100644 --- a/fiboa_cli/datasets/de_sl.py +++ b/fiboa_cli/datasets/de_sl.py @@ -54,4 +54,4 @@ class Converter(AdminConverterMixin, FiboaBaseConverter): def migrate(self, gdf): gdf["flik"] = gdf["description"].apply(parse_flik) gdf["area"] = gdf["description"].apply(parse_size) - return gdf + return super().migrate(gdf) diff --git a/fiboa_cli/datasets/de_th.py b/fiboa_cli/datasets/de_th.py index 97db484f..d272b1d9 100644 --- a/fiboa_cli/datasets/de_th.py +++ b/fiboa_cli/datasets/de_th.py @@ -67,7 +67,7 @@ class Converter(AdminConverterMixin, FiboaBaseConverter): def migrate(self, gdf): col = "GEO_UPDAT" gdf[col] = pd.to_datetime(gdf[col], format="%d.%m.%Y", utc=True) - return gdf + return super().migrate(gdf) column_filters = {"LF": lambda col: col == "LF"} diff --git a/fiboa_cli/datasets/dk.py b/fiboa_cli/datasets/dk.py index 66434298..7dab9ec4 100644 --- a/fiboa_cli/datasets/dk.py +++ b/fiboa_cli/datasets/dk.py @@ -2,10 +2,10 @@ from vecorel_cli.conversion.admin import AdminConverterMixin from ..conversion.fiboa_converter import FiboaBaseConverter -from .commons.ec import ec_url +from .commons.hcat import AddHCATMixin -class DKConverter(AdminConverterMixin, FiboaBaseConverter): +class DKConverter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter): variants = { str(variant): f"https://landbrugsgeodata.fvm.dk/Download/Marker/Marker_{variant}.zip" for variant in range(2024, 2008 - 1, -1) @@ -15,11 +15,8 @@ class DKConverter(AdminConverterMixin, FiboaBaseConverter): title = "Denmark Crop Fields (Marker)" description = "The Danish Ministry of Food, Agriculture and Fisheries publishes Crop Fields (Marker) for each year." - extensions = {"https://fiboa.org/crop-extension/v0.2.0/schema.yaml"} - column_additions = {"crop:code_list": ec_url("nl_2020.csv")} - provider = "Danish Agricultural Agency " - + ec_mapping_csv = "dk_2019.csv" license = "CC0-1.0" columns = { "geometry": "geometry", @@ -30,5 +27,6 @@ class DKConverter(AdminConverterMixin, FiboaBaseConverter): } def migrate(self, gdf) -> gpd.GeoDataFrame: + gdf["Afgkode"] = gdf["Afgkode"].astype(float).fillna(value=0).astype(int).astype(str) gdf["determination:datetime"] = f"{self.variant}-01-01T00:00:00Z" - return gdf + return super().migrate(gdf) diff --git a/fiboa_cli/datasets/ec_nl_crop.py b/fiboa_cli/datasets/ec_nl_crop.py index 3fb0a82c..c81c57d1 100644 --- a/fiboa_cli/datasets/ec_nl_crop.py +++ b/fiboa_cli/datasets/ec_nl_crop.py @@ -1,5 +1,5 @@ from .commons.ec import EuroCropsConverterMixin -from .nl_crop import NLCropConverter +from .nl import NLCropConverter class NLEuroCropConverter(EuroCropsConverterMixin, NLCropConverter): diff --git a/fiboa_cli/datasets/ee.py b/fiboa_cli/datasets/ee.py new file mode 100644 index 00000000..777e827b --- /dev/null +++ b/fiboa_cli/datasets/ee.py @@ -0,0 +1,39 @@ +import pandas as pd + +from ..conversion.fiboa_converter import FiboaBaseConverter +from .commons.hcat import AddHCATMixin + +COLUMNS = { + "geometry": "geometry", + "pollu_id": "id", + "taotlusaasta": "determination:datetime", # year + "pindala_ha": "metrics:area", # area (in ha) + "taotletud_kultuur": "crop:name", # requested crop culture +} +ATTRIBUTES = ",".join(["geom" if k == "geometry" else k for k in COLUMNS.keys()]) + + +class Convert(AddHCATMixin, FiboaBaseConverter): + variants = { + str( + year + ): f"https://kls.pria.ee/geoserver/inspire_gsaa/wfs?service=WFS&version=2.0.0&request=GetFeature&typeName=inspire_gsaa:LU.GSAA.AGRICULTURAL_PARCELS_{year}&propertyName={ATTRIBUTES}" + for year in range(2024, 2009, -1) + } + ec_mapping_csv = "https://fiboa.org/code/ee/ee.csv" + id = "ee" + short_name = "Estonia" + title = "Field boundaries for Estonia" + description = """ +Geospatial Aid Application Estonia Agricultural parcels. +The original dataset is provided by ARIB and obtained from the INSPIRE theme GSAA (specifically Geospaial Aid Application Estonia Agricultural parcels) through which the data layer Fields and Eco Areas (GSAA) is made available. +The data comes from ARIB's database of agricultural parcels. + """ + provider = "Põllumajanduse Registrite ja Informatsiooni Amet " + attribution = "© Põllumajanduse Registrite ja Informatsiooni Amet" + license = "CC-BY-SA-3.0" + columns = COLUMNS + column_migrations = {"taotlusaasta": lambda col: pd.to_datetime(col, format="%Y")} + + def file_migration(self, gdf, path: str, uri: str, layer=None): + return gdf.set_crs(3301) diff --git a/fiboa_cli/datasets/es.py b/fiboa_cli/datasets/es.py index 46a97a35..4d2fd9d4 100644 --- a/fiboa_cli/datasets/es.py +++ b/fiboa_cli/datasets/es.py @@ -47,4 +47,4 @@ def migrate(self, gdf): mapping_en = {row["original_code"]: row["name_en"] for row in rows} gdf["crop:name"] = gdf[self.use_code_attribute].map(mapping) gdf["crop:name_en"] = gdf[self.use_code_attribute].map(mapping_en) - return gdf + return super().migrate(gdf) diff --git a/fiboa_cli/datasets/es_an.py b/fiboa_cli/datasets/es_an.py index f93e7dea..44abd2fe 100644 --- a/fiboa_cli/datasets/es_an.py +++ b/fiboa_cli/datasets/es_an.py @@ -6,6 +6,7 @@ class ANConverter(ESBaseConverter): variants = { + "2025": "https://www.juntadeandalucia.es/ssdigitales/festa/agriculturapescaaguaydesarrollorural/2025/SP25_REC_PROV_{code}.zip", "2024": "https://www.juntadeandalucia.es/ssdigitales/festa/agriculturapescaaguaydesarrollorural/2024/SP24_REC_{code}.zip", "2023": "https://www.juntadeandalucia.es/ssdigitales/festa/agriculturapescaaguaydesarrollorural/2023/SP23_REC_{code}.zip", "2022": "https://www.juntadeandalucia.es/export/drupaljda/01_SP22_REC_PROV_{code}.zip", diff --git a/fiboa_cli/datasets/es_cat.py b/fiboa_cli/datasets/es_cat.py index 5a8498d9..1987bcfe 100644 --- a/fiboa_cli/datasets/es_cat.py +++ b/fiboa_cli/datasets/es_cat.py @@ -77,4 +77,4 @@ def migrate(self, gdf): assert len(missing) == 0, f"Can not map crops {missing}" gdf["crop:code"] = gdf["cultiu"].map(mapping) gdf["crop:name_en"] = gdf["cultiu"].map(mapping_en) - return gdf + return super().migrate(gdf) diff --git a/fiboa_cli/datasets/es_cl.py b/fiboa_cli/datasets/es_cl.py index 7e4e6ed4..573f0ceb 100644 --- a/fiboa_cli/datasets/es_cl.py +++ b/fiboa_cli/datasets/es_cl.py @@ -51,12 +51,10 @@ def download_files(self, uris, cache_folder=None): def get_urls(self): if not self.variant: - self.variant = "2024" + self.variant = "2025" logger.warning(f"Choosing first year {self.variant}") else: - assert self.variant in "2024 2023 2022 2021 2020 2019".split(), ( - f"Wrong year {self.variant}" - ) + assert 2019 <= int(self.variant) <= 2025, f"Wrong year {self.variant}" base = f"http://ftp.itacyl.es/cartografia/05_SIGPAC/{self.variant}_ETRS89/Parcelario_SIGPAC_CyL_Provincias/" response = requests.get(base) assert response.status_code == 200, f"Error getting urls {response}\n{response.content}" diff --git a/fiboa_cli/datasets/es_pv.py b/fiboa_cli/datasets/es_pv.py index deda3e8f..e4fbe3b5 100644 --- a/fiboa_cli/datasets/es_pv.py +++ b/fiboa_cli/datasets/es_pv.py @@ -10,7 +10,7 @@ class ESPVConverter(ESBaseConverter): str( year ): f"https://www.geo.euskadi.eus/cartografia/DatosDescarga/Agricultura/SIGPAC/SIGPAC_CAMPA%C3%91A_{year}_V1/" - for year in range(2024, 2015, -1) + for year in range(2025, 2015, -1) } id = "es_pv" short_name = "Spain Basque Country" diff --git a/fiboa_cli/datasets/fi.py b/fiboa_cli/datasets/fi.py index 36ebf420..ed79c059 100644 --- a/fiboa_cli/datasets/fi.py +++ b/fiboa_cli/datasets/fi.py @@ -2,10 +2,10 @@ from vecorel_cli.conversion.admin import AdminConverterMixin from ..conversion.fiboa_converter import FiboaBaseConverter -from .commons.ec import ec_url +from .commons.hcat import AddHCATMixin -class Converter(AdminConverterMixin, FiboaBaseConverter): +class Converter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter): sources = "https://download.inspire.ruokavirasto-awsa.com/data/2023/LandUse.ExistingLandUse.GSAAAgriculturalParcel.gpkg" id = "fi" short_name = "Finland" @@ -31,8 +31,7 @@ class Converter(AdminConverterMixin, FiboaBaseConverter): # Make year (1st January) from column "VUOSI" "VUOSI": lambda col: pd.to_datetime(col, format="%Y"), } - extensions = {"https://fiboa.org/crop-extension/v0.2.0/schema.yaml"} - column_additions = {"crop:code_list": ec_url("fi_2020.csv")} + ec_mapping_csv = "https://fiboa.org/code/fi/fi_2023.csv" area_is_in_ha = False area_calculate_missing = True diff --git a/fiboa_cli/datasets/hr.py b/fiboa_cli/datasets/hr.py index 59b7476a..08ea26e0 100644 --- a/fiboa_cli/datasets/hr.py +++ b/fiboa_cli/datasets/hr.py @@ -1,11 +1,11 @@ +import geopandas as gpd from vecorel_cli.conversion.admin import AdminConverterMixin from ..conversion.fiboa_converter import FiboaBaseConverter -from .commons.data import read_data_csv -from .commons.ec import ec_url +from .commons.hcat import AddHCATMixin -class Converter(AdminConverterMixin, FiboaBaseConverter): +class Converter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter): sources = "https://www.apprrr.hr/wp-content/uploads/nipp/land_parcels.gpkg" id = "hr" short_name = "Croatia" @@ -31,22 +31,20 @@ class Converter(AdminConverterMixin, FiboaBaseConverter): columns = { "id": "id", "land_use_id": "crop:code", - "crop:name": "crop:name", - "crop:name_en": "crop:name_en", "area": "metrics:area", "geometry": "geometry", "home_name": "home_name", "perim": "metrics:perimeter", "slope": "slope", - "z_avg": "z_avg", + "z_avg": "height", "eligibility_coef": "eligibility_coef", - "mines_status": "mines_status", - "mines_year_removed": "mines_year_removed", + # "mines_status": "mines_status", + # "mines_year_removed": "mines_year_removed", "water_protect_zone": "water_protect_zone", "natura2000": "natura2000", - "natura2000_ok": "natura2000_ok", - "natura2000_pop": "natura2000_pop", - "natura2000_povs": "natura2000_povs", + # "natura2000_ok": "natura2000_ok", + # "natura2000_pop": "natura2000_pop", + # "natura2000_povs": "natura2000_povs", "anc": "anc", "anc_area": "anc_area", "rp": "rp", @@ -60,8 +58,11 @@ class Converter(AdminConverterMixin, FiboaBaseConverter): "jpaid": "jpaid", } - extensions = {"https://fiboa.org/crop-extension/v0.2.0/schema.yaml"} - column_additions = {"crop:code_list": ec_url("hr_2020.csv")} + ec_mapping_csv = "hr_2020.csv" + + def migrate(self, gdf) -> gpd.GeoDataFrame: + gdf["land_use_id"] = gdf["land_use_id"].astype(int) + return super().migrate(gdf) missing_schemas = { "required": [ @@ -73,10 +74,10 @@ class Converter(AdminConverterMixin, FiboaBaseConverter): "jpaid", ], "properties": { - "land_use_id": {"type": "double"}, + "land_use_id": {"type": "integer"}, "home_name": {"type": "string"}, "slope": {"type": "double"}, - "z_avg": {"type": "double"}, + "height": {"type": "double"}, "eligibility_coef": {"type": "double"}, "mines_status": {"type": "string", "enum": ["N", "M", "R"]}, "mines_year_removed": {"type": "int32"}, @@ -99,14 +100,5 @@ class Converter(AdminConverterMixin, FiboaBaseConverter): }, } - def migrate(self, gdf): - gdf = super().migrate(gdf) - rows = read_data_csv("hr_categories.csv", delimiter=";") - mapping = {int(row["code"]): row["name"] for row in rows} - mapping_en = {int(row["code"]): row["name_en"] for row in rows} - gdf["crop:name"] = gdf["land_use_id"].map(mapping) - gdf["crop:name_en"] = gdf["land_use_id"].map(mapping_en) - return gdf - area_is_in_ha = False area_calculate_missing = True diff --git a/fiboa_cli/datasets/ie.py b/fiboa_cli/datasets/ie.py index fa5ab0ea..611b47a9 100644 --- a/fiboa_cli/datasets/ie.py +++ b/fiboa_cli/datasets/ie.py @@ -3,13 +3,18 @@ from ..conversion.convert_gml import gml_assure_columns from ..conversion.fiboa_converter import FiboaBaseConverter +from .commons.data import read_data_csv +from .commons.hcat import AddHCATMixin -class IEConverter(AdminConverterMixin, FiboaBaseConverter): - sources = { - "https://osi-inspire-atom.s3-eu-west-1.amazonaws.com/IACSdata/IACS_GSAA_2022.zip": [ - "IACS_GSAA_2022.gml" - ] +class IEConverter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter): + variants = { + str(year): { + f"https://dafm-inspire-atom.s3.eu-west-1.amazonaws.com/files/LU/GSAA_{year}.zip": [ + f"GSAA_{year}.gml" + ] + } + for year in range(2024, 2021, -1) } id = "ie" @@ -22,24 +27,24 @@ class IEConverter(AdminConverterMixin, FiboaBaseConverter): license = "CC-BY-4.0" columns = { "geometry": "geometry", - "crop_name": "crop_name", + "crop_name": "crop:name", + "crop_code": "crop:code", "localId": "id", "determination:datetime": "determination:datetime", } - # TODO use crop-extension, maybe with reverse mapping - - missing_schemas = { - "properties": { - "crop_name": {"type": "string"}, - } - } + ec_mapping_csv = "https://fiboa.org/code/ie/ie.csv" def migrate(self, gdf) -> gpd.GeoDataFrame: # crop_name can be multiple: "crop1, crop2, crop3". We only read the main crop (first). gdf["crop_name"] = gdf["crop_name"].str.split(", ").str.get(0) gdf = gdf[gdf["crop_name"] != "Void"] # Exclude non-agriculture fields + + rows = read_data_csv("ie_2023.csv") + mapping = {row["original_name"]: index + 1 for index, row in enumerate(rows)} + gdf["crop_code"] = gdf["crop_name"].map(mapping) + gdf["determination:datetime"] = gdf["observationDate"].str.replace("+01:00", "T00:00:00Z") - return gdf + return super().migrate(gdf) def file_migration( self, gdf: gpd.GeoDataFrame, path: str, uri: str, layer: str = None diff --git a/fiboa_cli/datasets/jecam.py b/fiboa_cli/datasets/jecam.py index 720e9fb7..70b6cbfc 100644 --- a/fiboa_cli/datasets/jecam.py +++ b/fiboa_cli/datasets/jecam.py @@ -3,6 +3,9 @@ from fiboa_cli.conversion.fiboa_converter import FiboaBaseConverter from fiboa_cli.datasets.commons.data import read_data_csv +from fiboa_cli.datasets.commons.hcat import CROP_EXTENSION + +CODE_LIST = "https://fiboa.org/code/jecam/crop.csv" class JecamConvert(FiboaBaseConverter): @@ -38,13 +41,14 @@ class JecamConvert(FiboaBaseConverter): "AcquiDate": "determination:datetime", "admin:country_code": "admin:country_code", "SiteName": "site_name", + "crop:code": "crop:code", "CropType1": "crop:name", "Irrigated": "irrigated", } column_additions = { - "crop:code_list": "https://fiboa.org/code/jecam/crop.csv", + "crop:code_list": CODE_LIST, } - extensions = {ADMIN_DIVISION} + extensions = {ADMIN_DIVISION, CROP_EXTENSION} missing_schemas = { "properties": { "site_name": {"type": "string"}, @@ -60,6 +64,10 @@ def migrate(self, gdf) -> gpd.GeoDataFrame: mapping = {row["name"]: row["alpha-2"] for row in rows} gdf["admin:country_code"] = gdf["Country"].map(mapping) + rows = read_data_csv("jecam_crop.csv") + mapping = {row["crop_name"]: index + 1 for index, row in enumerate(rows)} + gdf["crop:code"] = gdf["CropType1"].map(mapping) + gdf.loc[gdf["Area_ha"] == 0, "Area_ha"] = None gdf["Irrigated"] = gdf["Irrigated"].astype(bool) return gdf diff --git a/fiboa_cli/datasets/lt.py b/fiboa_cli/datasets/lt.py index b88a1346..42a00b3a 100644 --- a/fiboa_cli/datasets/lt.py +++ b/fiboa_cli/datasets/lt.py @@ -10,8 +10,4 @@ class LTConverter(EuroLandBaseConverter): provider = "Nacionalinė mokėjimo agentūra prie Žemės ūkio ministerijos " attribution = "Nacionalinė mokėjimo agentūra prie Žemės ūkio ministerijos" ec_mapping_csv = "lt_2021.csv" - sources = { - "https://zenodo.org/records/14384070/files/LT_2024.zip?download=1": [ - "GSA-LT-2024.geoparquet" - ] - } + sources = {"https://zenodo.org/records/14384070/files/LT_2024.zip": ["GSA-LT-2024.geoparquet"]} diff --git a/fiboa_cli/datasets/lv.py b/fiboa_cli/datasets/lv.py index 74259902..aa62293f 100644 --- a/fiboa_cli/datasets/lv.py +++ b/fiboa_cli/datasets/lv.py @@ -1,7 +1,7 @@ from vecorel_cli.conversion.admin import AdminConverterMixin from ..conversion.fiboa_converter import FiboaBaseConverter -from .commons.ec import AddHCATMixin, ec_url +from .commons.ec import AddHCATMixin count = 3000 @@ -27,7 +27,6 @@ class Converter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter): Relevant datasets are: Country blocks (Lauku Bloki), Fields (Lauki), and Landscape elements. """ - extensions = {"https://fiboa.org/crop-extension/v0.2.0/schema.yaml"} provider = "Rural Support Service Republic of Latvia (Lauku atbalsta dienests) " attribution = "Lauku atbalsta dienests" license = "CC-BY-SA-4.0" # Not sure, taken from Eurocrops. It is "public" and free and "available to any user" @@ -37,10 +36,8 @@ class Converter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter): "geometry": "geometry", "DATA_CHANGED_DATE": "determination:datetime", "area": "metrics:area", - "crop:code_list": "crop:code_list", "PRODUCT_CODE": "crop:code", - "crop:name": "crop:name", - "crop:name_en": "crop:name_en", + "PRODUCT_DESCRIPTION": "crop:name", } missing_schemas = { "properties": { @@ -50,17 +47,7 @@ class Converter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter): } } ec_mapping_csv = "lv_2021.csv" - + column_migrations = { + "PRODUCT_CODE": lambda col: col.fillna(0).astype(int).astype(str), + } area_calculate_missing = True - - def migrate(self, gdf): - gdf = super().migrate(gdf) - - original_name_mapping = { - int(e["original_code"]): e["original_name"] for e in self.ec_mapping - } - name_mapping = {int(e["original_code"]): e["translated_name"] for e in self.ec_mapping} - gdf["crop:code_list"] = ec_url("lv_2021.csv") - gdf["crop:name"] = gdf["PRODUCT_CODE"].map(original_name_mapping) - gdf["crop:name_en"] = gdf["PRODUCT_CODE"].map(name_mapping) - return gdf diff --git a/fiboa_cli/datasets/nl.py b/fiboa_cli/datasets/nl.py index 1d2ef574..3bdc9213 100644 --- a/fiboa_cli/datasets/nl.py +++ b/fiboa_cli/datasets/nl.py @@ -1,46 +1,69 @@ +import pandas as pd from vecorel_cli.conversion.admin import AdminConverterMixin from ..conversion.fiboa_converter import FiboaBaseConverter +from .commons.hcat import AddHCATMixin +# see https://service.pdok.nl/rvo/brpgewaspercelen/atom/v1_0/basisregistratie_gewaspercelen_brp.xml +base = "https://service.pdok.nl/rvo/brpgewaspercelen/atom/v1_0/downloads" -class Converter(AdminConverterMixin, FiboaBaseConverter): - sources = ( - "https://service.pdok.nl/rvo/referentiepercelen/atom/downloads/referentiepercelen.gpkg" - ) + +class NLCropConverter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter): + area_calculate_missing = True + variants = { + "2025": f"{base}/brpgewaspercelen_concept_2025.gpkg", + **{str(y): f"{base}/brpgewaspercelen_definitief_{y}.gpkg" for y in range(2024, 2020, -1)}, + **{str(y): f"{base}/brpgewaspercelen_definitief_{y}.zip" for y in range(2020, 2009, -1)}, + } id = "nl" - short_name = "Netherlands" - title = "Field boundaries for The Netherlands" + short_name = "Netherlands (Crops)" + title = "BRP Crop Field Boundaries for The Netherlands (CAP-based)" description = """ -A field block (Dutch: "Referentieperceel"), formerly known as "AAN" (Agrarisch Areaal Nederland), -is a contiguous agricultural area surrounded by permanent boundaries, which is cultivated by one or -more farmers with one or more crops, is fully or partially set aside or is fully or partially -taken out of production. +BasisRegistratie Percelen (BRP) combines the location of +agricultural plots with the crop grown. The data set +is published by RVO (Netherlands Enterprise Agency). The boundaries of the agricultural plots +are based within the reference parcels (formerly known as AAN). A user an agricultural plot +annually has to register his crop fields with crops (for the Common Agricultural Policy scheme). +A dataset is generated for each year with reference date May 15. +A view service and a download service are available for the most recent BRP crop plots. -The following field block types exist: + -- Woods (Hout) -- Agricultural area (Landbouwgrond) -- Other (Overig) -- Water (Water) - -We filter on "Agricultural area" in this converter. -For crop data, look at BasisRegistratie gewasPercelen (BRP) +Data is currently available for the years 2009 to 2024. """ - provider = "RVO / PDOK " + provider = ( + "RVO / PDOK " + ) # Both http://creativecommons.org/publicdomain/zero/1.0/deed.nl and http://creativecommons.org/publicdomain/mark/1.0/ license = "CC0-1.0" - column_additions = {"determination:datetime": "2023-06-15T00:00:00Z"} - columns = {"geometry": "geometry", "id": "id", "area": "metrics:area", "versiebron": "source"} + + columns = { + "geometry": "geometry", + "id": "id", + "area": "metrics:area", + "category": "coverage", + "gewascode": "crop:code", + "gewas": "crop:name", + "jaar": "determination:datetime", + } + column_filters = { - # type = "Hout" | "Landbouwgrond" | "Overig" | "Water" - "type": lambda col: col == "Landbouwgrond" + # category = "Grasland" | "Bouwland" | "Sloot" | "Landschapselement" + "category": lambda col: col.isin(["Grasland", "Bouwland"]) + } + + column_migrations = { + # Add 15th of may to original "year" (jaar) column + "jaar": lambda col: pd.to_datetime(col, format="%Y") + pd.DateOffset(months=4, days=14) } + extensions = {"https://fiboa.org/crop-extension/v0.2.0/schema.yaml"} + ec_mapping_csv = "https://fiboa.org/code/nl/nl.csv" index_as_id = True - area_calculate_missing = True + missing_schemas = { "properties": { - "source": {"type": "string"}, + "coverage": {"type": "string", "enum": ["Grasland", "Bouwland"]}, } } diff --git a/fiboa_cli/datasets/nl_block.py b/fiboa_cli/datasets/nl_block.py new file mode 100644 index 00000000..a0f25727 --- /dev/null +++ b/fiboa_cli/datasets/nl_block.py @@ -0,0 +1,46 @@ +from vecorel_cli.conversion.admin import AdminConverterMixin + +from ..conversion.fiboa_converter import FiboaBaseConverter + + +class Converter(AdminConverterMixin, FiboaBaseConverter): + sources = ( + "https://service.pdok.nl/rvo/referentiepercelen/atom/downloads/referentiepercelen.gpkg" + ) + + id = "nl_block" + short_name = "Netherlands" + title = "Field blocks for The Netherlands" + description = """ +A field block (Dutch: "Referentieperceel"), formerly known as "AAN" (Agrarisch Areaal Nederland), +is a contiguous agricultural area surrounded by permanent boundaries, which is cultivated by one or +more farmers with one or more crops, is fully or partially set aside or is fully or partially +taken out of production. + +The following field block types exist: + +- Woods (Hout) +- Agricultural area (Landbouwgrond) +- Other (Overig) +- Water (Water) + +We filter on "Agricultural area" in this converter. +For crop data, look at BasisRegistratie gewasPercelen (BRP) + """ + + provider = "RVO / PDOK " + # Both http://creativecommons.org/publicdomain/zero/1.0/deed.nl and http://creativecommons.org/publicdomain/mark/1.0/ + license = "CC0-1.0" + column_additions = {"determination:datetime": "2023-06-15T00:00:00Z"} + columns = {"geometry": "geometry", "id": "id", "area": "metrics:area", "versiebron": "source"} + column_filters = { + # type = "Hout" | "Landbouwgrond" | "Overig" | "Water" + "type": lambda col: col == "Landbouwgrond" + } + index_as_id = True + area_calculate_missing = True + missing_schemas = { + "properties": { + "source": {"type": "string"}, + } + } diff --git a/fiboa_cli/datasets/nl_crop.py b/fiboa_cli/datasets/nl_crop.py deleted file mode 100644 index a13f8ae3..00000000 --- a/fiboa_cli/datasets/nl_crop.py +++ /dev/null @@ -1,69 +0,0 @@ -import pandas as pd -from vecorel_cli.conversion.admin import AdminConverterMixin - -from ..conversion.fiboa_converter import FiboaBaseConverter -from .commons.ec import ec_url - -# see https://service.pdok.nl/rvo/brpgewaspercelen/atom/v1_0/basisregistratie_gewaspercelen_brp.xml -base = "https://service.pdok.nl/rvo/brpgewaspercelen/atom/v1_0/downloads" - - -class NLCropConverter(AdminConverterMixin, FiboaBaseConverter): - area_calculate_missing = True - variants = { - "2025": f"{base}/brpgewaspercelen_concept_2025.gpkg", - **{str(y): f"{base}/brpgewaspercelen_definitief_{y}.gpkg" for y in range(2024, 2020, -1)}, - **{str(y): f"{base}/brpgewaspercelen_definitief_{y}.zip" for y in range(2020, 2009, -1)}, - } - - id = "nl_crop" - short_name = "Netherlands (Crops)" - title = "BRP Crop Field Boundaries for The Netherlands (CAP-based)" - description = """ -BasisRegistratie Percelen (BRP) combines the location of -agricultural plots with the crop grown. The data set -is published by RVO (Netherlands Enterprise Agency). The boundaries of the agricultural plots -are based within the reference parcels (formerly known as AAN). A user an agricultural plot -annually has to register his crop fields with crops (for the Common Agricultural Policy scheme). -A dataset is generated for each year with reference date May 15. -A view service and a download service are available for the most recent BRP crop plots. - - - -Data is currently available for the years 2009 to 2024. - """ - - provider = ( - "RVO / PDOK " - ) - # Both http://creativecommons.org/publicdomain/zero/1.0/deed.nl and http://creativecommons.org/publicdomain/mark/1.0/ - license = "CC0-1.0" - - columns = { - "geometry": "geometry", - "id": "id", - "area": "metrics:area", - "category": "coverage", - "gewascode": "crop:code", - "gewas": "crop:name", - "jaar": "determination:datetime", - } - - column_filters = { - # category = "Grasland" | "Bouwland" | "Sloot" | "Landschapselement" - "category": lambda col: col.isin(["Grasland", "Bouwland"]) - } - - column_migrations = { - # Add 15th of may to original "year" (jaar) column - "jaar": lambda col: pd.to_datetime(col, format="%Y") + pd.DateOffset(months=4, days=14) - } - extensions = {"https://fiboa.org/crop-extension/v0.2.0/schema.yaml"} - column_additions = {"crop:code_list": ec_url("nl_2020.csv")} - index_as_id = True - - missing_schemas = { - "properties": { - "coverage": {"type": "string", "enum": ["Grasland", "Bouwland"]}, - } - } diff --git a/fiboa_cli/datasets/nz.py b/fiboa_cli/datasets/nz.py index 8d394973..524085a0 100644 --- a/fiboa_cli/datasets/nz.py +++ b/fiboa_cli/datasets/nz.py @@ -49,4 +49,4 @@ def migrate(self, gdf): rows = read_data_csv("nz_region_codes.csv") mapping = {row["Subdivision name"]: row["3166-2 code"][len("NZ-") :] for row in rows} gdf["Region"] = gdf["Region"].map(mapping) - return gdf + return super().migrate(gdf) diff --git a/fiboa_cli/datasets/pt.py b/fiboa_cli/datasets/pt.py index fdceebb5..25b0769e 100644 --- a/fiboa_cli/datasets/pt.py +++ b/fiboa_cli/datasets/pt.py @@ -1,13 +1,28 @@ +from vecorel_cli.conversion.admin import AdminConverterMixin + from ..conversion.fiboa_converter import FiboaBaseConverter -from .commons.ec import ec_url +from .commons.hcat import AddHCATMixin -class PTConverter(FiboaBaseConverter): +class PTConverter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter): id = "pt" title = "Field boundaries for Portugal" short_name = "Portugal" description = "Open field boundaries (identificação de parcelas) from Portugal" - sources = "https://www.ifap.pt/isip/ows/resources/2023/Continente.gpkg" + # see https://www.ifap.pt/isip/ows/ + BASE = "https://www.ifap.pt/isip/ows/resources/" + variants = { + "2023": BASE + "2023/Continente.gpkg", + "2022": BASE + "2022/2022.zip", + "2021": BASE + "2021/2021.zip", + "2020": BASE + "2017-2020/2020.zip", + "2019": BASE + "2017-2020/2019.zip", + "2018": BASE + "2017-2020/2018.zip", + "2017": BASE + "2017-2020/2017.zip", + "2016": BASE + "2011_2016/2016.zip", + "2015": BASE + "2011_2016/2015.zip", + # ... + } def layer_filter(self, layer, uri): return layer.startswith("Culturas_") @@ -26,8 +41,8 @@ def layer_filter(self, layer, uri): "Shape_Length": "metrics:perimeter", } extensions = {"https://fiboa.org/crop-extension/v0.2.0/schema.yaml"} + ec_mapping_csv = "https://fiboa.org/code/pt/pt.csv" column_additions = { - "crop:code_list": ec_url("pt_2021.csv"), "determination:datetime": "2023-01-01T00:00:00Z", } area_is_in_ha = False diff --git a/fiboa_cli/datasets/se.py b/fiboa_cli/datasets/se.py index ebb534ee..2853b11b 100644 --- a/fiboa_cli/datasets/se.py +++ b/fiboa_cli/datasets/se.py @@ -2,12 +2,17 @@ from vecorel_cli.conversion.admin import AdminConverterMixin from ..conversion.fiboa_converter import FiboaBaseConverter -from .commons.ec import ec_url, load_ec_mapping +from .commons.hcat import AddHCATMixin -class Converter(AdminConverterMixin, FiboaBaseConverter): - sources = { - "http://epub.sjv.se/inspire/inspire/wfs?SERVICE=WFS%20&REQUEST=GetFeature%20&VERSION=1.0.0%20&TYPENAMES=inspire:arslager_skifte%20&outputFormat=shape-zip%20&CQL_FILTER=arslager=%272023%27%20%20and%20geom%20is%20not%20null%20&format_options=CHARSET:UTF-8": "se2023.zip" +class Converter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter): + variants = { + "2024": { + "http://epub.sjv.se/inspire/inspire/wfs?SERVICE=WFS%20&REQUEST=GetFeature%20&VERSION=1.0.0%20&TYPENAMES=inspire:arslager_skifte%20&outputFormat=shape-zip%20&CQL_FILTER=arslager=%272024%27%20%20and%20geom%20is%20not%20null%20&format_options=CHARSET:UTF-8": "se2024.zip" + }, + "2023": { + "http://epub.sjv.se/inspire/inspire/wfs?SERVICE=WFS%20&REQUEST=GetFeature%20&VERSION=1.0.0%20&TYPENAMES=inspire:arslager_skifte%20&outputFormat=shape-zip%20&CQL_FILTER=arslager=%272023%27%20%20and%20geom%20is%20not%20null%20&format_options=CHARSET:UTF-8": "se2023.zip" + }, } id = "se" short_name = "Sweden" @@ -17,6 +22,8 @@ class Converter(AdminConverterMixin, FiboaBaseConverter): To receive compensation for agricultural support (EU support), farmers apply for support from the Swedish Agency for Agriculture via a SAM application. The data set contains parcels where the area applied for and the area decided on are the same. The data is published at the end of a year. + + Codes found at https://jordbruksverket.se/stod/jordbruk-tradgard-och-rennaring/sam-ansokan-och-allmant-om-jordbrukarstoden/grodkoder """ provider = "Jordbruksverket (The Swedish Board of Agriculture) " attribution = "Jordbruksverket" @@ -26,23 +33,15 @@ class Converter(AdminConverterMixin, FiboaBaseConverter): "id": "id", "faststalld": "metrics:area", "grdkod_mar": "crop:code", - "crop:name": "crop:name", "arslager": "determination:datetime", } extensions = {"https://fiboa.org/crop-extension/v0.2.0/schema.yaml"} - column_additions = {"crop:code_list": ec_url("se_2021.csv")} + ec_mapping_csv = "https://fiboa.org/code/se/se.csv" column_migrations = { # Make year (1st January) from column "arslager" "arslager": lambda col: pd.to_datetime(col, format="%Y") } def migrate(self, gdf): - """ - Perform migration of the GeoDataFrame (migrate step). - """ - ec_mapping = load_ec_mapping("se_2021.csv") - original_name_mapping = {int(e["original_code"]): e["original_name"] for e in ec_mapping} - gdf["id"] = gdf["blockid"] + "_" + gdf["skiftesbet"] - gdf["crop:name"] = gdf["grdkod_mar"].map(original_name_mapping) - return gdf + return super().migrate(gdf) diff --git a/fiboa_cli/datasets/si.py b/fiboa_cli/datasets/si.py index a1be013e..ea1784c1 100644 --- a/fiboa_cli/datasets/si.py +++ b/fiboa_cli/datasets/si.py @@ -1,11 +1,16 @@ from vecorel_cli.conversion.admin import AdminConverterMixin from ..conversion.fiboa_converter import FiboaBaseConverter -from .commons.ec import ec_url +from .commons.hcat import AddHCATMixin -class Converter(AdminConverterMixin, FiboaBaseConverter): - sources = {"https://rkg.gov.si/razno/portal_analysis/KMRS_2023.rar": ["KMRS_2023.shp"]} +class Converter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter): + variants = { + str(year): { + f"https://rkg.gov.si/razno/portal_analysis/KMRS_{year}.rar": [f"KMRS_{year}.shp"] + } + for year in range(2024, 2020, -1) + } id = "si" short_name = "Slovenia" title = "Slovenia Crop Fields" @@ -27,8 +32,7 @@ class Converter(AdminConverterMixin, FiboaBaseConverter): "RASTLINA": "crop:name", "CROP_LAT_E": "crop:name_en", } - extensions = {"https://fiboa.org/crop-extension/v0.2.0/schema.yaml"} - column_additions = {"crop:code_list": ec_url("si_2021.csv")} + ec_mapping_csv = "https://fiboa.org/code/si/si.csv" column_migrations = {"geometry": lambda col: col.make_valid()} area_is_in_ha = False missing_schemas = { diff --git a/fiboa_cli/datasets/sk.py b/fiboa_cli/datasets/sk.py index 4e638b7d..71b78891 100644 --- a/fiboa_cli/datasets/sk.py +++ b/fiboa_cli/datasets/sk.py @@ -1,14 +1,16 @@ from vecorel_cli.conversion.admin import AdminConverterMixin from ..conversion.fiboa_converter import FiboaBaseConverter +from .commons.hcat import AddHCATMixin, load_ec_mapping -class Converter(AdminConverterMixin, FiboaBaseConverter): +class Converter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter): sources = { "https://data.slovensko.sk/download?id=e39ad227-1899-4cff-b7c8-734f90aa0b59&blocksize=0": [ "HU2024_20240917shp/HU2024_20240917.shp" ] } + # https://data.slovensko.sk/download?id=626c1181-bc53-40b6-9715-3c10164760ec id = "sk" short_name = "Slovakia" title = "Slovakia Agricultural Land Identification System" @@ -24,19 +26,25 @@ class Converter(AdminConverterMixin, FiboaBaseConverter): """ provider = "Pôdohospodárska platobná agentúra " license = "CC0-1.0" # "Open Data" - # TODO look for a way to find codes for crop_name and implement crop-extension + ec_mapping_csv = "https://fiboa.org/code/sk/sk.csv" columns = { "geometry": "geometry", "KODKD": "id", - "PLODINA": "crop_name", + "PLODINA": "crop:name", "KULTURA_NA": "crop_group", "LOKALITA_N": "municipality", "VYMERA": "metrics:area", } missing_schemas = { "properties": { - "crop_name": {"type": "string"}, "crop_group": {"type": "string"}, "municipality": {"type": "string"}, } } + + def migrate(self, gdf): + if self.ec_mapping is None: + self.ec_mapping = load_ec_mapping(self.ec_mapping_csv, url=self.mapping_file) + mapping = {row["original_name"]: index + 1 for index, row in enumerate(self.ec_mapping)} + gdf["crop:code"] = gdf["PLODINA"].map(mapping) + return super().migrate(gdf) diff --git a/fiboa_cli/datasets/us_ca_scm.py b/fiboa_cli/datasets/us_ca_scm.py index 61f58b25..9a191ada 100644 --- a/fiboa_cli/datasets/us_ca_scm.py +++ b/fiboa_cli/datasets/us_ca_scm.py @@ -4,13 +4,21 @@ from ..conversion.fiboa_converter import FiboaBaseConverter from .commons.ec import load_ec_mapping +from .commons.hcat import AddHCATMixin -class Converter(AdminConverterMixin, FiboaBaseConverter): - sources = { - "https://data.cnra.ca.gov/dataset/6c3d65e3-35bb-49e1-a51e-49d5a2cf09a9/resource/f38d3f6f-dcf1-4553-9f07-4f381d494320/download/i15_crop_mapping_2022_provisional_gdb.zip": [ - "i15_Crop_Mapping_2022_Provisional_GDB/i15_Crop_Mapping_2022_Provisional.gdb" - ] +class Converter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter): + variants = { + "2023": { + "https://data.cnra.ca.gov/dataset/6c3d65e3-35bb-49e1-a51e-49d5a2cf09a9/resource/4e17ca38-268e-4bf5-bbc5-09636d44ed60/download/i15_crop_mapping_2023_provisional_20241127.gdb.zip": [ + "i15_Crop_Mapping_2023_Provisional_20241127.gdb" + ] + }, + "2022": { + "https://data.cnra.ca.gov/dataset/6c3d65e3-35bb-49e1-a51e-49d5a2cf09a9/resource/f38d3f6f-dcf1-4553-9f07-4f381d494320/download/i15_crop_mapping_2022_provisional_gdb.zip": [ + "i15_Crop_Mapping_2022_Provisional_GDB/i15_Crop_Mapping_2022_Provisional.gdb" + ] + }, } id = "us_ca_scm" admin_subdivision_code = "CA" @@ -24,7 +32,6 @@ class Converter(AdminConverterMixin, FiboaBaseConverter): seek to address resource management issues, regulatory compliance issues, environmental impacts, ecosystem services, urban and economic development, and other issues. """ - extensions = {"https://fiboa.org/crop-extension/v0.2.0/schema.yaml"} provider = "County of Santa Clara " license = "CC0-1.0" columns = { @@ -36,8 +43,8 @@ class Converter(AdminConverterMixin, FiboaBaseConverter): } column_additions = { "determination:datetime": "2023-05-01T00:00:00Z", - "crop:code_list": "https://fiboa.org/code/us/ca/scm.csv", } + ec_mapping_csv = "https://fiboa.org/code/us/ca/scm.csv" missing_schemas = { "properties": { "admin_level_2": {"type": "string"}, diff --git a/fiboa_cli/datasets/us_usda_cropland.py b/fiboa_cli/datasets/us_usda_cropland.py index e5aef1af..35850d48 100644 --- a/fiboa_cli/datasets/us_usda_cropland.py +++ b/fiboa_cli/datasets/us_usda_cropland.py @@ -1,18 +1,24 @@ -from os.path import dirname, join - import pandas as pd from loguru import logger from vecorel_cli.conversion.admin import AdminConverterMixin from ..conversion.fiboa_converter import FiboaBaseConverter from .commons.ec import load_ec_mapping +from .commons.hcat import AddHCATMixin -class Converter(AdminConverterMixin, FiboaBaseConverter): - sources = { - "https://www.nass.usda.gov/Research_and_Science/Crop-Sequence-Boundaries/datasets/NationalCSB_2016-2023_rev23.zip": [ - "NationalCSB_2016-2023_rev23/CSB1623.gdb" - ] +class Converter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter): + variants = { + "2024": { + "https://www.nass.usda.gov/Research_and_Science/Crop-Sequence-Boundaries/datasets/NationalCSB_2017-2024_rev23.zip": [ + "NationalCSB_2017-2024_rev23/CSB1724.gdb" + ] + }, + "2023": { + "https://www.nass.usda.gov/Research_and_Science/Crop-Sequence-Boundaries/datasets/NationalCSB_2016-2023_rev23.zip": [ + "NationalCSB_2016-2023_rev23/CSB1623.gdb" + ] + }, } id = "us_usda_cropland" short_name = "US (USDA CSB)" @@ -30,19 +36,19 @@ class Converter(AdminConverterMixin, FiboaBaseConverter): columns = { "geometry": "geometry", "CSBID": "id", - "CDL2023": "crop:code", + # "CDL2023": "crop:code", "crop:name": "crop:name", "CNTY": "administrative_area_level_2", } column_additions = { "determination:datetime": "2023-05-01T00:00:00Z", - "crop:code_list": "https://fiboa.org/code/us/usda/crop.csv", } missing_schemas = { "properties": { "administrative_area_level_2": {"type": "string"}, } } + ec_mapping_csv = "https://fiboa.org/code/us/usda/cropland.csv" def migrate(self, gdf): """ @@ -53,18 +59,24 @@ def migrate(self, gdf): geodataframe.Dissolve(method="unary") is **slow** for large datasets So we're handling this huge dataset in blocks, states are a natural grouping-method """ + assert self.variant, "Variant must be set" + crop_key = f"CDL{self.variant}" + self.columns[crop_key] = "crop:code" + gdf = super().migrate(gdf) states = list(gdf["STATEFIPS"].unique()) gdfs = [] for state in states: logger.info(f"Handling State {state}") df = gdf[gdf["STATEFIPS"] == state].explode() - # TODO is this correct, don't we need `df = df.dissolve?` - df.dissolve(by=["CDL2023"], aggfunc="first").explode() + df = df.dissolve(by=[crop_key], aggfunc="first", as_index=False).explode() gdfs.append(df) gdf = pd.concat(gdfs) del gdfs - mapping = load_ec_mapping(url=join(dirname(__file__), "data-files", "us_usda_cropland.csv")) - original_name_mapping = {int(e["original_code"]): e["original_name"] for e in mapping} - gdf["crop:name"] = gdf["CDL2023"].map(original_name_mapping) + if self.ec_mapping is None: + self.ec_mapping = load_ec_mapping(self.ec_mapping_csv, url=self.mapping_file) + original_name_mapping = { + int(e["original_code"]): e["original_name"] for e in self.ec_mapping + } + gdf["crop:name"] = gdf[crop_key].map(original_name_mapping) return gdf diff --git a/pixi.lock b/pixi.lock index f7ff81e9..ce310736 100644 --- a/pixi.lock +++ b/pixi.lock @@ -177,7 +177,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/da/af/5d24b8d49ef358468ecfdff5c556adf37f4fd28e336b96f923661a808329/types_python_dateutil-2.9.0.20251008-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/04/ed/c3c3cee45ca60be5f4766bc8dcf6ff904b70b5ca80c603d14297a830d1f0/vecorel_cli-0.2.11-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/95/f1/7b40045bfb8de925f0f510b36e269b474766711a5d4b78b47d97c9695e8d/vecorel_cli-0.2.12-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/60/e8/c0e05e4684d13459f93d312077a9a2efbe04d59c393bc2b8802248c908d4/webcolors-24.11.1-py3-none-any.whl - pypi: ./ osx-64: @@ -342,7 +342,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/da/af/5d24b8d49ef358468ecfdff5c556adf37f4fd28e336b96f923661a808329/types_python_dateutil-2.9.0.20251008-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/04/ed/c3c3cee45ca60be5f4766bc8dcf6ff904b70b5ca80c603d14297a830d1f0/vecorel_cli-0.2.11-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/95/f1/7b40045bfb8de925f0f510b36e269b474766711a5d4b78b47d97c9695e8d/vecorel_cli-0.2.12-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/60/e8/c0e05e4684d13459f93d312077a9a2efbe04d59c393bc2b8802248c908d4/webcolors-24.11.1-py3-none-any.whl - pypi: ./ osx-arm64: @@ -508,7 +508,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/da/af/5d24b8d49ef358468ecfdff5c556adf37f4fd28e336b96f923661a808329/types_python_dateutil-2.9.0.20251008-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/04/ed/c3c3cee45ca60be5f4766bc8dcf6ff904b70b5ca80c603d14297a830d1f0/vecorel_cli-0.2.11-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/95/f1/7b40045bfb8de925f0f510b36e269b474766711a5d4b78b47d97c9695e8d/vecorel_cli-0.2.12-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/60/e8/c0e05e4684d13459f93d312077a9a2efbe04d59c393bc2b8802248c908d4/webcolors-24.11.1-py3-none-any.whl - pypi: ./ win-64: @@ -672,7 +672,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/da/af/5d24b8d49ef358468ecfdff5c556adf37f4fd28e336b96f923661a808329/types_python_dateutil-2.9.0.20251008-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/04/ed/c3c3cee45ca60be5f4766bc8dcf6ff904b70b5ca80c603d14297a830d1f0/vecorel_cli-0.2.11-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/95/f1/7b40045bfb8de925f0f510b36e269b474766711a5d4b78b47d97c9695e8d/vecorel_cli-0.2.12-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/60/e8/c0e05e4684d13459f93d312077a9a2efbe04d59c393bc2b8802248c908d4/webcolors-24.11.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e1/07/c6fe3ad3e685340704d314d765b7912993bcb8dc198f0e7a89382d37974b/win32_setctime-1.2.0-py3-none-any.whl - pypi: ./ @@ -828,7 +828,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/da/af/5d24b8d49ef358468ecfdff5c556adf37f4fd28e336b96f923661a808329/types_python_dateutil-2.9.0.20251008-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/04/ed/c3c3cee45ca60be5f4766bc8dcf6ff904b70b5ca80c603d14297a830d1f0/vecorel_cli-0.2.11-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/95/f1/7b40045bfb8de925f0f510b36e269b474766711a5d4b78b47d97c9695e8d/vecorel_cli-0.2.12-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/60/e8/c0e05e4684d13459f93d312077a9a2efbe04d59c393bc2b8802248c908d4/webcolors-24.11.1-py3-none-any.whl - pypi: ./ osx-64: @@ -968,7 +968,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/da/af/5d24b8d49ef358468ecfdff5c556adf37f4fd28e336b96f923661a808329/types_python_dateutil-2.9.0.20251008-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/04/ed/c3c3cee45ca60be5f4766bc8dcf6ff904b70b5ca80c603d14297a830d1f0/vecorel_cli-0.2.11-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/95/f1/7b40045bfb8de925f0f510b36e269b474766711a5d4b78b47d97c9695e8d/vecorel_cli-0.2.12-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/60/e8/c0e05e4684d13459f93d312077a9a2efbe04d59c393bc2b8802248c908d4/webcolors-24.11.1-py3-none-any.whl - pypi: ./ osx-arm64: @@ -1109,7 +1109,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/da/af/5d24b8d49ef358468ecfdff5c556adf37f4fd28e336b96f923661a808329/types_python_dateutil-2.9.0.20251008-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/04/ed/c3c3cee45ca60be5f4766bc8dcf6ff904b70b5ca80c603d14297a830d1f0/vecorel_cli-0.2.11-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/95/f1/7b40045bfb8de925f0f510b36e269b474766711a5d4b78b47d97c9695e8d/vecorel_cli-0.2.12-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/60/e8/c0e05e4684d13459f93d312077a9a2efbe04d59c393bc2b8802248c908d4/webcolors-24.11.1-py3-none-any.whl - pypi: ./ win-64: @@ -1249,7 +1249,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/da/af/5d24b8d49ef358468ecfdff5c556adf37f4fd28e336b96f923661a808329/types_python_dateutil-2.9.0.20251008-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/04/ed/c3c3cee45ca60be5f4766bc8dcf6ff904b70b5ca80c603d14297a830d1f0/vecorel_cli-0.2.11-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/95/f1/7b40045bfb8de925f0f510b36e269b474766711a5d4b78b47d97c9695e8d/vecorel_cli-0.2.12-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/60/e8/c0e05e4684d13459f93d312077a9a2efbe04d59c393bc2b8802248c908d4/webcolors-24.11.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e1/07/c6fe3ad3e685340704d314d765b7912993bcb8dc198f0e7a89382d37974b/win32_setctime-1.2.0-py3-none-any.whl - pypi: ./ @@ -1368,7 +1368,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/da/af/5d24b8d49ef358468ecfdff5c556adf37f4fd28e336b96f923661a808329/types_python_dateutil-2.9.0.20251008-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/04/ed/c3c3cee45ca60be5f4766bc8dcf6ff904b70b5ca80c603d14297a830d1f0/vecorel_cli-0.2.11-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/95/f1/7b40045bfb8de925f0f510b36e269b474766711a5d4b78b47d97c9695e8d/vecorel_cli-0.2.12-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/60/e8/c0e05e4684d13459f93d312077a9a2efbe04d59c393bc2b8802248c908d4/webcolors-24.11.1-py3-none-any.whl - pypi: ./ osx-64: @@ -1471,7 +1471,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/da/af/5d24b8d49ef358468ecfdff5c556adf37f4fd28e336b96f923661a808329/types_python_dateutil-2.9.0.20251008-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/04/ed/c3c3cee45ca60be5f4766bc8dcf6ff904b70b5ca80c603d14297a830d1f0/vecorel_cli-0.2.11-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/95/f1/7b40045bfb8de925f0f510b36e269b474766711a5d4b78b47d97c9695e8d/vecorel_cli-0.2.12-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/60/e8/c0e05e4684d13459f93d312077a9a2efbe04d59c393bc2b8802248c908d4/webcolors-24.11.1-py3-none-any.whl - pypi: ./ osx-arm64: @@ -1575,7 +1575,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/da/af/5d24b8d49ef358468ecfdff5c556adf37f4fd28e336b96f923661a808329/types_python_dateutil-2.9.0.20251008-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/04/ed/c3c3cee45ca60be5f4766bc8dcf6ff904b70b5ca80c603d14297a830d1f0/vecorel_cli-0.2.11-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/95/f1/7b40045bfb8de925f0f510b36e269b474766711a5d4b78b47d97c9695e8d/vecorel_cli-0.2.12-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/60/e8/c0e05e4684d13459f93d312077a9a2efbe04d59c393bc2b8802248c908d4/webcolors-24.11.1-py3-none-any.whl - pypi: ./ win-64: @@ -1676,7 +1676,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/da/af/5d24b8d49ef358468ecfdff5c556adf37f4fd28e336b96f923661a808329/types_python_dateutil-2.9.0.20251008-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/04/ed/c3c3cee45ca60be5f4766bc8dcf6ff904b70b5ca80c603d14297a830d1f0/vecorel_cli-0.2.11-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/95/f1/7b40045bfb8de925f0f510b36e269b474766711a5d4b78b47d97c9695e8d/vecorel_cli-0.2.12-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/60/e8/c0e05e4684d13459f93d312077a9a2efbe04d59c393bc2b8802248c908d4/webcolors-24.11.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e1/07/c6fe3ad3e685340704d314d765b7912993bcb8dc198f0e7a89382d37974b/win32_setctime-1.2.0-py3-none-any.whl - pypi: ./ @@ -1820,7 +1820,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/da/af/5d24b8d49ef358468ecfdff5c556adf37f4fd28e336b96f923661a808329/types_python_dateutil-2.9.0.20251008-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/04/ed/c3c3cee45ca60be5f4766bc8dcf6ff904b70b5ca80c603d14297a830d1f0/vecorel_cli-0.2.11-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/95/f1/7b40045bfb8de925f0f510b36e269b474766711a5d4b78b47d97c9695e8d/vecorel_cli-0.2.12-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/60/e8/c0e05e4684d13459f93d312077a9a2efbe04d59c393bc2b8802248c908d4/webcolors-24.11.1-py3-none-any.whl - pypi: ./ osx-64: @@ -1948,7 +1948,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/da/af/5d24b8d49ef358468ecfdff5c556adf37f4fd28e336b96f923661a808329/types_python_dateutil-2.9.0.20251008-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/04/ed/c3c3cee45ca60be5f4766bc8dcf6ff904b70b5ca80c603d14297a830d1f0/vecorel_cli-0.2.11-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/95/f1/7b40045bfb8de925f0f510b36e269b474766711a5d4b78b47d97c9695e8d/vecorel_cli-0.2.12-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/60/e8/c0e05e4684d13459f93d312077a9a2efbe04d59c393bc2b8802248c908d4/webcolors-24.11.1-py3-none-any.whl - pypi: ./ osx-arm64: @@ -2077,7 +2077,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/da/af/5d24b8d49ef358468ecfdff5c556adf37f4fd28e336b96f923661a808329/types_python_dateutil-2.9.0.20251008-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/04/ed/c3c3cee45ca60be5f4766bc8dcf6ff904b70b5ca80c603d14297a830d1f0/vecorel_cli-0.2.11-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/95/f1/7b40045bfb8de925f0f510b36e269b474766711a5d4b78b47d97c9695e8d/vecorel_cli-0.2.12-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/60/e8/c0e05e4684d13459f93d312077a9a2efbe04d59c393bc2b8802248c908d4/webcolors-24.11.1-py3-none-any.whl - pypi: ./ win-64: @@ -2202,7 +2202,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/da/af/5d24b8d49ef358468ecfdff5c556adf37f4fd28e336b96f923661a808329/types_python_dateutil-2.9.0.20251008-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/04/ed/c3c3cee45ca60be5f4766bc8dcf6ff904b70b5ca80c603d14297a830d1f0/vecorel_cli-0.2.11-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/95/f1/7b40045bfb8de925f0f510b36e269b474766711a5d4b78b47d97c9695e8d/vecorel_cli-0.2.12-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/60/e8/c0e05e4684d13459f93d312077a9a2efbe04d59c393bc2b8802248c908d4/webcolors-24.11.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e1/07/c6fe3ad3e685340704d314d765b7912993bcb8dc198f0e7a89382d37974b/win32_setctime-1.2.0-py3-none-any.whl - pypi: ./ @@ -2917,9 +2917,9 @@ packages: - pypi: ./ name: fiboa-cli version: 0.20.3 - sha256: 0b3743a9a5f590df1aa36c60901b73456e1f2f7dbc25163ba4597134928c1132 + sha256: 37684f32135ca252711dae982a7f5f80dbc23a622225cf0c6118a9632584059f requires_dist: - - vecorel-cli==0.2.11 + - vecorel-cli==0.2.12 - spdx-license-list==3.27.0 requires_python: '>=3.10,<3.14' editable: true @@ -7322,10 +7322,10 @@ packages: purls: [] size: 113963 timestamp: 1753739198723 -- pypi: https://files.pythonhosted.org/packages/04/ed/c3c3cee45ca60be5f4766bc8dcf6ff904b70b5ca80c603d14297a830d1f0/vecorel_cli-0.2.11-py3-none-any.whl +- pypi: https://files.pythonhosted.org/packages/95/f1/7b40045bfb8de925f0f510b36e269b474766711a5d4b78b47d97c9695e8d/vecorel_cli-0.2.12-py3-none-any.whl name: vecorel-cli - version: 0.2.11 - sha256: 405e04bac3f4818787d3a17e57d13985dcfc9a8f3704454cfa45ca7702243951 + version: 0.2.12 + sha256: c984d5522e84f5b374d9778a617006afa06e549fdde423a2f8170cbc399731a1 requires_dist: - pyyaml>=6.0,<7.0 - click>=8.1,<9.0 diff --git a/pyproject.toml b/pyproject.toml index cb36db97..12e7ccc2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,7 @@ classifiers = [ ] requires-python = ">=3.10,<3.14" dependencies = [ - "vecorel-cli==0.2.11", + "vecorel-cli==0.2.12", "spdx-license-list==3.27.0", ] diff --git a/tests/data-files/convert/at_crop/at_2021.csv b/tests/data-files/convert/at/at_2021.csv similarity index 100% rename from tests/data-files/convert/at_crop/at_2021.csv rename to tests/data-files/convert/at/at_2021.csv diff --git a/tests/data-files/convert/at/inspire_schlaege_2025-1_polygon.gpkg.zip b/tests/data-files/convert/at/inspire_schlaege_2025-1_polygon.gpkg.zip new file mode 100644 index 00000000..88c1333e Binary files /dev/null and b/tests/data-files/convert/at/inspire_schlaege_2025-1_polygon.gpkg.zip differ diff --git a/tests/data-files/convert/at/inspire_referenzen_2021_polygon.gpkg.zip b/tests/data-files/convert/at_block/inspire_referenzen_2021_polygon.gpkg.zip similarity index 100% rename from tests/data-files/convert/at/inspire_referenzen_2021_polygon.gpkg.zip rename to tests/data-files/convert/at_block/inspire_referenzen_2021_polygon.gpkg.zip diff --git a/tests/data-files/convert/at_crop/inspire_schlaege_2024-2_polygon.gpkg.zip b/tests/data-files/convert/at_crop/inspire_schlaege_2024-2_polygon.gpkg.zip deleted file mode 100644 index e5a20357..00000000 Binary files a/tests/data-files/convert/at_crop/inspire_schlaege_2024-2_polygon.gpkg.zip and /dev/null differ diff --git a/tests/data-files/convert/de_bb/antrag.zip b/tests/data-files/convert/de_bb/antrag.zip new file mode 100644 index 00000000..0ddd1f97 Binary files /dev/null and b/tests/data-files/convert/de_bb/antrag.zip differ diff --git a/tests/data-files/convert/ie/GSAA_2024.zip b/tests/data-files/convert/ie/GSAA_2024.zip new file mode 100644 index 00000000..2141fdba Binary files /dev/null and b/tests/data-files/convert/ie/GSAA_2024.zip differ diff --git a/tests/data-files/convert/ie/IACS_GSAA_2022.zip b/tests/data-files/convert/ie/IACS_GSAA_2022.zip deleted file mode 100644 index cd0578aa..00000000 Binary files a/tests/data-files/convert/ie/IACS_GSAA_2022.zip and /dev/null differ diff --git a/tests/data-files/convert/nl_crop/brpgewaspercelen_definitief_2023.gpkg b/tests/data-files/convert/nl/brpgewaspercelen_definitief_2023.gpkg similarity index 100% rename from tests/data-files/convert/nl_crop/brpgewaspercelen_definitief_2023.gpkg rename to tests/data-files/convert/nl/brpgewaspercelen_definitief_2023.gpkg diff --git a/tests/data-files/convert/nl/referentiepercelen.gpkg b/tests/data-files/convert/nl_block/referentiepercelen.gpkg similarity index 100% rename from tests/data-files/convert/nl/referentiepercelen.gpkg rename to tests/data-files/convert/nl_block/referentiepercelen.gpkg diff --git a/tests/data-files/convert/us_ca_scm/i15_crop_mapping_2023_provisional_20241127.gdb.zip b/tests/data-files/convert/us_ca_scm/i15_crop_mapping_2023_provisional_20241127.gdb.zip new file mode 100644 index 00000000..0b662e1e Binary files /dev/null and b/tests/data-files/convert/us_ca_scm/i15_crop_mapping_2023_provisional_20241127.gdb.zip differ diff --git a/tests/data-files/convert/us_usda_cropland/NationalCSB_2016-2023_rev23.zip b/tests/data-files/convert/us_usda_cropland/NationalCSB_2016-2023_rev23.zip deleted file mode 100644 index fcc4ec9c..00000000 Binary files a/tests/data-files/convert/us_usda_cropland/NationalCSB_2016-2023_rev23.zip and /dev/null differ diff --git a/tests/data-files/convert/us_usda_cropland/NationalCSB_2017-2024_rev23.zip b/tests/data-files/convert/us_usda_cropland/NationalCSB_2017-2024_rev23.zip new file mode 100644 index 00000000..93c13ae0 Binary files /dev/null and b/tests/data-files/convert/us_usda_cropland/NationalCSB_2017-2024_rev23.zip differ diff --git a/tests/data-files/convert/us_usda_cropland/cropland.csv b/tests/data-files/convert/us_usda_cropland/cropland.csv new file mode 100644 index 00000000..cda4d0f1 --- /dev/null +++ b/tests/data-files/convert/us_usda_cropland/cropland.csv @@ -0,0 +1,256 @@ +original_code,original_name,translated_name,HCAT3_name,HCAT3_code +0,Background,Background,, +1,Corn,Corn,grain_maize_corn_popcorn,3301010600 +2,Cotton,Cotton,cotton,3301060300 +3,Rice,Rice,rice,3301010700 +4,Sorghum,Sorghum,millet_sorghum,3301010900 +5,Soybeans,Soybeans,soy_soybeans,3301160000 +6,Sunflower,Sunflower,sunflower,3301060500 +7,,, +8,,, +9,,, +10,Peanuts,Peanuts,nuts,3303030000 +11,Tobacco,Tobacco,tobacco,3301060100 +12,Sweet Corn,Sweet Corn,grain_maize_corn_popcorn,3301010600 +13,Pop or Orn Corn,Pop or Orn Corn,grain_maize_corn_popcorn,3301010600 +14,Mint,Mint,aromatic_medicinal_culinary_plants_spices_herbs,3301061200 +15,,, +16,,, +17,,, +18,,, +19,,, +20,,, +21,Barley,Barley,barley,3301010400 +22,Durum Wheat,Durum Wheat,unspecified_season_durum_hard_wheat,3301010299 +23,Spring Wheat,Spring Wheat,spring_common_soft_wheat,3301010102 +24,Winter Wheat,Winter Wheat,winter_common_soft_wheat,3301010101 +25,Other Small Grains,Other Small Grains,cereal,3301010000 +26,Dbl Crop WinWht/Soybeans,Dbl Crop WinWht/Soybeans,soy_soybeans,3301160000 +27,Rye,Rye,rye,3301010300 +28,Oats,Oats,oats,3301010500 +29,Millet,Millet,millet_sorghum,3301010900 +30,Speltz,Speltz,spelt,3301011000 +31,Canola,Canola,unspecified_season_rapeseed_rape,3301060499 +32,Flaxseed,Flaxseed,flax_linseed_oil,3301060702 +33,Safflower,Safflower,safflower,3301083900 +34,Rape Seed,Rape Seed,unspecified_season_rapeseed_rape,3301060499 +35,Mustard,Mustard,mustard,3301210100 +36,Alfalfa,Alfalfa,alfalfa_lucerne,3301090301 +37,Other Hay/Non Alfalfa,Other Hay/Non Alfalfa,alfalfa_lucerne,3301090301 +38,Camelina,Camelina,camelina,3301061500 +39,Buckwheat,Buckwheat,buckwheat,3301150200 +40,,, +41,Sugarbeets,Sugarbeets,sugar_beet,3301290700 +42,Dry Beans,Dry Beans,beans,3301020100 +43,Potatoes,Potatoes,potatoes,3301030000 +44,Other Crops,Other Crops,, +45,Sugarcane,Sugarcane,, +46,Sweet Potatoes,Sweet Potatoes,sweet_potatoes,3301040000 +47,Misc Vegs & Fruits,Misc Vegs & Fruits,unspecified_orchards_fruits,3303019800 +48,Watermelons,Watermelons,watermelon,3301140500 +49,Onions,Onions,onions,3301220400 +50,Cucumbers,Cucumbers,cucumber_pickle,3301140100 +51,Chick Peas,Chick Peas,chickpeas,3301020200 +52,Lentils,Lentils,lentils,3301020500 +53,Peas,Peas,peas,3301020600 +54,Tomatoes,Tomatoes,tomato,3301280000 +55,Caneberries,Caneberries,unspecified_berries_berry_species,3303029800 +56,Hops,Hops,hops,3301060200 +57,Herbs,Herbs,aromatic_medicinal_culinary_plants_spices_herbs,3301061200 +58,Clover/Wildflowers,Clover/Wildflowers,clover,3301090303 +59,Sod/Grass Seed,Sod/Grass Seed,poaceae_grasses,3301090200 +60,Switchgrass,Switchgrass,switchgrass,3301090208 +61,Fallow/Idle Cropland,Fallow/Idle Cropland,fallow_land_not_crop,3301110000 +62,,, +63,Forest,Forest,tree_wood_forest,3306000000 +64,Shrubland,Shrubland,unmaintained,3308000000 +65,Barren,Barren,unmaintained,3308000000 +66,Cherries,Cherries,cherry_cherries,3303010400 +67,Peaches,Peaches,peach,3303011100 +68,Apples,Apples,apples,3303010200 +69,Grapes,Grapes,vineyards_wine_vine_rebland_grapes,3303060000 +70,Christmas Trees,Christmas Trees,other_tree_wood_forest,3306990000 +71,Other Tree Crops,Other Tree Crops,unspecified_permanent_crops,3303120000 +72,Citrus,Citrus,citrus_plantations,3303040000 +73,,, +74,Pecans,Pecans,pecan,3303030300 +75,Almonds,Almonds,almond,3303030100 +76,Walnuts,Walnuts,walnuts,3303030600 +77,Pears,Pears,pears,3303011200 +78,,, +79,,, +80,,, +81,Clouds/No Data,Clouds/No Data,, +82,Developed,Developed,, +83,Water,Water,, +84,,, +85,,, +86,,, +87,Wetlands,Wetlands,, +88,Nonag/Undefined,Nonag/Undefined,, +89,,, +90,,, +91,,, +92,Aquaculture,Aquaculture,, +93,,, +94,,, +95,,, +96,,, +97,,, +98,,, +99,,, +100,,, +101,,, +102,,, +103,,, +104,,, +105,,, +106,,, +107,,, +108,,, +109,,, +110,,, +111,Open Water,Open Water,, +112,Perennial Ice/Snow,Perennial Ice/Snow,, +113,,, +114,,, +115,,, +116,,, +117,,, +118,,, +119,,, +120,,, +121,Developed/Open Space,Developed/Open Space,, +122,Developed/Low Intensity,Developed/Low Intensity,, +123,Developed/Med Intensity,Developed/Med Intensity,, +124,Developed/High Intensity,Developed/High Intensity,, +125,,, +126,,, +127,,, +128,,, +129,,, +130,,, +131,Barren,Barren,, +132,,, +133,,, +134,,, +135,,, +136,,, +137,,, +138,,, +139,,, +140,,, +141,Deciduous Forest,Deciduous Forest,tree_wood_forest,3306000000 +142,Evergreen Forest,Evergreen Forest,tree_wood_forest,3306000000 +143,Mixed Forest,Mixed Forest,tree_wood_forest,3306000000 +144,,, +145,,, +146,,, +147,,, +148,,, +149,,, +150,,, +151,,, +152,Shrubland,Shrubland,unmaintained,3308000000 +154,,, +155,,, +156,,, +157,,, +158,,, +159,,, +160,,, +161,,, +162,,, +163,,, +164,,, +165,,, +166,,, +167,,, +168,,, +169,,, +170,,, +171,,, +172,,, +173,,, +174,,, +175,,, +176,Grass/Pasture,Grass/Pasture,pasture_meadow_grassland_grass,3302000000 +177,,, +178,,, +179,,, +180,,, +181,,, +182,,, +183,,, +184,,, +185,,, +186,,, +187,,, +188,,, +189,,, +190,Woody Wetlands,Woody Wetlands,, +191,,, +192,,, +193,,, +194,,, +195,Herbaceous Wetlands,Herbaceous Wetlands,, +196,,, +197,,, +198,,, +199,,, +200,,, +201,,, +202,,, +203,,, +204,Pistachios,Pistachios,pistachio,3303030400 +205,Triticale,Triticale,triticale,3301010800 +206,Carrots,Carrots,carrots_daucus,3301290300 +207,Asparagus,Asparagus,asparagus,3301200000 +208,Garlic,Garlic,garlic,3301220200 +209,Cantaloupes,Cantaloupes,melon,3301140300 +210,Prunes,Prunes,plums,3303011300 +211,Olives,Olives,olive_plantations,3303050000 +212,Oranges,Oranges,citrus_plantations,3303040000 +213,Honeydew Melons,Honeydew Melons,melon,3301140300 +214,Broccoli,Broccoli,broccoli,3301210202 +215,Avocados,Avocados,avocado,3303100000 +216,Peppers,Peppers,capsicum,3301300000 +217,Pomegranates,Pomegranates,pomegranate,3303011400 +218,Nectarines,Nectarines,nectarine,3303010900 +219,Greens,Greens,salads_lettuce_leaf_vegetables,3301310000 +220,Plums,Plums,plums,3303011300 +221,Strawberries,Strawberries,strawberries,3301130000 +222,Squash,Squash,pumpkin_squash_gourd,3301140400 +223,Apricots,Apricots,apricots,3303010300 +224,Vetch,Vetch,vetches,3301090305 +225,Dbl Crop WinWht/Corn,Dbl Crop WinWht/Corn,winter_common_soft_wheat,3301010101 +226,Dbl Crop Oats/Corn,Dbl Crop Oats/Corn,oats,3301010500 +227,Lettuce,Lettuce,salads_lettuce_leaf_vegetables,3301310000 +228,Dbl Crop Triticale/Corn,Dbl Crop Triticale/Corn,triticale,3301010800 +229,Pumpkins,Pumpkins,pumpkin_squash_gourd,3301140400 +230,Dbl Crop Lettuce/Durum Wht,Dbl Crop Lettuce/Durum Wht,salads_lettuce_leaf_vegetables,3301310000 +231,Dbl Crop Lettuce/Cantaloupe,Dbl Crop Lettuce/Cantaloupe,salads_lettuce_leaf_vegetables,3301310000 +232,Dbl Crop Lettuce/Cotton,Dbl Crop Lettuce/Cotton,salads_lettuce_leaf_vegetables,3301310000 +233,Dbl Crop Lettuce/Barley,Dbl Crop Lettuce/Barley,salads_lettuce_leaf_vegetables,3301310000 +234,Dbl Crop Durum Wht/Sorghum,Dbl Crop Durum Wht/Sorghum,unspecified_season_durum_hard_wheat,3301010299 +235,Dbl Crop Barley/Sorghum,Dbl Crop Barley/Sorghum,barley,3301010400 +236,Dbl Crop WinWht/Sorghum,Dbl Crop WinWht/Sorghum,winter_common_soft_wheat,3301010101 +237,Dbl Crop Barley/Corn,Dbl Crop Barley/Corn,barley,3301010400 +238,Dbl Crop WinWht/Cotton,Dbl Crop WinWht/Cotton,winter_common_soft_wheat,3301010101 +239,Dbl Crop Soybeans/Cotton,Dbl Crop Soybeans/Cotton,cotton,3301060300 +240,Dbl Crop Soybeans/Oats,Dbl Crop Soybeans/Oats,oats,3301010500 +241,Dbl Crop Corn/Soybeans,Dbl Crop Corn/Soybeans,grain_maize_corn_popcorn,3301010600 +242,Blueberries,Blueberries,blueberry,3303020400 +243,Cabbage,Cabbage,brassica_oleracea_cabbage,3301210200 +244,Cauliflower,Cauliflower,cauliflower,3301210204 +245,Celery,Celery,celery,3301250000 +246,Radishes,Radishes,radish,3301290600 +247,Turnips,Turnips,turnips,3301290800 +248,Eggplants,Eggplants,aubergine_eggplant,3301260000 +249,Gourds,Gourds,pumpkin_squash_gourd,3301140400 +250,Cranberries,Cranberries,cranberry,3303020500 +251,,, +252,,, +253,,, +254,Dbl Crop Barley/Soybeans,Dbl Crop Barley/Soybeans,barley,3301010400 +255,,, diff --git a/tests/data-files/stac-collection.json b/tests/data-files/stac-collection.json index 6fc921da..fef2cb62 100644 --- a/tests/data-files/stac-collection.json +++ b/tests/data-files/stac-collection.json @@ -1,7 +1,7 @@ { "stac_version": "1.1.0", "stac_extensions": [ - "https://stac-extensions.github.io/processing/v1.1.0/schema.json" + "https://stac-extensions.github.io/processing/v1.2.0/schema.json" ], "type": "Collection", "id": "de_nrw", diff --git a/tests/test_convert.py b/tests/test_convert.py index 04a1c56b..dc7698d0 100644 --- a/tests/test_convert.py +++ b/tests/test_convert.py @@ -16,18 +16,19 @@ tests = [ "at", - "at_crop", + "at_block", "be_vlg", "br_ba_lem", "bg", "de_sh", + "de_bb", "ec_lv", "ec_si", "fi", "fr", "hr", "nl", - "nl_crop", + "nl_block", "pt", "dk", "be_wal", @@ -36,6 +37,7 @@ "ch", "cz", "us_usda_cropland", + "us_ca_scm", "jp", "lv", "ie", @@ -57,7 +59,9 @@ def _input_files(converter, *names): extra_convert_parameters = { "ai4sf": _input_files("ai4sf", "1_vietnam_areas.gpkg", "4_cambodia_areas.gpkg"), - "nl_crop": {"variant": "2023"}, + "nl": {"variant": "2023"}, + "se": {"variant": "2023"}, + "si": {"variant": "2023"}, "be_vlg": {"variant": "2023"}, "br_ba_lem": _input_files("br_ba_lem", "LEM_dataset.zip"), "ch": _input_files("ch", "lwb_nutzungsflaechen_v2_0_lv95.gpkg"), @@ -74,6 +78,8 @@ def test_converter(load_ec_mock, capsys, tmp_parquet_file, converter): from fiboa_cli import Registry # noqa def load_ec(csv_file=None, url=None): + if csv_file and "://" in csv_file: + csv_file = csv_file.split("/")[-1] path = url if url and "://" not in url else f"{test_path}/{converter}/{csv_file}" return list(DictReader(open(path, "r", encoding="utf-8"))) @@ -95,6 +101,6 @@ def load_ec(csv_file=None, url=None): ValidateData().validate(tmp_parquet_file) df = pq.read_table(tmp_parquet_file).to_pandas() - if "metrics:area" in df.columns: + if "metrics:area" in df.columns and converter not in ("de_bb",): # Check for accidental hectare conversion; fields should be more than 10 square meters assert (df["metrics:area"] > 10).all() diff --git a/tests/test_publish.py b/tests/test_publish.py index 663b2c4b..cf960760 100644 --- a/tests/test_publish.py +++ b/tests/test_publish.py @@ -3,7 +3,7 @@ from fiboa_cli.publish import Publish -class TestPublish(Publish): +class PublishTest(Publish): def generate_pmtiles(self, target, file_name, parquet_file): pass @@ -22,7 +22,7 @@ def test_publish(tmp_folder): body=open(f"tests/data-files/publish/{base}-survey.md").read(), ) responses.add(rsp1) - TestPublish(converter).run( + PublishTest(converter).run( variant="2023", target=tmp_folder, cache=path, generate_meta=True, yes=True ) files = [f.name for f in tmp_folder.iterdir() if f.is_file()]