diff --git a/config/common/leaflet-layers.json b/config/common/leaflet-layers.json new file mode 100644 index 00000000000..970fb60eb63 --- /dev/null +++ b/config/common/leaflet-layers.json @@ -0,0 +1,118 @@ +{ + "baseMaps": { + "Street Map (OpenStreetMap)": { + "endpoint": "https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png", + "serverType": "tileServer", + "layerOptions": { + "maxZoom": 19, + "attribution": "© OpenStreetMap contributors" + }, + "styles": [ + "auto-dark-mode" + ] + }, + "Street Map (ESRI)": { + "endpoint": "https://server.arcgisonline.com/ArcGIS/rest/services/World_Street_Map/MapServer/tile/{z}/{y}/{x}", + "serverType": "tileServer", + "layerOptions": { + "maxZoom": 23, + "attribution": "Esri, HERE, Garmin, USGS, Intermap, INCREMENT P, NRCan, Esri Japan, METI, Esri China (Hong Kong), Esri Korea, Esri (Thailand), NGCC, (c) OpenStreetMap contributors, and the GIS User Community" + }, + "styles": [ + "auto-dark-mode" + ] + }, + "Topographic Map (ESRI)": { + "endpoint": "https://server.arcgisonline.com/ArcGIS/rest/services/World_Topo_Map/MapServer/tile/{z}/{y}/{x}", + "serverType": "tileServer", + "layerOptions": { + "maxZoom": 23, + "attribution": "Sources: Esri, HERE, Garmin, Intermap, increment P Corp., GEBCO, USGS, FAO, NPS, NRCAN, GeoBase, IGN, Kadaster NL, Ordnance Survey, Esri Japan, METI, Esri China (Hong Kong), (c) OpenStreetMap contributors, and the GIS User Community" + }, + "styles": [ + "auto-dark-mode" + ] + }, + "Satellite Map (ESRI)": { + "endpoint": "https://server.arcgisonline.com/ArcGIS/rest/services/World_Imagery/MapServer/tile/{z}/{y}/{x}", + "serverType": "tileServer", + "layerOptions": { + "maxZoom": 23, + "attribution": "Tiles © Esri — Source: Esri, i-cubed, USDA, USGS, AEX, GeoEye, Getmapping, Aerogrid, IGN, IGP, UPR-EGP, and the GIS User Community" + } + }, + "Satellite Map (ESRI) (grayscale)": { + "endpoint": "https://server.arcgisonline.com/ArcGIS/rest/services/World_Imagery/MapServer/tile/{z}/{y}/{x}", + "serverType": "tileServer", + "layerOptions": { + "maxZoom": 23, + "attribution": "Tiles © Esri — Source: Esri, i-cubed, USDA, USGS, AEX, GeoEye, Getmapping, Aerogrid, IGN, IGP, UPR-EGP, and the GIS User Community" + }, + "styles": [ + "grayscale" + ] + }, + "Satellite Map (Géoportail/France)": { + "endpoint": "https://wxs.ign.fr/{apikey}/geoportail/wmts?REQUEST=GetTile&SERVICE=WMTS&VERSION=1.0.0&STYLE={style}&TILEMATRIXSET=PM&FORMAT={format}&LAYER=ORTHOIMAGERY.ORTHOPHOTOS&TILEMATRIX={z}&TILEROW={y}&TILECOL={x}", + "serverType": "tileServer", + "layerOptions": { + "attribution": "Geoportail France", + "bounds": [ + [ + -75, + -180 + ], + [ + 81, + 180 + ] + ], + "minZoom": 2, + "maxZoom": 12, + "apikey": "choisirgeoportail", + "format": "image/jpeg", + "style": "normal" + } + }, + "Satellite Map (USGS)": { + "endpoint": "https://basemap.nationalmap.gov/arcgis/rest/services/USGSImageryOnly/MapServer/tile/{z}/{y}/{x}", + "serverType": "tileServer", + "layerOptions": { + "maxZoom": 16, + "attribution": "Tiles courtesy of the U.S. Geological Survey" + } + }, + "Live Satellite Map (NASA/GIBS)": { + "endpoint": "https://gibs.earthdata.nasa.gov/wmts/epsg3031/best/MODIS_Terra_CorrectedReflectance_TrueColor/default/{time}/{tilematrixset}/{z}/{y}/{x}.{format}", + "serverType": "tileServer", + "layerOptions": { + "attribution": "Imagery provided by services from the Global Imagery Browse Services (GIBS), operated by the NASA/GSFC/Earth Science Data and Information System (ESDIS) with funding provided by NASA/HQ.", + "bounds": [ + [ + -85.0511287776, + -179.999999975 + ], + [ + 85.0511287776, + 179.999999975 + ] + ], + "minZoom": 1, + "maxZoom": 9, + "format": "jpg", + "time": "", + "tilematrixset": "250m" + } + } + }, + "overlays": { + "Labels and boundaries": { + "endpoint": "https://esp.usdoj.gov/arcweb/rest/services/World_Boundaries_and_Places/MapServer/tile/{z}/{y}/{x}", + "serverType": "tileServer", + "layerOptions": { + "maxZoom": 23, + "attribution": "Esri, HERE, Garmin, (c) OpenStreetMap contributors, and the GIS user community" + } + } + } +} diff --git a/config/default_trees/2026_02_03_IMA_minerals.csv.gz b/config/default_trees/2026_02_03_IMA_minerals.csv.gz new file mode 100644 index 00000000000..607de4d6191 Binary files /dev/null and b/config/default_trees/2026_02_03_IMA_minerals.csv.gz differ diff --git a/config/default_trees/GeologicTimePeriod.csv.gz b/config/default_trees/GeologicTimePeriod.csv.gz new file mode 100644 index 00000000000..28bd1accc3c Binary files /dev/null and b/config/default_trees/GeologicTimePeriod.csv.gz differ diff --git a/config/default_trees/col2008_aves.csv.gz b/config/default_trees/col2008_aves.csv.gz new file mode 100644 index 00000000000..f5fa5afc5da Binary files /dev/null and b/config/default_trees/col2008_aves.csv.gz differ diff --git a/config/default_trees/col2008_fishes.csv.gz b/config/default_trees/col2008_fishes.csv.gz new file mode 100644 index 00000000000..8835537366b Binary files /dev/null and b/config/default_trees/col2008_fishes.csv.gz differ diff --git a/config/default_trees/col2008_herps.csv.gz b/config/default_trees/col2008_herps.csv.gz new file mode 100644 index 00000000000..b42e47e7a9a Binary files /dev/null and b/config/default_trees/col2008_herps.csv.gz differ diff --git a/config/default_trees/col2008_inverts.csv.gz b/config/default_trees/col2008_inverts.csv.gz new file mode 100644 index 00000000000..c7e05352edf Binary files /dev/null and b/config/default_trees/col2008_inverts.csv.gz differ diff --git a/config/default_trees/col2008_mammalia.csv.gz b/config/default_trees/col2008_mammalia.csv.gz new file mode 100644 index 00000000000..08431d6a846 Binary files /dev/null and b/config/default_trees/col2008_mammalia.csv.gz differ diff --git a/config/default_trees/col2008_orthoptera.csv.gz b/config/default_trees/col2008_orthoptera.csv.gz new file mode 100644 index 00000000000..7999c27a720 Binary files /dev/null and b/config/default_trees/col2008_orthoptera.csv.gz differ diff --git a/config/default_trees/col2008_poales.csv.gz b/config/default_trees/col2008_poales.csv.gz new file mode 100644 index 00000000000..88040a59f7f Binary files /dev/null and b/config/default_trees/col2008_poales.csv.gz differ diff --git a/config/default_trees/col2021_actinopterygii.csv.gz b/config/default_trees/col2021_actinopterygii.csv.gz new file mode 100644 index 00000000000..b9f30d1e9a5 Binary files /dev/null and b/config/default_trees/col2021_actinopterygii.csv.gz differ diff --git a/config/default_trees/col2021_amphibia_reptilia.csv.gz b/config/default_trees/col2021_amphibia_reptilia.csv.gz new file mode 100644 index 00000000000..df294c51096 Binary files /dev/null and b/config/default_trees/col2021_amphibia_reptilia.csv.gz differ diff --git a/config/default_trees/col2021_aves.csv.gz b/config/default_trees/col2021_aves.csv.gz new file mode 100644 index 00000000000..eea995ffbf8 Binary files /dev/null and b/config/default_trees/col2021_aves.csv.gz differ diff --git a/config/default_trees/col2021_bryophyta.csv.gz b/config/default_trees/col2021_bryophyta.csv.gz new file mode 100644 index 00000000000..71bf9b0158f Binary files /dev/null and b/config/default_trees/col2021_bryophyta.csv.gz differ diff --git a/config/default_trees/col2021_fungi.csv.gz b/config/default_trees/col2021_fungi.csv.gz new file mode 100644 index 00000000000..b24802fa54f Binary files /dev/null and b/config/default_trees/col2021_fungi.csv.gz differ diff --git a/config/default_trees/col2021_mammalia.csv.gz b/config/default_trees/col2021_mammalia.csv.gz new file mode 100644 index 00000000000..22f0266178e Binary files /dev/null and b/config/default_trees/col2021_mammalia.csv.gz differ diff --git a/config/default_trees/col2021_mollusca.csv.gz b/config/default_trees/col2021_mollusca.csv.gz new file mode 100644 index 00000000000..cf190c1fa76 Binary files /dev/null and b/config/default_trees/col2021_mollusca.csv.gz differ diff --git a/config/default_trees/col2021_mollusca.xlsx b/config/default_trees/col2021_mollusca.xlsx new file mode 100644 index 00000000000..1feb38b3750 Binary files /dev/null and b/config/default_trees/col2021_mollusca.xlsx differ diff --git a/config/default_trees/col2021_polypodiopsida.csv.gz b/config/default_trees/col2021_polypodiopsida.csv.gz new file mode 100644 index 00000000000..73dcd96a733 Binary files /dev/null and b/config/default_trees/col2021_polypodiopsida.csv.gz differ diff --git a/config/default_trees/col2021_tracheophyta.csv.gz b/config/default_trees/col2021_tracheophyta.csv.gz new file mode 100644 index 00000000000..25dc15fd1ea Binary files /dev/null and b/config/default_trees/col2021_tracheophyta.csv.gz differ diff --git a/config/default_trees/geonames.csv.gz b/config/default_trees/geonames.csv.gz new file mode 100644 index 00000000000..6abaf6c4aa2 Binary files /dev/null and b/config/default_trees/geonames.csv.gz differ diff --git a/config/default_trees/mapping_files/2026_02_03_IMA_minerals.json b/config/default_trees/mapping_files/2026_02_03_IMA_minerals.json new file mode 100644 index 00000000000..608f1b89844 --- /dev/null +++ b/config/default_trees/mapping_files/2026_02_03_IMA_minerals.json @@ -0,0 +1,21 @@ +{ + "all_columns": ["Mineral Name","Mineral Name (plain)","Valence Chemistry","IMA Chemistry","IMA Number","Country of Type Locality","IMA Status","isMineral"], + "root": { + "name": "Root" + }, + "ranks": [ + { + "name": "Minerals", + "rank": 10, + "column": "Mineral Name", + "fields": { + "name": "Mineral Name (plain)", + "text10": "Valence Chemistry", + "text14": "IMA Chemistry", + "text12": "IMA Number", + "text13": "Country of Type Locality", + "text11": "IMA Status" + } + } + ] +} \ No newline at end of file diff --git a/config/default_trees/mapping_files/col2021_actinopterygii.json b/config/default_trees/mapping_files/col2021_actinopterygii.json new file mode 100644 index 00000000000..7853736c746 --- /dev/null +++ b/config/default_trees/mapping_files/col2021_actinopterygii.json @@ -0,0 +1,94 @@ +{ + "all_columns": [ + "kingdom", + "phylum", + "class", + "order", + "family", + "genus", + "species", + "subspecies", + "family common name", + "species author", + "species source", + "species common name", + "subspecies author", + "subspecies source", + "subspecies common name" + ], + "ranks": [ + { + "name": "Kingdom", + "rank": 10, + "column": "kingdom", + "fields": { + "name": "kingdom" + } + }, + { + "name": "Phylum", + "rank": 30, + "column": "phylum", + "fields": { + "name": "phylum" + } + }, + { + "name": "Class", + "rank": 60, + "column": "class", + "fields": { + "name": "class" + } + }, + { + "name": "Order", + "rank": 100, + "column": "order", + "fields": { + "name": "order" + } + }, + { + "name": "Family", + "rank": 140, + "column": "family", + "fields": { + "name": "family" + } + }, + { + "name": "Genus", + "rank": 180, + "column": "genus", + "infullname": true, + "fields": { + "name": "genus" + } + }, + { + "name": "Species", + "rank": 220, + "column": "species", + "infullname": true, + "fields": { + "name": "species", + "author": "species author", + "source": "species source", + "commonname": "species common name" + } + }, + { + "name": "Subspecies", + "rank": 230, + "column": "subspecies", + "infullname": true, + "fields": { + "name": "subspecies", + "author": "subspecies author", + "source": "subspecies source", + "commonname": "subspecies common name" + } + } + ] +} \ No newline at end of file diff --git a/config/default_trees/mapping_files/col2021_amphibia_reptilia.json b/config/default_trees/mapping_files/col2021_amphibia_reptilia.json new file mode 100644 index 00000000000..7853736c746 --- /dev/null +++ b/config/default_trees/mapping_files/col2021_amphibia_reptilia.json @@ -0,0 +1,94 @@ +{ + "all_columns": [ + "kingdom", + "phylum", + "class", + "order", + "family", + "genus", + "species", + "subspecies", + "family common name", + "species author", + "species source", + "species common name", + "subspecies author", + "subspecies source", + "subspecies common name" + ], + "ranks": [ + { + "name": "Kingdom", + "rank": 10, + "column": "kingdom", + "fields": { + "name": "kingdom" + } + }, + { + "name": "Phylum", + "rank": 30, + "column": "phylum", + "fields": { + "name": "phylum" + } + }, + { + "name": "Class", + "rank": 60, + "column": "class", + "fields": { + "name": "class" + } + }, + { + "name": "Order", + "rank": 100, + "column": "order", + "fields": { + "name": "order" + } + }, + { + "name": "Family", + "rank": 140, + "column": "family", + "fields": { + "name": "family" + } + }, + { + "name": "Genus", + "rank": 180, + "column": "genus", + "infullname": true, + "fields": { + "name": "genus" + } + }, + { + "name": "Species", + "rank": 220, + "column": "species", + "infullname": true, + "fields": { + "name": "species", + "author": "species author", + "source": "species source", + "commonname": "species common name" + } + }, + { + "name": "Subspecies", + "rank": 230, + "column": "subspecies", + "infullname": true, + "fields": { + "name": "subspecies", + "author": "subspecies author", + "source": "subspecies source", + "commonname": "subspecies common name" + } + } + ] +} \ No newline at end of file diff --git a/config/default_trees/mapping_files/col2021_aves.json b/config/default_trees/mapping_files/col2021_aves.json new file mode 100644 index 00000000000..7853736c746 --- /dev/null +++ b/config/default_trees/mapping_files/col2021_aves.json @@ -0,0 +1,94 @@ +{ + "all_columns": [ + "kingdom", + "phylum", + "class", + "order", + "family", + "genus", + "species", + "subspecies", + "family common name", + "species author", + "species source", + "species common name", + "subspecies author", + "subspecies source", + "subspecies common name" + ], + "ranks": [ + { + "name": "Kingdom", + "rank": 10, + "column": "kingdom", + "fields": { + "name": "kingdom" + } + }, + { + "name": "Phylum", + "rank": 30, + "column": "phylum", + "fields": { + "name": "phylum" + } + }, + { + "name": "Class", + "rank": 60, + "column": "class", + "fields": { + "name": "class" + } + }, + { + "name": "Order", + "rank": 100, + "column": "order", + "fields": { + "name": "order" + } + }, + { + "name": "Family", + "rank": 140, + "column": "family", + "fields": { + "name": "family" + } + }, + { + "name": "Genus", + "rank": 180, + "column": "genus", + "infullname": true, + "fields": { + "name": "genus" + } + }, + { + "name": "Species", + "rank": 220, + "column": "species", + "infullname": true, + "fields": { + "name": "species", + "author": "species author", + "source": "species source", + "commonname": "species common name" + } + }, + { + "name": "Subspecies", + "rank": 230, + "column": "subspecies", + "infullname": true, + "fields": { + "name": "subspecies", + "author": "subspecies author", + "source": "subspecies source", + "commonname": "subspecies common name" + } + } + ] +} \ No newline at end of file diff --git a/config/default_trees/mapping_files/col2021_bryophyta.json b/config/default_trees/mapping_files/col2021_bryophyta.json new file mode 100644 index 00000000000..7853736c746 --- /dev/null +++ b/config/default_trees/mapping_files/col2021_bryophyta.json @@ -0,0 +1,94 @@ +{ + "all_columns": [ + "kingdom", + "phylum", + "class", + "order", + "family", + "genus", + "species", + "subspecies", + "family common name", + "species author", + "species source", + "species common name", + "subspecies author", + "subspecies source", + "subspecies common name" + ], + "ranks": [ + { + "name": "Kingdom", + "rank": 10, + "column": "kingdom", + "fields": { + "name": "kingdom" + } + }, + { + "name": "Phylum", + "rank": 30, + "column": "phylum", + "fields": { + "name": "phylum" + } + }, + { + "name": "Class", + "rank": 60, + "column": "class", + "fields": { + "name": "class" + } + }, + { + "name": "Order", + "rank": 100, + "column": "order", + "fields": { + "name": "order" + } + }, + { + "name": "Family", + "rank": 140, + "column": "family", + "fields": { + "name": "family" + } + }, + { + "name": "Genus", + "rank": 180, + "column": "genus", + "infullname": true, + "fields": { + "name": "genus" + } + }, + { + "name": "Species", + "rank": 220, + "column": "species", + "infullname": true, + "fields": { + "name": "species", + "author": "species author", + "source": "species source", + "commonname": "species common name" + } + }, + { + "name": "Subspecies", + "rank": 230, + "column": "subspecies", + "infullname": true, + "fields": { + "name": "subspecies", + "author": "subspecies author", + "source": "subspecies source", + "commonname": "subspecies common name" + } + } + ] +} \ No newline at end of file diff --git a/config/default_trees/mapping_files/col2021_fungi.json b/config/default_trees/mapping_files/col2021_fungi.json new file mode 100644 index 00000000000..7853736c746 --- /dev/null +++ b/config/default_trees/mapping_files/col2021_fungi.json @@ -0,0 +1,94 @@ +{ + "all_columns": [ + "kingdom", + "phylum", + "class", + "order", + "family", + "genus", + "species", + "subspecies", + "family common name", + "species author", + "species source", + "species common name", + "subspecies author", + "subspecies source", + "subspecies common name" + ], + "ranks": [ + { + "name": "Kingdom", + "rank": 10, + "column": "kingdom", + "fields": { + "name": "kingdom" + } + }, + { + "name": "Phylum", + "rank": 30, + "column": "phylum", + "fields": { + "name": "phylum" + } + }, + { + "name": "Class", + "rank": 60, + "column": "class", + "fields": { + "name": "class" + } + }, + { + "name": "Order", + "rank": 100, + "column": "order", + "fields": { + "name": "order" + } + }, + { + "name": "Family", + "rank": 140, + "column": "family", + "fields": { + "name": "family" + } + }, + { + "name": "Genus", + "rank": 180, + "column": "genus", + "infullname": true, + "fields": { + "name": "genus" + } + }, + { + "name": "Species", + "rank": 220, + "column": "species", + "infullname": true, + "fields": { + "name": "species", + "author": "species author", + "source": "species source", + "commonname": "species common name" + } + }, + { + "name": "Subspecies", + "rank": 230, + "column": "subspecies", + "infullname": true, + "fields": { + "name": "subspecies", + "author": "subspecies author", + "source": "subspecies source", + "commonname": "subspecies common name" + } + } + ] +} \ No newline at end of file diff --git a/config/default_trees/mapping_files/col2021_mammalia.json b/config/default_trees/mapping_files/col2021_mammalia.json new file mode 100644 index 00000000000..7853736c746 --- /dev/null +++ b/config/default_trees/mapping_files/col2021_mammalia.json @@ -0,0 +1,94 @@ +{ + "all_columns": [ + "kingdom", + "phylum", + "class", + "order", + "family", + "genus", + "species", + "subspecies", + "family common name", + "species author", + "species source", + "species common name", + "subspecies author", + "subspecies source", + "subspecies common name" + ], + "ranks": [ + { + "name": "Kingdom", + "rank": 10, + "column": "kingdom", + "fields": { + "name": "kingdom" + } + }, + { + "name": "Phylum", + "rank": 30, + "column": "phylum", + "fields": { + "name": "phylum" + } + }, + { + "name": "Class", + "rank": 60, + "column": "class", + "fields": { + "name": "class" + } + }, + { + "name": "Order", + "rank": 100, + "column": "order", + "fields": { + "name": "order" + } + }, + { + "name": "Family", + "rank": 140, + "column": "family", + "fields": { + "name": "family" + } + }, + { + "name": "Genus", + "rank": 180, + "column": "genus", + "infullname": true, + "fields": { + "name": "genus" + } + }, + { + "name": "Species", + "rank": 220, + "column": "species", + "infullname": true, + "fields": { + "name": "species", + "author": "species author", + "source": "species source", + "commonname": "species common name" + } + }, + { + "name": "Subspecies", + "rank": 230, + "column": "subspecies", + "infullname": true, + "fields": { + "name": "subspecies", + "author": "subspecies author", + "source": "subspecies source", + "commonname": "subspecies common name" + } + } + ] +} \ No newline at end of file diff --git a/config/default_trees/mapping_files/col2021_mollusca.json b/config/default_trees/mapping_files/col2021_mollusca.json new file mode 100644 index 00000000000..7853736c746 --- /dev/null +++ b/config/default_trees/mapping_files/col2021_mollusca.json @@ -0,0 +1,94 @@ +{ + "all_columns": [ + "kingdom", + "phylum", + "class", + "order", + "family", + "genus", + "species", + "subspecies", + "family common name", + "species author", + "species source", + "species common name", + "subspecies author", + "subspecies source", + "subspecies common name" + ], + "ranks": [ + { + "name": "Kingdom", + "rank": 10, + "column": "kingdom", + "fields": { + "name": "kingdom" + } + }, + { + "name": "Phylum", + "rank": 30, + "column": "phylum", + "fields": { + "name": "phylum" + } + }, + { + "name": "Class", + "rank": 60, + "column": "class", + "fields": { + "name": "class" + } + }, + { + "name": "Order", + "rank": 100, + "column": "order", + "fields": { + "name": "order" + } + }, + { + "name": "Family", + "rank": 140, + "column": "family", + "fields": { + "name": "family" + } + }, + { + "name": "Genus", + "rank": 180, + "column": "genus", + "infullname": true, + "fields": { + "name": "genus" + } + }, + { + "name": "Species", + "rank": 220, + "column": "species", + "infullname": true, + "fields": { + "name": "species", + "author": "species author", + "source": "species source", + "commonname": "species common name" + } + }, + { + "name": "Subspecies", + "rank": 230, + "column": "subspecies", + "infullname": true, + "fields": { + "name": "subspecies", + "author": "subspecies author", + "source": "subspecies source", + "commonname": "subspecies common name" + } + } + ] +} \ No newline at end of file diff --git a/config/default_trees/mapping_files/col2021_polypodiopsida.json b/config/default_trees/mapping_files/col2021_polypodiopsida.json new file mode 100644 index 00000000000..7853736c746 --- /dev/null +++ b/config/default_trees/mapping_files/col2021_polypodiopsida.json @@ -0,0 +1,94 @@ +{ + "all_columns": [ + "kingdom", + "phylum", + "class", + "order", + "family", + "genus", + "species", + "subspecies", + "family common name", + "species author", + "species source", + "species common name", + "subspecies author", + "subspecies source", + "subspecies common name" + ], + "ranks": [ + { + "name": "Kingdom", + "rank": 10, + "column": "kingdom", + "fields": { + "name": "kingdom" + } + }, + { + "name": "Phylum", + "rank": 30, + "column": "phylum", + "fields": { + "name": "phylum" + } + }, + { + "name": "Class", + "rank": 60, + "column": "class", + "fields": { + "name": "class" + } + }, + { + "name": "Order", + "rank": 100, + "column": "order", + "fields": { + "name": "order" + } + }, + { + "name": "Family", + "rank": 140, + "column": "family", + "fields": { + "name": "family" + } + }, + { + "name": "Genus", + "rank": 180, + "column": "genus", + "infullname": true, + "fields": { + "name": "genus" + } + }, + { + "name": "Species", + "rank": 220, + "column": "species", + "infullname": true, + "fields": { + "name": "species", + "author": "species author", + "source": "species source", + "commonname": "species common name" + } + }, + { + "name": "Subspecies", + "rank": 230, + "column": "subspecies", + "infullname": true, + "fields": { + "name": "subspecies", + "author": "subspecies author", + "source": "subspecies source", + "commonname": "subspecies common name" + } + } + ] +} \ No newline at end of file diff --git a/config/default_trees/mapping_files/col2021_tracheophyta.json b/config/default_trees/mapping_files/col2021_tracheophyta.json new file mode 100644 index 00000000000..7853736c746 --- /dev/null +++ b/config/default_trees/mapping_files/col2021_tracheophyta.json @@ -0,0 +1,94 @@ +{ + "all_columns": [ + "kingdom", + "phylum", + "class", + "order", + "family", + "genus", + "species", + "subspecies", + "family common name", + "species author", + "species source", + "species common name", + "subspecies author", + "subspecies source", + "subspecies common name" + ], + "ranks": [ + { + "name": "Kingdom", + "rank": 10, + "column": "kingdom", + "fields": { + "name": "kingdom" + } + }, + { + "name": "Phylum", + "rank": 30, + "column": "phylum", + "fields": { + "name": "phylum" + } + }, + { + "name": "Class", + "rank": 60, + "column": "class", + "fields": { + "name": "class" + } + }, + { + "name": "Order", + "rank": 100, + "column": "order", + "fields": { + "name": "order" + } + }, + { + "name": "Family", + "rank": 140, + "column": "family", + "fields": { + "name": "family" + } + }, + { + "name": "Genus", + "rank": 180, + "column": "genus", + "infullname": true, + "fields": { + "name": "genus" + } + }, + { + "name": "Species", + "rank": 220, + "column": "species", + "infullname": true, + "fields": { + "name": "species", + "author": "species author", + "source": "species source", + "commonname": "species common name" + } + }, + { + "name": "Subspecies", + "rank": 230, + "column": "subspecies", + "infullname": true, + "fields": { + "name": "subspecies", + "author": "subspecies author", + "source": "subspecies source", + "commonname": "subspecies common name" + } + } + ] +} \ No newline at end of file diff --git a/config/default_trees/mapping_files/geography.json b/config/default_trees/mapping_files/geography.json new file mode 100644 index 00000000000..6dc5528c6bb --- /dev/null +++ b/config/default_trees/mapping_files/geography.json @@ -0,0 +1,60 @@ +{ + "all_columns": [ + "Continent", + "Country", + "State", + "County", + "GeographyCode", + "CentroidLat", + "CentroidLon" + ], + "root": { + "name": "Earth" + }, + "ranks": [ + { + "name": "Continent", + "rank": 100, + "column": "Continent", + "fields": { + "name": "Continent", + "geographycode": "GeographyCode", + "centroidlat": "CentroidLat", + "centroidlon": "CentroidLon" + } + }, + { + "name": "Country", + "rank": 200, + "column": "Country", + "fields": { + "name": "Country", + "geographycode": "GeographyCode", + "centroidlat": "CentroidLat", + "centroidlon": "CentroidLon" + } + }, + { + "name": "State", + "rank": 300, + "column": "State", + "fields": { + "name": "State", + "geographycode": "GeographyCode", + "centroidlat": "CentroidLat", + "centroidlon": "CentroidLon" + } + }, + { + "name": "County", + "rank": 400, + "column": "County", + "fields": { + "name": "County", + "geographycode": "GeographyCode", + "centroidlat": "CentroidLat", + "centroidlon": "CentroidLon" + } + } + ] +} diff --git a/config/default_trees/mapping_files/geologictimeperiod.json b/config/default_trees/mapping_files/geologictimeperiod.json new file mode 100644 index 00000000000..a2a382ac47f --- /dev/null +++ b/config/default_trees/mapping_files/geologictimeperiod.json @@ -0,0 +1,66 @@ +{ + "all_columns": [ + "Erathem/Era", + "System/Period", + "Series/Epoch", + "Stage/Age", + "Start Period", + "Start Uncertainty", + "End Period", + "End Uncertainty" + ], + "root": { + "name": "Time", + "fullnameseparator": ", " + }, + "ranks": [ + { + "name": "Erathem/Era", + "rank": 100, + "column": "Erathem/Era", + "fields": { + "name": "Erathem/Era", + "startperiod": "Start Period", + "startuncertainty": "Start Uncertainty", + "endperiod": "End Period", + "enduncertainty": "End Uncertainty" + } + }, + { + "name": "System/Period", + "rank": 200, + "column": "System/Period", + "fields": { + "name": "System/Period", + "startperiod": "Start Period", + "startuncertainty": "Start Uncertainty", + "endperiod": "End Period", + "enduncertainty": "End Uncertainty" + } + }, + { + "name": "Series/Epoch", + "rank": 300, + "column": "Series/Epoch", + "fields": { + "name": "Series/Epoch", + "startperiod": "Start Period", + "startuncertainty": "Start Uncertainty", + "endperiod": "End Period", + "enduncertainty": "End Uncertainty" + } + }, + { + "name": "Stage/Age", + "rank": 400, + "column": "Stage/Age", + "fields": { + "name": "Stage/Age", + "startperiod": "Start Period", + "startuncertainty": "Start Uncertainty", + "endperiod": "End Period", + "enduncertainty": "End Uncertainty" + } + } + ] +} diff --git a/config/default_trees/mapping_files/insect.json b/config/default_trees/mapping_files/insect.json new file mode 100644 index 00000000000..6f7bf647f6e --- /dev/null +++ b/config/default_trees/mapping_files/insect.json @@ -0,0 +1,91 @@ +{ + "all_columns": ["kingdom","phylum","class","order","superfamily","family","genus","species","subspecies","species author","species source","species lsid","species common name","family common name","subspecies author","subspecies source","subspecies lsid","subspecies common name"], + "ranks": [ + { + "name": "Kingdom", + "rank": 10, + "column": "kingdom", + "fields": { + "name": "kingdom" + } + }, + { + "name": "Phylum", + "rank": 30, + "column": "phylum", + "fields": { + "name": "phylum" + } + }, + { + "name": "Class", + "rank": 60, + "column": "class", + "fields": { + "name": "class" + } + }, + { + "name": "Order", + "rank": 100, + "column": "order", + "fields": { + "name": "order" + } + }, + { + "name": "Superfamily", + "rank": 130, + "enforced": false, + "column": "superfamily", + "fields": { + "name": "superfamily" + } + }, + { + "name": "Family", + "rank": 140, + "column": "family", + "fields": { + "name": "family", + "commonname": "family common name" + } + }, + { + "name": "Genus", + "rank": 180, + "infullname": true, + "column": "genus", + "fields": { + "name": "genus" + } + }, + { + "name": "Species", + "rank": 220, + "infullname": true, + "column": "species", + "fields": { + "name": "species", + "author": "species author", + "source": "species source", + "lsid": "species lsid", + "commonname": "species common name" + } + }, + { + "name": "Subspecies", + "rank": 230, + "infullname": true, + "enforced": false, + "column": "subspecies", + "fields": { + "name": "subspecies", + "author": "subspecies author", + "source": "subspecies source", + "lsid": "subspecies lsid", + "commonname": "subspecies common name" + } + } + ] +} \ No newline at end of file diff --git a/config/default_trees/remote_taxonfiles.json b/config/default_trees/remote_taxonfiles.json new file mode 100644 index 00000000000..b7d2cefa910 --- /dev/null +++ b/config/default_trees/remote_taxonfiles.json @@ -0,0 +1,112 @@ +[ + { + "discipline": "fish", + "title": "Ichthyology", + "coverage": "", + "file": "https://specify-software-public.s3.us-east-1.amazonaws.com/default_trees/col2021_actinopterygii.csv", + "mappingFile": "https://specify-software-public.s3.us-east-1.amazonaws.com/default_trees/mapping_files/col2021_actinopterygii.json", + "src": "Catalog of Life 2021", + "size": 5821850, + "rows": 37959, + "description": "Actinopterygii: 488 families; 4908 genera; 32513 species." + }, + { + "discipline": "herpetology", + "title": "Herpetology", + "coverage": "", + "file": "https://specify-software-public.s3.us-east-1.amazonaws.com/default_trees/col2021_amphibia_reptilia.csv", + "mappingFile": "https://specify-software-public.s3.us-east-1.amazonaws.com/default_trees/mapping_files/col2021_amphibia_reptilia.json", + "src": "Catalog of Life 2021", + "size": 3986066, + "rows": 24731, + "description": "Amphibs: 75 families; 565 genera; 8054 species. Reptiles: 91 families; 1224 genera; 11570 species; 3140 subspecies." + }, + { + "discipline": "bird", + "title": "Aves", + "coverage": "", + "file": "https://specify-software-public.s3.us-east-1.amazonaws.com/default_trees/col2021_aves.csv", + "mappingFile": "https://specify-software-public.s3.us-east-1.amazonaws.com/default_trees/mapping_files/col2021_aves.json", + "src": "Catalog of Life 2021", + "size": 6723988, + "rows": 34039, + "description": "Aves: 230 families; 2254 genera; 10521 species; 20988 subspecies." + }, + { + "discipline": "insect", + "title": "Entomology", + "coverage": "", + "file": "https://specify-software-public.s3.us-east-1.amazonaws.com/default_trees/col2008_orthoptera.csv", + "mappingFile": "https://files.specifysoftware.org/treerows/col2008_orthoptera.json", + "src": "Catalog of Life 2008", + "size": 9502720, + "rows": 25604, + "description": "Order Orthoptera only: 40 families; 4320 genera; 23042 species." + }, + { + "discipline": "botany", + "title": "Botany (Bryophyta)", + "coverage": "", + "file": "https://specify-software-public.s3.us-east-1.amazonaws.com/default_trees/col2021_bryophyta.csv", + "mappingFile": "https://specify-software-public.s3.us-east-1.amazonaws.com/default_trees/mapping_files/col2021_bryophyta.json", + "src": "Catalog of Life 2021", + "size": 2219282, + "rows": 14426, + "description": "Phylum Bryophyta only: 96 families; 941 genera; 13362 species." + }, + { + "discipline": "botany", + "title": "Fungi", + "coverage": "", + "file": "https://specify-software-public.s3.us-east-1.amazonaws.com/default_trees/col2021_fungi.csv", + "mappingFile": "https://specify-software-public.s3.us-east-1.amazonaws.com/default_trees/mapping_files/col2021_fungi.json", + "src": "Catalog of Life 2021", + "size": 25417818, + "rows": 159229, + "description": "Fungi: 870 families; 11847 genera; 146154 species; 47 subspecies." + }, + { + "discipline": "mammal", + "title": "Mammalogy", + "coverage": "", + "file": "https://specify-software-public.s3.us-east-1.amazonaws.com/default_trees/col2021_mammalia.csv", + "mappingFile": "https://specify-software-public.s3.us-east-1.amazonaws.com/default_trees/mapping_files/col2021_mammalia.json", + "src": "Catalog of Life 2021", + "size": 2417890, + "rows": 13557, + "description": "146 families; 1151 genera; 4832 species." + }, + { + "discipline": "invertebrate", + "title": "Invertebrate Zoology", + "coverage": "", + "file": "https://specify-software-public.s3.us-east-1.amazonaws.com/default_trees/col2021_mollusca.csv", + "mappingFile": "https://specify-software-public.s3.us-east-1.amazonaws.com/default_trees/mapping_files/col2021_mollusca.json", + "src": "Catalog of Life 2021", + "size": 11614254, + "rows": 79277, + "description": "Phyla Mollusca: 1600 families; 14083 genera; 112144 species; 8580 subspecies." + }, + { + "discipline": "botany", + "title": "Polypodiopsida", + "coverage": "", + "file": "https://specify-software-public.s3.us-east-1.amazonaws.com/default_trees/col2021_polypodiopsida.csv", + "mappingFile": "https://specify-software-public.s3.us-east-1.amazonaws.com/default_trees/mapping_files/col2021_polypodiopsida.json", + "src": "Catalog of Life 2021", + "size": 2177153, + "rows": 13072, + "description": "Polypodiopsida: fern allies." + }, + { + "discipline": "botany", + "title": "Tracheophyta", + "coverage": "", + "file": "https://specify-software-public.s3.us-east-1.amazonaws.com/default_trees/col2021_tracheophyta.csv", + "mappingFile": "https://specify-software-public.s3.us-east-1.amazonaws.com/default_trees/mapping_files/col2021_tracheophyta.json", + "src": "Catalog of Life 2021", + "size": 62080445, + "rows": 395041, + "description": "Tracheophyta: vascular plants." + } +] diff --git a/config/default_trees/taxonfiles.json b/config/default_trees/taxonfiles.json new file mode 100644 index 00000000000..9d4f374acdd --- /dev/null +++ b/config/default_trees/taxonfiles.json @@ -0,0 +1,112 @@ +[ + { + "discipline": "fish", + "title": "Ichthyology", + "coverage": "", + "file": "/static/config/default_trees/col2021_actinopterygii.csv.gz", + "mappingFile": "/static/config/default_trees/mapping_files/col2021_actinopterygii.json", + "src": "Catalog of Life 2021", + "size": 5821850, + "rows": 37959, + "description": "Actinopterygii: 488 families; 4908 genera; 32513 species." + }, + { + "discipline": "herpetology", + "title": "Herpetology", + "coverage": "", + "file": "/static/config/default_trees/col2021_amphibia_reptilia.csv.gz", + "mappingFile": "/static/config/default_trees/mapping_files/col2021_amphibia_reptilia.json", + "src": "Catalog of Life 2021", + "size": 3986066, + "rows": 24731, + "description": "Amphibs: 75 families; 565 genera; 8054 species. Reptiles: 91 families; 1224 genera; 11570 species; 3140 subspecies." + }, + { + "discipline": "bird", + "title": "Aves", + "coverage": "", + "file": "/static/config/default_trees/col2021_aves.csv.gz", + "mappingFile": "/static/config/default_trees/mapping_files/col2021_aves.json", + "src": "Catalog of Life 2021", + "size": 6723988, + "rows": 34039, + "description": "Aves: 230 families; 2254 genera; 10521 species; 20988 subspecies." + }, + { + "discipline": "insect", + "title": "Entomology", + "coverage": "", + "file": "/static/config/default_trees/col2008_orthoptera.csv.gz", + "mappingFile": "/static/config/default_trees/mapping_files/insect.json", + "src": "Catalog of Life 2008", + "size": 9502720, + "rows": 25604, + "description": "Order Orthoptera only: 40 families; 4320 genera; 23042 species." + }, + { + "discipline": "botany", + "title": "Botany (Bryophyta)", + "coverage": "", + "file": "/static/config/default_trees/col2021_bryophyta.csv.gz", + "mappingFile": "/static/config/default_trees/mapping_files/col2021_bryophyta.json", + "src": "Catalog of Life 2021", + "size": 2219282, + "rows": 14426, + "description": "Phylum Bryophyta only: 96 families; 941 genera; 13362 species." + }, + { + "discipline": "botany", + "title": "Fungi", + "coverage": "", + "file": "/static/config/default_trees/col2021_fungi.csv.gz", + "mappingFile": "/static/config/default_trees/mapping_files/col2021_fungi.json", + "src": "Catalog of Life 2021", + "size": 25417818, + "rows": 159229, + "description": "Fungi: 870 families; 11847 genera; 146154 species; 47 subspecies." + }, + { + "discipline": "mammal", + "title": "Mammalogy", + "coverage": "", + "file": "/static/config/default_trees/col2021_mammalia.csv.gz", + "mappingFile": "/static/config/default_trees/mapping_files/col2021_mammalia.json", + "src": "Catalog of Life 2021", + "size": 2417890, + "rows": 13557, + "description": "146 families; 1151 genera; 4832 species." + }, + { + "discipline": "invertebrate", + "title": "Invertebrate Zoology", + "coverage": "", + "file": "/static/config/default_trees/col2021_mollusca.csv.gz", + "mappingFile": "/static/config/default_trees/mapping_files/col2021_mollusca.json", + "src": "Catalog of Life 2021", + "size": 11614254, + "rows": 79277, + "description": "Phyla Mollusca: 1600 families; 14083 genera; 112144 species; 8580 subspecies." + }, + { + "discipline": "botany", + "title": "Polypodiopsida", + "coverage": "", + "file": "/static/config/default_trees/col2021_polypodiopsida.csv.gz", + "mappingFile": "/static/config/default_trees/mapping_files/col2021_polypodiopsida.json", + "src": "Catalog of Life 2021", + "size": 2177153, + "rows": 13072, + "description": "Polypodiopsida: fern allies." + }, + { + "discipline": "botany", + "title": "Tracheophyta", + "coverage": "", + "file": "/static/config/default_trees/col2021_tracheophyta.csv.gz", + "mappingFile": "/static/config/default_trees/mapping_files/col2021_tracheophyta.json", + "src": "Catalog of Life 2021", + "size": 62080445, + "rows": 395041, + "description": "Tracheophyta: vascular plants." + } +] diff --git a/config/default_trees/test22_fishes.csv.gz b/config/default_trees/test22_fishes.csv.gz new file mode 100644 index 00000000000..c97151aa71b Binary files /dev/null and b/config/default_trees/test22_fishes.csv.gz differ diff --git a/config/default_trees/test_fishes.csv.gz b/config/default_trees/test_fishes.csv.gz new file mode 100644 index 00000000000..465daff55ab Binary files /dev/null and b/config/default_trees/test_fishes.csv.gz differ diff --git a/specifyweb/backend/setup_tool/tree_defaults.py b/specifyweb/backend/setup_tool/tree_defaults.py index 18b4d70166e..ae1a5f203eb 100644 --- a/specifyweb/backend/setup_tool/tree_defaults.py +++ b/specifyweb/backend/setup_tool/tree_defaults.py @@ -2,9 +2,9 @@ from typing import Type, Optional from pathlib import Path from uuid import uuid4 -import requests from .utils import load_json_from_file +from specifyweb.backend.trees.default_tree_files import load_default_tree_json from specifyweb.backend.trees.defaults import initialize_default_tree, create_default_tree_task, queue_create_default_tree_task from specifyweb.backend.trees.utils import TREE_NAMES @@ -19,14 +19,14 @@ 'Lithostrat': Path(__file__).parent.parent.parent.parent / 'config' / 'common' / 'lithostrat_tree.json', 'Tectonicunit': Path(__file__).parent.parent.parent.parent / 'config' / 'common' / 'tectonicunit_tree.json' } -DEFAULT_TAXON_TREE_LIST_URL = 'https://files.specifysoftware.org/taxonfiles/taxonfiles.json' +DEFAULT_TAXON_TREE_LIST_URL = '/static/config/default_trees/taxonfiles.json' DEFAULT_TREE_URLS = { - 'Geography': 'https://files.specifysoftware.org/geographyfiles/geonames.csv', - 'Geologictimeperiod': 'https://files.specifysoftware.org/chronostratfiles/GeologicTimePeriod.csv', + 'Geography': 'default_trees/geonames.csv.gz', + 'Geologictimeperiod': 'default_trees/GeologicTimePeriod.csv.gz', } DEFAULT_TREE_MAPPING_URLS = { - 'Geography': 'https://files.specifysoftware.org/treerows/geography.json', - 'Geologictimeperiod': 'https://files.specifysoftware.org/treerows/geologictimeperiod.json', + 'Geography': 'default_trees/mapping_files/geography.json', + 'Geologictimeperiod': 'default_trees/mapping_files/geologictimeperiod.json', } def start_default_tree_from_configuration(tree_type: str, kwargs: dict, user_rank_cfg: dict): @@ -113,9 +113,7 @@ def start_preload_default_tree(tree_type: str, discipline_id: Optional[int], col logger.warning(f'Can\'t preload tree, no default tree URLs for {tree_discipline_name} tree.') return - resp = requests.get(mapping_url) - resp.raise_for_status() - tree_cfg = resp.json() + tree_cfg = load_default_tree_json(mapping_url) task_id = str(uuid4()) async_result = create_default_tree_task.apply_async( @@ -134,4 +132,4 @@ def start_preload_default_tree(tree_type: str, discipline_id: Optional[int], col def update_tree_scoping(treedef: Type[DjangoModel], discipline_id: int): """Trees may be created before a discipline is created. This will update their discipline.""" setattr(treedef, "discipline_id", discipline_id) - treedef.save(update_fields=["discipline_id"]) \ No newline at end of file + treedef.save(update_fields=["discipline_id"]) diff --git a/specifyweb/backend/trees/default_tree_files.py b/specifyweb/backend/trees/default_tree_files.py new file mode 100644 index 00000000000..fbcfe775eec --- /dev/null +++ b/specifyweb/backend/trees/default_tree_files.py @@ -0,0 +1,177 @@ +import csv +import gzip +import json +import time +from pathlib import Path +from typing import Dict, Iterator, Optional +from urllib.parse import unquote, urlparse + +import requests +from django.conf import settings +from requests.exceptions import ChunkedEncodingError, ConnectionError + + +STATIC_CONFIG_PREFIX = '/static/config/' +DEFAULT_TREE_MAPPING_DIR = 'default_trees/mapping_files' + + +def _config_root() -> Path: + return Path(settings.SPECIFY_CONFIG_DIR).resolve() + + +def _resolve_config_path(relative_path: str) -> Optional[Path]: + config_root = _config_root() + candidate = (config_root / relative_path).resolve() + try: + candidate.relative_to(config_root) + except ValueError: + return None + if not candidate.is_file() and candidate.suffix == '.csv': + gz_candidate = candidate.with_name(f'{candidate.name}.gz') + if gz_candidate.is_file(): + return gz_candidate + return candidate + + +def _legacy_default_tree_relative_path(source: str) -> Optional[str]: + parsed = urlparse(source) + path = unquote(parsed.path) + + if path.startswith(STATIC_CONFIG_PREFIX): + return path[len(STATIC_CONFIG_PREFIX):] + + if parsed.netloc == 'specify-software-public.s3.us-east-1.amazonaws.com': + if path.startswith('/default_trees/'): + return path.lstrip('/') + return None + + if parsed.netloc != 'files.specifysoftware.org': + return None + + filename = Path(path).name + if path.startswith('/taxonfiles/'): + return f'default_trees/{filename}' + if path.startswith('/geographyfiles/'): + return f'default_trees/{filename}' + if path.startswith('/chronostratfiles/'): + return f'default_trees/{filename}' + if path.startswith('/treerows/'): + if filename == 'col2008_orthoptera.json': + filename = 'insect.json' + return f'{DEFAULT_TREE_MAPPING_DIR}/{filename}' + return None + + +def get_local_default_tree_path(source: str) -> Optional[Path]: + if not source: + return None + + parsed = urlparse(source) + parsed_path = unquote(parsed.path or source) + + if parsed.scheme in ('http', 'https'): + relative_path = _legacy_default_tree_relative_path(source) + if relative_path is None: + return None + return _resolve_config_path(relative_path) + + if parsed_path.startswith(STATIC_CONFIG_PREFIX): + return _resolve_config_path(parsed_path[len(STATIC_CONFIG_PREFIX):]) + + if parsed_path.startswith('config/'): + return _resolve_config_path(parsed_path[len('config/'):]) + + path = Path(parsed_path) + if path.is_absolute(): + resolved = path.resolve() + try: + resolved.relative_to(_config_root()) + except ValueError: + return None + return resolved + + return _resolve_config_path(parsed_path) + + +def load_default_tree_json(source: str): + local_path = get_local_default_tree_path(source) + if local_path is not None: + if not local_path.is_file(): + raise FileNotFoundError(f'Default tree JSON file not found: {local_path}') + with local_path.open('r', encoding='utf-8') as file_handle: + return json.load(file_handle) + + parsed = urlparse(source) + if parsed.scheme not in ('http', 'https'): + raise FileNotFoundError(f'Default tree JSON file not found: {source}') + + resp = requests.get(source, timeout=(5, 30)) + resp.raise_for_status() + return resp.json() + + +def stream_default_tree_csv(source: str) -> Iterator[Dict[str, str]]: + local_path = get_local_default_tree_path(source) + if local_path is not None: + if not local_path.is_file(): + raise FileNotFoundError(f'Default tree CSV file not found: {local_path}') + open_file = gzip.open if local_path.suffix == '.gz' else Path.open + with open_file(local_path, 'rt', encoding='utf-8-sig', newline='') as file_handle: + yield from csv.DictReader(file_handle) + return + + parsed = urlparse(source) + if parsed.scheme not in ('http', 'https'): + raise FileNotFoundError(f'Default tree CSV file not found: {source}') + + yield from _stream_remote_csv(source) + + +def _stream_remote_csv(source: str) -> Iterator[Dict[str, str]]: + chunk_size = 8192 + max_retries = 10 + + def lines_iter() -> Iterator[str]: + buffer = b"" + bytes_downloaded = 0 + retries = 0 + + headers = {} + while True: + if bytes_downloaded > 0: + headers['Range'] = f'bytes={bytes_downloaded}-' + + try: + with requests.get( + source, + stream=True, + timeout=(5, 30), + headers=headers, + ) as resp: + resp.raise_for_status() + for chunk in resp.iter_content(chunk_size=chunk_size): + chunk_length = len(chunk) + if chunk_length == 0: + continue + buffer += chunk + bytes_downloaded += chunk_length + + while True: + new_line_index = buffer.find(b'\n') + if new_line_index == -1: + break + line = buffer[: new_line_index + 1] + buffer = buffer[new_line_index + 1 :] + yield line.decode('utf-8-sig', errors='replace') + + if buffer: + yield buffer.decode('utf-8-sig', errors='replace') + return + except (ChunkedEncodingError, ConnectionError): + if retries < max_retries: + retries += 1 + time.sleep(2**retries) + continue + raise + + yield from csv.DictReader(lines_iter()) diff --git a/specifyweb/backend/trees/defaults.py b/specifyweb/backend/trees/defaults.py index b6482ed1ad1..a16c659a0d4 100644 --- a/specifyweb/backend/trees/defaults.py +++ b/specifyweb/backend/trees/defaults.py @@ -1,16 +1,12 @@ -from typing import Any, Callable, List, Dict, Iterator, Optional, TypedDict, NotRequired +from typing import Any, Dict, Optional, TypedDict, NotRequired import json -import requests -import csv -import time -from requests.exceptions import ChunkedEncodingError, ConnectionError from django.db import transaction -from django.conf import settings from specifyweb.backend.notifications.models import Message from specifyweb.celery_tasks import LogErrorsTask, app import specifyweb.specify.models as spmodels +from specifyweb.backend.trees.default_tree_files import stream_default_tree_csv from specifyweb.backend.trees.utils import get_models, TREE_ROOT_NODES from specifyweb.backend.trees.extras import renumber_tree, set_fullnames from specifyweb.backend.redis_cache.store import add_to_set, remove_from_set, set_members @@ -432,7 +428,7 @@ def progress(cur: int, additional_total: int=0) -> None: total_rows = row_count-2 progress(0, total_rows) - for row in stream_csv_from_url(url): + for row in stream_default_tree_csv(url): add_default_tree_record(context, row, tree_cfg, current) context.flush() progress(1, 0) @@ -468,59 +464,3 @@ def progress(cur: int, additional_total: int=0) -> None: ) finish_create_default_tree_task(f'create_default_tree_{tree_type}_{existing_tree_def_id or self.request.id}') - -def stream_csv_from_url(url: str) -> Iterator[Dict[str, str]]: - """ - Streams a taxon CSV from a URL. Yields each row. - """ - chunk_size = 8192 - max_retries = 10 - - def lines_iter() -> Iterator[str]: - # Streams data from the server in -chunks-, yields -lines-. - buffer = b"" - bytes_downloaded = 0 - retries = 0 - - headers = {} - while True: - # Request data starting from the last downloaded bytes - if bytes_downloaded > 0: - headers['Range'] = f'bytes={bytes_downloaded}-' - - try: - with requests.get(url, stream=True, timeout=(5, 30), headers=headers) as resp: - resp.raise_for_status() - for chunk in resp.iter_content(chunk_size=chunk_size): - chunk_length = len(chunk) - if chunk_length == 0: - continue - buffer += chunk - bytes_downloaded += chunk_length - - # Extract all lines from chunk - while True: - new_line_index = buffer.find(b'\n') - if new_line_index == -1: break - line = buffer[:new_line_index + 1] # extract line - buffer = buffer[new_line_index + 1 :] # clear read buffer - yield line.decode('utf-8-sig', errors='replace') - - if buffer: - # yield last line - yield buffer.decode('utf-8-sig', errors='replace') - return - except (ChunkedEncodingError, ConnectionError) as e: - # Trigger retry - if retries < max_retries: - retries += 1 - time.sleep(2 ** retries) - continue - raise - except Exception: - raise - - reader = csv.DictReader(lines_iter()) - - for row in reader: - yield row \ No newline at end of file diff --git a/specifyweb/backend/trees/views.py b/specifyweb/backend/trees/views.py index f636c869944..75e9bcf2141 100644 --- a/specifyweb/backend/trees/views.py +++ b/specifyweb/backend/trees/views.py @@ -1,13 +1,11 @@ -import csv from functools import wraps import json from uuid import uuid4 from django import http -from typing import Iterator, Literal, TypedDict, Any, Dict +from typing import Literal, TypedDict, Any from django.db import connection, transaction from django.http import HttpResponse from django.views.decorators.http import require_POST -import requests from specifyweb.backend.trees.tree_mutations import perm_target from specifyweb.specify.views import login_maybe_required, openapi from sqlalchemy import distinct @@ -29,6 +27,7 @@ from specifyweb.backend.notifications.models import Message from specifyweb.backend.trees.utils import get_search_filters +from specifyweb.backend.trees.default_tree_files import load_default_tree_json from specifyweb.backend.trees.defaults import create_default_tree_task, queue_create_default_tree_task, get_active_create_default_tree_tasks from specifyweb.specify.utils.field_change_info import FieldChangeInfo from specifyweb.backend.trees.ranks import tree_rank_count @@ -758,9 +757,7 @@ def create_default_tree_view(request): mapping_url = data.get('mappingUrl', None) if mapping_url: try: - resp = requests.get(mapping_url) - resp.raise_for_status() - tree_cfg = resp.json() + tree_cfg = load_default_tree_json(mapping_url) except Exception: return http.JsonResponse({'error': f'Could not retrieve default tree mapping from {mapping_url}.'}, status=404) try: @@ -917,12 +914,11 @@ def default_tree_mapping(request) -> http.HttpResponse: """Retrieves a default populated tree's mapping from a url""" # TODO: Reuse code from create_default_tree data = json.loads(request.body) + tree_cfg = data.get("mapping") mapping_url = data.get("mappingUrl") if mapping_url: try: - resp = requests.get(mapping_url) - resp.raise_for_status() - tree_cfg = resp.json() + tree_cfg = load_default_tree_json(mapping_url) except Exception: return http.JsonResponse({'error': f'Could not retrieve default tree mapping from {mapping_url}.'}, status=404) try: diff --git a/specifyweb/frontend/js_src/lib/components/Leaflet/layers.ts b/specifyweb/frontend/js_src/lib/components/Leaflet/layers.ts index 7b6fe4c3f70..2babf928e06 100644 --- a/specifyweb/frontend/js_src/lib/components/Leaflet/layers.ts +++ b/specifyweb/frontend/js_src/lib/components/Leaflet/layers.ts @@ -22,7 +22,7 @@ type SerializedLayer = { type Layers = RR<'baseMaps' | 'overlays', IR>; export const leafletLayersEndpoint = - 'https://files.specifysoftware.org/specify7/7.8.10/leaflet-layers.json'; + '/static/config/common/leaflet-layers.json'; /** * Optional filters to apply to a layer @@ -47,7 +47,7 @@ export const preferredOverlay = 'Labels and boundaries'; * leafletLayersEndpoint (defined above) * * User didn't define a resource file `leaflet-layers` * - * On any updates to this file, you should also update the one at + * On any updates to the checked-in fallback config, you should also update * leafletLayersEndpoint * * Documentation: diff --git a/specifyweb/frontend/js_src/lib/components/TreeView/CreateTree.tsx b/specifyweb/frontend/js_src/lib/components/TreeView/CreateTree.tsx index ca214d44a11..57cfc8fce7e 100644 --- a/specifyweb/frontend/js_src/lib/components/TreeView/CreateTree.tsx +++ b/specifyweb/frontend/js_src/lib/components/TreeView/CreateTree.tsx @@ -60,9 +60,7 @@ export type TreeCreationProgressInfo = { }; export async function fetchDefaultTrees(): Promise { - const response = await fetch( - 'https://files.specifysoftware.org/taxonfiles/taxonfiles.json' - ); + const response = await fetch('/static/config/default_trees/taxonfiles.json'); if (!response.ok) { throw new Error( `Failed to fetch default trees: ${response.status} ${response.statusText}`