diff --git a/api/main.py b/api/main.py index bf3ebef6..d7102151 100644 --- a/api/main.py +++ b/api/main.py @@ -12,13 +12,13 @@ from export_metrics import add_metrics from fastapi import FastAPI from fastapi import Request +from fastapi.middleware.cors import CORSMiddleware +from middleware.x_forwarded_headers import ForwardedHostAndPrefixMiddleware from openapi.collections_metadata import collections_metadata from openapi.openapi_metadata import openapi_metadata from routers import edr from routers import feature from utilities import create_url_from_request -from fastapi.middleware.cors import CORSMiddleware -from middleware.x_forwarded_headers import ForwardedHostAndPrefixMiddleware all_collections = collections_metadata.keys() diff --git a/api/routers/edr.py b/api/routers/edr.py index 164e63d2..72da8a43 100644 --- a/api/routers/edr.py +++ b/api/routers/edr.py @@ -5,11 +5,7 @@ from typing import Literal -import datastore_pb2 as dstore import formatters -from openapi import custom_dimension_examples -from openapi import openapi_examples -from openapi import edr_query_parameter_descriptions from covjson_pydantic.coverage import Coverage from covjson_pydantic.coverage import CoverageCollection from covjson_pydantic.parameter import Parameter @@ -22,9 +18,12 @@ from formatters.covjson import make_parameter from geojson_pydantic import Feature from geojson_pydantic import Point -from grpc_getter import get_obs_request from grpc_getter import get_locations_request +from grpc_getter import get_obs_request from grpc_getter import get_ts_ag_request +from openapi import custom_dimension_examples +from openapi import edr_query_parameter_descriptions +from openapi import openapi_examples from response_classes import CoverageJsonResponse from response_classes import GeoJsonResponse from shapely import geometry @@ -33,6 +32,8 @@ from utilities import add_request_parameters from utilities import validate_bbox +import datastore_pb2 as dstore + router = APIRouter(prefix="/collections/observations") response_fields_needed_for_data_api = [ diff --git a/datastore/data-loader/client_knmi_station_ingest.py b/datastore/data-loader/client_knmi_station_ingest.py index 02c0e775..8e51803f 100755 --- a/datastore/data-loader/client_knmi_station_ingest.py +++ b/datastore/data-loader/client_knmi_station_ingest.py @@ -2,19 +2,21 @@ # tested with Python 3.11 import math import os -import requests -from multiprocessing import cpu_count, Pool +from functools import partial +from multiprocessing import cpu_count +from multiprocessing import Pool from pathlib import Path from time import perf_counter from typing import List from typing import Tuple -from functools import partial import pandas as pd +import requests import xarray as xr from google.protobuf.timestamp_pb2 import Timestamp from parameters import knmi_parameter_names -from utilities import generate_parameter_name, convert_unit_names +from utilities import convert_unit_names +from utilities import generate_parameter_name def netcdf_file_to_requests(file_path: Path | str) -> Tuple[List, List]: @@ -153,6 +155,46 @@ def main(): create_requests_start = perf_counter() file_path = Path(Path(__file__).parent / "test-data" / "KNMI" / "20221231.nc") observation_request_messages = netcdf_file_to_requests(file_path=file_path) + observation_request_messages.append( + [ + { + "geometry": {"coordinates": {"lat": 52.098821802977, "lon": 5.1797058644882}, "type": "Point"}, + "links": [{"href": "Insert documentation about E-SOH datastore", "rel": "canonical"}], + "properties": { + "Conventions": "CF-1.8", + "content": { + "encoding": "utf-8", + "standard_name": "air_temperature", + "unit": "Cel", + "value": "123.456", + }, + "creator_email": "datacentrum@knmi.nl", + "creator_name": "KNMI", + "creator_type": "institution", + "creator_url": "http://data.knmi.nl", + "datetime": "2022-12-30T00:00:00Z", + "function": "root_mean_square", + "institution": "Royal Netherlands Meteorological Institute (KNMI)", + "instrument": "accordion", + "keywords": "temperature, pressure, relative humidity, visibility, wind speed, wind direction, " + "wind gust", + "keywords_vocabulary": "http://data.knmi.nl", + "level": "-9.34", + "license": "CC BY 4.0", + "naming_authority": "nl.knmi", + "parameter_name": "air_temperature:-9.34:point:P1Y", + "period": "P365D", + "platform": "0-20000-0-06260", + "platform_name": "DE BILT AWS", + "source": "Royal Netherlands Meteorological Institute (KNMI)", + "summary": "A test value for the tests.", + "title": "Test Value", + }, + "type": "Feature", + "version": "4.0", + } + ] + ) print("Finished creating the time series and observation requests " f"{perf_counter() - create_requests_start}.") insert_data( diff --git a/datastore/datastore/storagebackend/postgresql/getlocations.go b/datastore/datastore/storagebackend/postgresql/getlocations.go index b9ff677e..7a0c0341 100644 --- a/datastore/datastore/storagebackend/postgresql/getlocations.go +++ b/datastore/datastore/storagebackend/postgresql/getlocations.go @@ -36,21 +36,33 @@ func getLocs( // define and execute query query := fmt.Sprintf(` - SELECT DISTINCT ON (ts_id) - point, - platform, - platform_name, - parameter_name - FROM observation - JOIN time_series on observation.ts_id = time_series.id - JOIN geo_point ON observation.geo_point_id = geo_point.id - WHERE %s AND %s AND %s AND %s - ORDER BY ts_id, obstime_instant DESC; + WITH platforms AS ( + SELECT DISTINCT ON (platform) + platform + FROM time_series + JOIN observation ON observation.ts_id = time_series.id + JOIN geo_point ON observation.geo_point_id = geo_point.id + -- time AND geolocation AND integer parameters AND string parameters + WHERE %s AND %s AND %s AND %s + ORDER BY platform, obstime_instant DESC + ) + SELECT point, platform, platform_name, parameter_name FROM ( -- just to deal with final sorting + SELECT DISTINCT ON (ts_id) + point, platform, platform_name, parameter_name + FROM time_series + JOIN observation ON observation.ts_id = time_series.id + JOIN geo_point ON observation.geo_point_id = geo_point.id + WHERE platform in (SELECT platform from platforms) + AND %s + ORDER BY ts_id, platform, obstime_instant DESC + ) t ORDER BY platform, parameter_name; `, timeFilter, geoFilter, int64MdataFilter, - stringMdataFilter) + stringMdataFilter, + timeFilter, + ) rows, err := db.Query(query, phVals...) if err != nil { diff --git a/datastore/integration-test/response/data_locations_datetime_filters_where_only_one_parameter_has_data.json b/datastore/integration-test/response/data_locations_datetime_filters_where_only_one_parameter_has_data.json new file mode 100644 index 00000000..6d03b8e6 --- /dev/null +++ b/datastore/integration-test/response/data_locations_datetime_filters_where_only_one_parameter_has_data.json @@ -0,0 +1,53 @@ +{ + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 5.1797058644882, + 52.098821802977 + ] + }, + "properties": { + "name": "DE BILT AWS", + "detail": "https://oscar.wmo.int/surface/rest/api/search/station?wigosId=0-20000-0-06260", + "parameter-name": [ + "air_temperature:-9.34:root_mean_square:P365D" + ], + "timeseries-link": "http://localhost:8008/collections/observations/items?platform=0-20000-0-06260" + }, + "id": "0-20000-0-06260" + } + ], + "parameters": { + "air_temperature:-9.34:root_mean_square:P365D": { + "type": "Parameter", + "description": { + "en": "Air temperature at -9.34m, aggregated over P365D with method 'root_mean_square'" + }, + "observedProperty": { + "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", + "label": { + "en": "Air temperature" + } + }, + "unit": { + "label": { + "en": "Cel" + }, + "symbol": { + "value": "°C", + "type": "https://qudt.org/vocab/unit/DEG_C" + } + }, + "metocean:measurementType": { + "method": "root_mean_square", + "duration": "P365D" + }, + "metocean:standard_name": "air_temperature", + "metocean:level": -9.34 + } + } +} diff --git a/datastore/integration-test/response/data_locations_one_location_with_one_parameter.json b/datastore/integration-test/response/data_locations_one_location_with_one_parameter.json new file mode 100644 index 00000000..66480a26 --- /dev/null +++ b/datastore/integration-test/response/data_locations_one_location_with_one_parameter.json @@ -0,0 +1,893 @@ +{ + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 5.1797058644882, + 52.098821802977 + ] + }, + "properties": { + "name": "DE BILT AWS", + "detail": "https://oscar.wmo.int/surface/rest/api/search/station?wigosId=0-20000-0-06260", + "parameter-name": [ + "air_pressure_at_mean_sea_level:1.0:mean:PT1M", + "air_temperature:-9.34:root_mean_square:P365D", + "air_temperature:0.0:minimum:PT12H", + "air_temperature:0.0:minimum:PT14H", + "air_temperature:0.0:minimum:PT6H", + "air_temperature:0.1:minimum:PT10M", + "air_temperature:1.5:maximum:PT10M", + "air_temperature:1.5:minimum:PT10M", + "air_temperature:2.0:maximum:PT12H", + "air_temperature:2.0:maximum:PT24H", + "air_temperature:2.0:maximum:PT6H", + "air_temperature:2.0:mean:PT1M", + "air_temperature:2.0:minimum:PT12H", + "air_temperature:2.0:minimum:PT14H", + "air_temperature:2.0:minimum:PT6H", + "cloud_area_fraction:2.0:point:PT0S", + "cloud_base_altitude:2.0:point:PT0S", + "dew_point_temperature:1.5:mean:PT1M", + "duration_of_sunshine:2.0:point:PT0S", + "lwe_precipitation_rate:2.5:mean:PT10M", + "rainfall_amount:2.0:point:PT0S", + "rainfall_amount:2.0:point:PT12H", + "rainfall_amount:2.0:point:PT24H", + "rainfall_amount:2.0:point:PT6H", + "rainfall_rate:2.0:mean:PT10M", + "relative_humidity:2.0:mean:PT1M", + "surface_downwelling_shortwave_flux_in_air:2.0:mean:PT10M", + "visibility_in_air:2.5:mean:PT10M", + "wind_from_direction:2.0:mean:PT10M", + "wind_speed:10.0:mean:PT10M", + "wind_speed_of_gust:10.0:maximum:PT10M" + ], + "timeseries-link": "http://localhost:8008/collections/observations/items?platform=0-20000-0-06260" + }, + "id": "0-20000-0-06260" + } + ], + "parameters": { + "air_pressure_at_mean_sea_level:1.0:mean:PT1M": { + "type": "Parameter", + "description": { + "en": "Air pressure at mean sea level at 1.0m, aggregated over PT1M with method 'mean'" + }, + "observedProperty": { + "id": "https://vocab.nerc.ac.uk/standard_name/air_pressure_at_mean_sea_level", + "label": { + "en": "Air pressure at mean sea level" + } + }, + "unit": { + "label": { + "en": "hPa" + }, + "symbol": { + "value": "hPa", + "type": "https://qudt.org/vocab/unit/HectoPA" + } + }, + "metocean:measurementType": { + "method": "mean", + "duration": "PT1M" + }, + "metocean:standard_name": "air_pressure_at_mean_sea_level", + "metocean:level": 1.0 + }, + "air_temperature:-9.34:root_mean_square:P365D": { + "type": "Parameter", + "description": { + "en": "Air temperature at -9.34m, aggregated over P365D with method 'root_mean_square'" + }, + "observedProperty": { + "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", + "label": { + "en": "Air temperature" + } + }, + "unit": { + "label": { + "en": "Cel" + }, + "symbol": { + "value": "°C", + "type": "https://qudt.org/vocab/unit/DEG_C" + } + }, + "metocean:measurementType": { + "method": "root_mean_square", + "duration": "P365D" + }, + "metocean:standard_name": "air_temperature", + "metocean:level": -9.34 + }, + "air_temperature:0.0:minimum:PT12H": { + "type": "Parameter", + "description": { + "en": "Air temperature at 0.0m, aggregated over PT12H with method 'minimum'" + }, + "observedProperty": { + "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", + "label": { + "en": "Air temperature" + } + }, + "unit": { + "label": { + "en": "Cel" + }, + "symbol": { + "value": "°C", + "type": "https://qudt.org/vocab/unit/DEG_C" + } + }, + "metocean:measurementType": { + "method": "minimum", + "duration": "PT12H" + }, + "metocean:standard_name": "air_temperature", + "metocean:level": 0.0 + }, + "air_temperature:0.0:minimum:PT14H": { + "type": "Parameter", + "description": { + "en": "Air temperature at 0.0m, aggregated over PT14H with method 'minimum'" + }, + "observedProperty": { + "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", + "label": { + "en": "Air temperature" + } + }, + "unit": { + "label": { + "en": "Cel" + }, + "symbol": { + "value": "°C", + "type": "https://qudt.org/vocab/unit/DEG_C" + } + }, + "metocean:measurementType": { + "method": "minimum", + "duration": "PT14H" + }, + "metocean:standard_name": "air_temperature", + "metocean:level": 0.0 + }, + "air_temperature:0.0:minimum:PT6H": { + "type": "Parameter", + "description": { + "en": "Air temperature at 0.0m, aggregated over PT6H with method 'minimum'" + }, + "observedProperty": { + "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", + "label": { + "en": "Air temperature" + } + }, + "unit": { + "label": { + "en": "Cel" + }, + "symbol": { + "value": "°C", + "type": "https://qudt.org/vocab/unit/DEG_C" + } + }, + "metocean:measurementType": { + "method": "minimum", + "duration": "PT6H" + }, + "metocean:standard_name": "air_temperature", + "metocean:level": 0.0 + }, + "air_temperature:0.1:minimum:PT10M": { + "type": "Parameter", + "description": { + "en": "Air temperature at 0.1m, aggregated over PT10M with method 'minimum'" + }, + "observedProperty": { + "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", + "label": { + "en": "Air temperature" + } + }, + "unit": { + "label": { + "en": "Cel" + }, + "symbol": { + "value": "°C", + "type": "https://qudt.org/vocab/unit/DEG_C" + } + }, + "metocean:measurementType": { + "method": "minimum", + "duration": "PT10M" + }, + "metocean:standard_name": "air_temperature", + "metocean:level": 0.1 + }, + "air_temperature:1.5:maximum:PT10M": { + "type": "Parameter", + "description": { + "en": "Air temperature at 1.5m, aggregated over PT10M with method 'maximum'" + }, + "observedProperty": { + "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", + "label": { + "en": "Air temperature" + } + }, + "unit": { + "label": { + "en": "Cel" + }, + "symbol": { + "value": "°C", + "type": "https://qudt.org/vocab/unit/DEG_C" + } + }, + "metocean:measurementType": { + "method": "maximum", + "duration": "PT10M" + }, + "metocean:standard_name": "air_temperature", + "metocean:level": 1.5 + }, + "air_temperature:1.5:minimum:PT10M": { + "type": "Parameter", + "description": { + "en": "Air temperature at 1.5m, aggregated over PT10M with method 'minimum'" + }, + "observedProperty": { + "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", + "label": { + "en": "Air temperature" + } + }, + "unit": { + "label": { + "en": "Cel" + }, + "symbol": { + "value": "°C", + "type": "https://qudt.org/vocab/unit/DEG_C" + } + }, + "metocean:measurementType": { + "method": "minimum", + "duration": "PT10M" + }, + "metocean:standard_name": "air_temperature", + "metocean:level": 1.5 + }, + "air_temperature:2.0:maximum:PT12H": { + "type": "Parameter", + "description": { + "en": "Air temperature at 2.0m, aggregated over PT12H with method 'maximum'" + }, + "observedProperty": { + "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", + "label": { + "en": "Air temperature" + } + }, + "unit": { + "label": { + "en": "Cel" + }, + "symbol": { + "value": "°C", + "type": "https://qudt.org/vocab/unit/DEG_C" + } + }, + "metocean:measurementType": { + "method": "maximum", + "duration": "PT12H" + }, + "metocean:standard_name": "air_temperature", + "metocean:level": 2.0 + }, + "air_temperature:2.0:maximum:PT24H": { + "type": "Parameter", + "description": { + "en": "Air temperature at 2.0m, aggregated over PT24H with method 'maximum'" + }, + "observedProperty": { + "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", + "label": { + "en": "Air temperature" + } + }, + "unit": { + "label": { + "en": "Cel" + }, + "symbol": { + "value": "°C", + "type": "https://qudt.org/vocab/unit/DEG_C" + } + }, + "metocean:measurementType": { + "method": "maximum", + "duration": "PT24H" + }, + "metocean:standard_name": "air_temperature", + "metocean:level": 2.0 + }, + "air_temperature:2.0:maximum:PT6H": { + "type": "Parameter", + "description": { + "en": "Air temperature at 2.0m, aggregated over PT6H with method 'maximum'" + }, + "observedProperty": { + "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", + "label": { + "en": "Air temperature" + } + }, + "unit": { + "label": { + "en": "Cel" + }, + "symbol": { + "value": "°C", + "type": "https://qudt.org/vocab/unit/DEG_C" + } + }, + "metocean:measurementType": { + "method": "maximum", + "duration": "PT6H" + }, + "metocean:standard_name": "air_temperature", + "metocean:level": 2.0 + }, + "air_temperature:2.0:mean:PT1M": { + "type": "Parameter", + "description": { + "en": "Air temperature at 2.0m, aggregated over PT1M with method 'mean'" + }, + "observedProperty": { + "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", + "label": { + "en": "Air temperature" + } + }, + "unit": { + "label": { + "en": "Cel" + }, + "symbol": { + "value": "°C", + "type": "https://qudt.org/vocab/unit/DEG_C" + } + }, + "metocean:measurementType": { + "method": "mean", + "duration": "PT1M" + }, + "metocean:standard_name": "air_temperature", + "metocean:level": 2.0 + }, + "air_temperature:2.0:minimum:PT12H": { + "type": "Parameter", + "description": { + "en": "Air temperature at 2.0m, aggregated over PT12H with method 'minimum'" + }, + "observedProperty": { + "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", + "label": { + "en": "Air temperature" + } + }, + "unit": { + "label": { + "en": "Cel" + }, + "symbol": { + "value": "°C", + "type": "https://qudt.org/vocab/unit/DEG_C" + } + }, + "metocean:measurementType": { + "method": "minimum", + "duration": "PT12H" + }, + "metocean:standard_name": "air_temperature", + "metocean:level": 2.0 + }, + "air_temperature:2.0:minimum:PT14H": { + "type": "Parameter", + "description": { + "en": "Air temperature at 2.0m, aggregated over PT14H with method 'minimum'" + }, + "observedProperty": { + "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", + "label": { + "en": "Air temperature" + } + }, + "unit": { + "label": { + "en": "Cel" + }, + "symbol": { + "value": "°C", + "type": "https://qudt.org/vocab/unit/DEG_C" + } + }, + "metocean:measurementType": { + "method": "minimum", + "duration": "PT14H" + }, + "metocean:standard_name": "air_temperature", + "metocean:level": 2.0 + }, + "air_temperature:2.0:minimum:PT6H": { + "type": "Parameter", + "description": { + "en": "Air temperature at 2.0m, aggregated over PT6H with method 'minimum'" + }, + "observedProperty": { + "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", + "label": { + "en": "Air temperature" + } + }, + "unit": { + "label": { + "en": "Cel" + }, + "symbol": { + "value": "°C", + "type": "https://qudt.org/vocab/unit/DEG_C" + } + }, + "metocean:measurementType": { + "method": "minimum", + "duration": "PT6H" + }, + "metocean:standard_name": "air_temperature", + "metocean:level": 2.0 + }, + "cloud_area_fraction:2.0:point:PT0S": { + "type": "Parameter", + "description": { + "en": "Cloud area fraction at 2.0m, aggregated over PT0S with method 'point'" + }, + "observedProperty": { + "id": "https://vocab.nerc.ac.uk/standard_name/cloud_area_fraction", + "label": { + "en": "Cloud area fraction" + } + }, + "unit": { + "label": { + "en": "okta" + }, + "symbol": { + "value": "okta", + "type": "https://qudt.org/vocab/unit/OKTA" + } + }, + "metocean:measurementType": { + "method": "point", + "duration": "PT0S" + }, + "metocean:standard_name": "cloud_area_fraction", + "metocean:level": 2.0 + }, + "cloud_base_altitude:2.0:point:PT0S": { + "type": "Parameter", + "description": { + "en": "Cloud base altitude at 2.0m, aggregated over PT0S with method 'point'" + }, + "observedProperty": { + "id": "https://vocab.nerc.ac.uk/standard_name/cloud_base_altitude", + "label": { + "en": "Cloud base altitude" + } + }, + "unit": { + "label": { + "en": "m" + }, + "symbol": { + "value": "m", + "type": "https://qudt.org/vocab/unit/M" + } + }, + "metocean:measurementType": { + "method": "point", + "duration": "PT0S" + }, + "metocean:standard_name": "cloud_base_altitude", + "metocean:level": 2.0 + }, + "dew_point_temperature:1.5:mean:PT1M": { + "type": "Parameter", + "description": { + "en": "Dew point temperature at 1.5m, aggregated over PT1M with method 'mean'" + }, + "observedProperty": { + "id": "https://vocab.nerc.ac.uk/standard_name/dew_point_temperature", + "label": { + "en": "Dew point temperature" + } + }, + "unit": { + "label": { + "en": "Cel" + }, + "symbol": { + "value": "°C", + "type": "https://qudt.org/vocab/unit/DEG_C" + } + }, + "metocean:measurementType": { + "method": "mean", + "duration": "PT1M" + }, + "metocean:standard_name": "dew_point_temperature", + "metocean:level": 1.5 + }, + "duration_of_sunshine:2.0:point:PT0S": { + "type": "Parameter", + "description": { + "en": "Duration of sunshine at 2.0m, aggregated over PT0S with method 'point'" + }, + "observedProperty": { + "id": "https://vocab.nerc.ac.uk/standard_name/duration_of_sunshine", + "label": { + "en": "Duration of sunshine" + } + }, + "unit": { + "label": { + "en": "s" + }, + "symbol": { + "value": "s", + "type": "https://qudt.org/vocab/unit/SEC" + } + }, + "metocean:measurementType": { + "method": "point", + "duration": "PT0S" + }, + "metocean:standard_name": "duration_of_sunshine", + "metocean:level": 2.0 + }, + "lwe_precipitation_rate:2.5:mean:PT10M": { + "type": "Parameter", + "description": { + "en": "Lwe precipitation rate at 2.5m, aggregated over PT10M with method 'mean'" + }, + "observedProperty": { + "id": "https://vocab.nerc.ac.uk/standard_name/lwe_precipitation_rate", + "label": { + "en": "Lwe precipitation rate" + } + }, + "unit": { + "label": { + "en": "mm/h" + }, + "symbol": { + "value": "mm/h", + "type": "https://qudt.org/vocab/unit/MilliM-PER-HR" + } + }, + "metocean:measurementType": { + "method": "mean", + "duration": "PT10M" + }, + "metocean:standard_name": "lwe_precipitation_rate", + "metocean:level": 2.5 + }, + "rainfall_amount:2.0:point:PT0S": { + "type": "Parameter", + "description": { + "en": "Rainfall amount at 2.0m, aggregated over PT0S with method 'point'" + }, + "observedProperty": { + "id": "https://vocab.nerc.ac.uk/standard_name/rainfall_amount", + "label": { + "en": "Rainfall amount" + } + }, + "unit": { + "label": { + "en": "kg/m2" + }, + "symbol": { + "value": "kg/m²", + "type": "https://qudt.org/vocab/unit/KiloGM-PER-M2" + } + }, + "metocean:measurementType": { + "method": "point", + "duration": "PT0S" + }, + "metocean:standard_name": "rainfall_amount", + "metocean:level": 2.0 + }, + "rainfall_amount:2.0:point:PT12H": { + "type": "Parameter", + "description": { + "en": "Rainfall amount at 2.0m, aggregated over PT12H with method 'point'" + }, + "observedProperty": { + "id": "https://vocab.nerc.ac.uk/standard_name/rainfall_amount", + "label": { + "en": "Rainfall amount" + } + }, + "unit": { + "label": { + "en": "kg/m2" + }, + "symbol": { + "value": "kg/m²", + "type": "https://qudt.org/vocab/unit/KiloGM-PER-M2" + } + }, + "metocean:measurementType": { + "method": "point", + "duration": "PT12H" + }, + "metocean:standard_name": "rainfall_amount", + "metocean:level": 2.0 + }, + "rainfall_amount:2.0:point:PT24H": { + "type": "Parameter", + "description": { + "en": "Rainfall amount at 2.0m, aggregated over PT24H with method 'point'" + }, + "observedProperty": { + "id": "https://vocab.nerc.ac.uk/standard_name/rainfall_amount", + "label": { + "en": "Rainfall amount" + } + }, + "unit": { + "label": { + "en": "kg/m2" + }, + "symbol": { + "value": "kg/m²", + "type": "https://qudt.org/vocab/unit/KiloGM-PER-M2" + } + }, + "metocean:measurementType": { + "method": "point", + "duration": "PT24H" + }, + "metocean:standard_name": "rainfall_amount", + "metocean:level": 2.0 + }, + "rainfall_amount:2.0:point:PT6H": { + "type": "Parameter", + "description": { + "en": "Rainfall amount at 2.0m, aggregated over PT6H with method 'point'" + }, + "observedProperty": { + "id": "https://vocab.nerc.ac.uk/standard_name/rainfall_amount", + "label": { + "en": "Rainfall amount" + } + }, + "unit": { + "label": { + "en": "kg/m2" + }, + "symbol": { + "value": "kg/m²", + "type": "https://qudt.org/vocab/unit/KiloGM-PER-M2" + } + }, + "metocean:measurementType": { + "method": "point", + "duration": "PT6H" + }, + "metocean:standard_name": "rainfall_amount", + "metocean:level": 2.0 + }, + "rainfall_rate:2.0:mean:PT10M": { + "type": "Parameter", + "description": { + "en": "Rainfall rate at 2.0m, aggregated over PT10M with method 'mean'" + }, + "observedProperty": { + "id": "https://vocab.nerc.ac.uk/standard_name/rainfall_rate", + "label": { + "en": "Rainfall rate" + } + }, + "unit": { + "label": { + "en": "mm/h" + }, + "symbol": { + "value": "mm/h", + "type": "https://qudt.org/vocab/unit/MilliM-PER-HR" + } + }, + "metocean:measurementType": { + "method": "mean", + "duration": "PT10M" + }, + "metocean:standard_name": "rainfall_rate", + "metocean:level": 2.0 + }, + "relative_humidity:2.0:mean:PT1M": { + "type": "Parameter", + "description": { + "en": "Relative humidity at 2.0m, aggregated over PT1M with method 'mean'" + }, + "observedProperty": { + "id": "https://vocab.nerc.ac.uk/standard_name/relative_humidity", + "label": { + "en": "Relative humidity" + } + }, + "unit": { + "label": { + "en": "%" + }, + "symbol": { + "value": "%", + "type": "https://qudt.org/vocab/unit/PERCENT" + } + }, + "metocean:measurementType": { + "method": "mean", + "duration": "PT1M" + }, + "metocean:standard_name": "relative_humidity", + "metocean:level": 2.0 + }, + "surface_downwelling_shortwave_flux_in_air:2.0:mean:PT10M": { + "type": "Parameter", + "description": { + "en": "Surface downwelling shortwave flux in air at 2.0m, aggregated over PT10M with method 'mean'" + }, + "observedProperty": { + "id": "https://vocab.nerc.ac.uk/standard_name/surface_downwelling_shortwave_flux_in_air", + "label": { + "en": "Surface downwelling shortwave flux in air" + } + }, + "unit": { + "label": { + "en": "W/m2" + }, + "symbol": { + "value": "W/m²", + "type": "https://qudt.org/vocab/unit/W-PER-M2" + } + }, + "metocean:measurementType": { + "method": "mean", + "duration": "PT10M" + }, + "metocean:standard_name": "surface_downwelling_shortwave_flux_in_air", + "metocean:level": 2.0 + }, + "visibility_in_air:2.5:mean:PT10M": { + "type": "Parameter", + "description": { + "en": "Visibility in air at 2.5m, aggregated over PT10M with method 'mean'" + }, + "observedProperty": { + "id": "https://vocab.nerc.ac.uk/standard_name/visibility_in_air", + "label": { + "en": "Visibility in air" + } + }, + "unit": { + "label": { + "en": "m" + }, + "symbol": { + "value": "m", + "type": "https://qudt.org/vocab/unit/M" + } + }, + "metocean:measurementType": { + "method": "mean", + "duration": "PT10M" + }, + "metocean:standard_name": "visibility_in_air", + "metocean:level": 2.5 + }, + "wind_from_direction:2.0:mean:PT10M": { + "type": "Parameter", + "description": { + "en": "Wind from direction at 2.0m, aggregated over PT10M with method 'mean'" + }, + "observedProperty": { + "id": "https://vocab.nerc.ac.uk/standard_name/wind_from_direction", + "label": { + "en": "Wind from direction" + } + }, + "unit": { + "label": { + "en": "deg" + }, + "symbol": { + "value": "°", + "type": "https://qudt.org/vocab/unit/DEG" + } + }, + "metocean:measurementType": { + "method": "mean", + "duration": "PT10M" + }, + "metocean:standard_name": "wind_from_direction", + "metocean:level": 2.0 + }, + "wind_speed:10.0:mean:PT10M": { + "type": "Parameter", + "description": { + "en": "Wind speed at 10.0m, aggregated over PT10M with method 'mean'" + }, + "observedProperty": { + "id": "https://vocab.nerc.ac.uk/standard_name/wind_speed", + "label": { + "en": "Wind speed" + } + }, + "unit": { + "label": { + "en": "m/s" + }, + "symbol": { + "value": "m/s", + "type": "https://qudt.org/vocab/unit/M-PER-SEC" + } + }, + "metocean:measurementType": { + "method": "mean", + "duration": "PT10M" + }, + "metocean:standard_name": "wind_speed", + "metocean:level": 10.0 + }, + "wind_speed_of_gust:10.0:maximum:PT10M": { + "type": "Parameter", + "description": { + "en": "Wind speed of gust at 10.0m, aggregated over PT10M with method 'maximum'" + }, + "observedProperty": { + "id": "https://vocab.nerc.ac.uk/standard_name/wind_speed_of_gust", + "label": { + "en": "Wind speed of gust" + } + }, + "unit": { + "label": { + "en": "m/s" + }, + "symbol": { + "value": "m/s", + "type": "https://qudt.org/vocab/unit/M-PER-SEC" + } + }, + "metocean:measurementType": { + "method": "maximum", + "duration": "PT10M" + }, + "metocean:standard_name": "wind_speed_of_gust", + "metocean:level": 10.0 + } + } +} diff --git a/datastore/integration-test/response/data_locations_two_points_with_five_parameters.json b/datastore/integration-test/response/data_locations_two_points_with_five_parameters.json deleted file mode 100644 index 5e128e39..00000000 --- a/datastore/integration-test/response/data_locations_two_points_with_five_parameters.json +++ /dev/null @@ -1,188 +0,0 @@ -{ - "type": "FeatureCollection", - "features": [ - { - "type": "Feature", - "geometry": { - "type": "Point", - "coordinates": [ - 5.1797058644882, - 52.098821802977 - ] - }, - "properties": { - "name": "DE BILT AWS", - "detail": "https://oscar.wmo.int/surface/rest/api/search/station?wigosId=0-20000-0-06260", - "parameter-name": [ - "air_temperature:0.1:minimum:PT10M", - "air_temperature:1.5:maximum:PT10M", - "air_temperature:1.5:minimum:PT10M", - "air_temperature:2.0:mean:PT1M", - "wind_from_direction:2.0:mean:PT10M" - ], - "timeseries-link": "http://localhost:8008/collections/observations/items?platform=0-20000-0-06260" - }, - "id": "0-20000-0-06260" - }, - { - "type": "Feature", - "geometry": { - "type": "Point", - "coordinates": [ - 5.8723225499118, - 52.0548617826 - ] - }, - "properties": { - "name": "DEELEN", - "detail": "https://oscar.wmo.int/surface/rest/api/search/station?wigosId=0-20000-0-06275", - "parameter-name": [ - "air_temperature:0.1:minimum:PT10M", - "air_temperature:1.5:maximum:PT10M", - "air_temperature:1.5:minimum:PT10M", - "air_temperature:2.0:mean:PT1M", - "wind_from_direction:2.0:mean:PT10M" - ], - "timeseries-link": "http://localhost:8008/collections/observations/items?platform=0-20000-0-06275" - }, - "id": "0-20000-0-06275" - } - ], - "parameters": { - "air_temperature:0.1:minimum:PT10M": { - "type": "Parameter", - "description": { - "en": "Air temperature at 0.1m, aggregated over PT10M with method 'minimum'" - }, - "observedProperty": { - "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", - "label": { - "en": "Air temperature" - } - }, - "unit": { - "label": { - "en": "Cel" - }, - "symbol": { - "type": "https://qudt.org/vocab/unit/DEG_C", - "value": "°C" - } - }, - "metocean:measurementType": { - "method": "minimum", - "duration": "PT10M" - }, - "metocean:standard_name": "air_temperature", - "metocean:level": 0.1 - }, - "air_temperature:1.5:maximum:PT10M": { - "type": "Parameter", - "description": { - "en": "Air temperature at 1.5m, aggregated over PT10M with method 'maximum'" - }, - "observedProperty": { - "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", - "label": { - "en": "Air temperature" - } - }, - "unit": { - "label": { - "en": "Cel" - }, - "symbol": { - "type": "https://qudt.org/vocab/unit/DEG_C", - "value": "°C" - } - }, - "metocean:measurementType": { - "method": "maximum", - "duration": "PT10M" - }, - "metocean:standard_name": "air_temperature", - "metocean:level": 1.5 - }, - "air_temperature:1.5:minimum:PT10M": { - "type": "Parameter", - "description": { - "en": "Air temperature at 1.5m, aggregated over PT10M with method 'minimum'" - }, - "observedProperty": { - "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", - "label": { - "en": "Air temperature" - } - }, - "unit": { - "label": { - "en": "Cel" - }, - "symbol": { - "type": "https://qudt.org/vocab/unit/DEG_C", - "value": "°C" - } - }, - "metocean:measurementType": { - "method": "minimum", - "duration": "PT10M" - }, - "metocean:standard_name": "air_temperature", - "metocean:level": 1.5 - }, - "air_temperature:2.0:mean:PT1M": { - "type": "Parameter", - "description": { - "en": "Air temperature at 2.0m, aggregated over PT1M with method 'mean'" - }, - "observedProperty": { - "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", - "label": { - "en": "Air temperature" - } - }, - "unit": { - "label": { - "en": "Cel" - }, - "symbol": { - "type": "https://qudt.org/vocab/unit/DEG_C", - "value": "°C" - } - }, - "metocean:measurementType": { - "method": "mean", - "duration": "PT1M" - }, - "metocean:standard_name": "air_temperature", - "metocean:level": 2.0 - }, - "wind_from_direction:2.0:mean:PT10M": { - "type": "Parameter", - "description": { - "en": "Wind from direction at 2.0m, aggregated over PT10M with method 'mean'" - }, - "observedProperty": { - "id": "https://vocab.nerc.ac.uk/standard_name/wind_from_direction", - "label": { - "en": "Wind from direction" - } - }, - "unit": { - "label": { - "en": "deg" - }, - "symbol": { - "type": "https://qudt.org/vocab/unit/DEG", - "value": "°" - } - }, - "metocean:measurementType": { - "method": "mean", - "duration": "PT10M" - }, - "metocean:standard_name": "wind_from_direction", - "metocean:level": 2.0 - } - } -} diff --git a/datastore/integration-test/response/data_locations_two_points_with_multiple_parameters.json b/datastore/integration-test/response/data_locations_two_points_with_multiple_parameters.json index f7f1915e..6a938457 100644 --- a/datastore/integration-test/response/data_locations_two_points_with_multiple_parameters.json +++ b/datastore/integration-test/response/data_locations_two_points_with_multiple_parameters.json @@ -6,13 +6,13 @@ "geometry": { "type": "Point", "coordinates": [ - 5.1797058644882, - 52.098821802977 + 4.436389, + 52.139722 ] }, "properties": { - "name": "DE BILT AWS", - "detail": "https://oscar.wmo.int/surface/rest/api/search/station?wigosId=0-20000-0-06260", + "name": "VOORSCHOTEN AWS", + "detail": "https://oscar.wmo.int/surface/rest/api/search/station?wigosId=0-20000-0-06215", "parameter-name": [ "air_pressure_at_mean_sea_level:1.0:mean:PT1M", "air_temperature:0.0:minimum:PT12H", @@ -45,22 +45,22 @@ "wind_speed:10.0:mean:PT10M", "wind_speed_of_gust:10.0:maximum:PT10M" ], - "timeseries-link": "http://localhost:8008/collections/observations/items?platform=0-20000-0-06260" + "timeseries-link": "http://localhost:8008/collections/observations/items?platform=0-20000-0-06215" }, - "id": "0-20000-0-06260" + "id": "0-20000-0-06215" }, { "type": "Feature", "geometry": { "type": "Point", "coordinates": [ - 5.8723225499118, - 52.0548617826 + 5.1797058644882, + 52.098821802977 ] }, "properties": { - "name": "DEELEN", - "detail": "https://oscar.wmo.int/surface/rest/api/search/station?wigosId=0-20000-0-06275", + "name": "DE BILT AWS", + "detail": "https://oscar.wmo.int/surface/rest/api/search/station?wigosId=0-20000-0-06260", "parameter-name": [ "air_pressure_at_mean_sea_level:1.0:mean:PT1M", "air_temperature:0.0:minimum:PT12H", @@ -80,7 +80,7 @@ "cloud_base_altitude:2.0:point:PT0S", "dew_point_temperature:1.5:mean:PT1M", "duration_of_sunshine:2.0:point:PT0S", - "lwe_precipitation_rate:1.8:mean:PT10M", + "lwe_precipitation_rate:2.5:mean:PT10M", "rainfall_amount:2.0:point:PT0S", "rainfall_amount:2.0:point:PT12H", "rainfall_amount:2.0:point:PT24H", @@ -88,14 +88,14 @@ "rainfall_rate:2.0:mean:PT10M", "relative_humidity:2.0:mean:PT1M", "surface_downwelling_shortwave_flux_in_air:2.0:mean:PT10M", - "visibility_in_air:1.8:mean:PT10M", + "visibility_in_air:2.5:mean:PT10M", "wind_from_direction:2.0:mean:PT10M", "wind_speed:10.0:mean:PT10M", "wind_speed_of_gust:10.0:maximum:PT10M" ], - "timeseries-link": "http://localhost:8008/collections/observations/items?platform=0-20000-0-06275" + "timeseries-link": "http://localhost:8008/collections/observations/items?platform=0-20000-0-06260" }, - "id": "0-20000-0-06275" + "id": "0-20000-0-06260" } ], "parameters": { @@ -115,8 +115,8 @@ "en": "hPa" }, "symbol": { - "type": "https://qudt.org/vocab/unit/HectoPA", - "value": "hPa" + "value": "hPa", + "type": "https://qudt.org/vocab/unit/HectoPA" } }, "metocean:measurementType": { @@ -142,8 +142,8 @@ "en": "Cel" }, "symbol": { - "type": "https://qudt.org/vocab/unit/DEG_C", - "value": "°C" + "value": "°C", + "type": "https://qudt.org/vocab/unit/DEG_C" } }, "metocean:measurementType": { @@ -169,8 +169,8 @@ "en": "Cel" }, "symbol": { - "type": "https://qudt.org/vocab/unit/DEG_C", - "value": "°C" + "value": "°C", + "type": "https://qudt.org/vocab/unit/DEG_C" } }, "metocean:measurementType": { @@ -196,8 +196,8 @@ "en": "Cel" }, "symbol": { - "type": "https://qudt.org/vocab/unit/DEG_C", - "value": "°C" + "value": "°C", + "type": "https://qudt.org/vocab/unit/DEG_C" } }, "metocean:measurementType": { @@ -223,8 +223,8 @@ "en": "Cel" }, "symbol": { - "type": "https://qudt.org/vocab/unit/DEG_C", - "value": "°C" + "value": "°C", + "type": "https://qudt.org/vocab/unit/DEG_C" } }, "metocean:measurementType": { @@ -250,8 +250,8 @@ "en": "Cel" }, "symbol": { - "type": "https://qudt.org/vocab/unit/DEG_C", - "value": "°C" + "value": "°C", + "type": "https://qudt.org/vocab/unit/DEG_C" } }, "metocean:measurementType": { @@ -277,8 +277,8 @@ "en": "Cel" }, "symbol": { - "type": "https://qudt.org/vocab/unit/DEG_C", - "value": "°C" + "value": "°C", + "type": "https://qudt.org/vocab/unit/DEG_C" } }, "metocean:measurementType": { @@ -304,8 +304,8 @@ "en": "Cel" }, "symbol": { - "type": "https://qudt.org/vocab/unit/DEG_C", - "value": "°C" + "value": "°C", + "type": "https://qudt.org/vocab/unit/DEG_C" } }, "metocean:measurementType": { @@ -331,8 +331,8 @@ "en": "Cel" }, "symbol": { - "type": "https://qudt.org/vocab/unit/DEG_C", - "value": "°C" + "value": "°C", + "type": "https://qudt.org/vocab/unit/DEG_C" } }, "metocean:measurementType": { @@ -358,8 +358,8 @@ "en": "Cel" }, "symbol": { - "type": "https://qudt.org/vocab/unit/DEG_C", - "value": "°C" + "value": "°C", + "type": "https://qudt.org/vocab/unit/DEG_C" } }, "metocean:measurementType": { @@ -385,8 +385,8 @@ "en": "Cel" }, "symbol": { - "type": "https://qudt.org/vocab/unit/DEG_C", - "value": "°C" + "value": "°C", + "type": "https://qudt.org/vocab/unit/DEG_C" } }, "metocean:measurementType": { @@ -412,8 +412,8 @@ "en": "Cel" }, "symbol": { - "type": "https://qudt.org/vocab/unit/DEG_C", - "value": "°C" + "value": "°C", + "type": "https://qudt.org/vocab/unit/DEG_C" } }, "metocean:measurementType": { @@ -439,8 +439,8 @@ "en": "Cel" }, "symbol": { - "type": "https://qudt.org/vocab/unit/DEG_C", - "value": "°C" + "value": "°C", + "type": "https://qudt.org/vocab/unit/DEG_C" } }, "metocean:measurementType": { @@ -466,8 +466,8 @@ "en": "Cel" }, "symbol": { - "type": "https://qudt.org/vocab/unit/DEG_C", - "value": "°C" + "value": "°C", + "type": "https://qudt.org/vocab/unit/DEG_C" } }, "metocean:measurementType": { @@ -493,8 +493,8 @@ "en": "okta" }, "symbol": { - "type": "https://qudt.org/vocab/unit/OKTA", - "value": "okta" + "value": "okta", + "type": "https://qudt.org/vocab/unit/OKTA" } }, "metocean:measurementType": { @@ -520,8 +520,8 @@ "en": "m" }, "symbol": { - "type": "https://qudt.org/vocab/unit/M", - "value": "m" + "value": "m", + "type": "https://qudt.org/vocab/unit/M" } }, "metocean:measurementType": { @@ -547,8 +547,8 @@ "en": "Cel" }, "symbol": { - "type": "https://qudt.org/vocab/unit/DEG_C", - "value": "°C" + "value": "°C", + "type": "https://qudt.org/vocab/unit/DEG_C" } }, "metocean:measurementType": { @@ -574,8 +574,8 @@ "en": "s" }, "symbol": { - "type": "https://qudt.org/vocab/unit/SEC", - "value": "s" + "value": "s", + "type": "https://qudt.org/vocab/unit/SEC" } }, "metocean:measurementType": { @@ -585,33 +585,6 @@ "metocean:standard_name": "duration_of_sunshine", "metocean:level": 2.0 }, - "lwe_precipitation_rate:1.8:mean:PT10M": { - "type": "Parameter", - "description": { - "en": "Lwe precipitation rate at 1.8m, aggregated over PT10M with method 'mean'" - }, - "observedProperty": { - "id": "https://vocab.nerc.ac.uk/standard_name/lwe_precipitation_rate", - "label": { - "en": "Lwe precipitation rate" - } - }, - "unit": { - "label": { - "en": "mm/h" - }, - "symbol": { - "type": "https://qudt.org/vocab/unit/MilliM-PER-HR", - "value": "mm/h" - } - }, - "metocean:measurementType": { - "method": "mean", - "duration": "PT10M" - }, - "metocean:standard_name": "lwe_precipitation_rate", - "metocean:level": 1.8 - }, "lwe_precipitation_rate:2.5:mean:PT10M": { "type": "Parameter", "description": { @@ -628,8 +601,8 @@ "en": "mm/h" }, "symbol": { - "type": "https://qudt.org/vocab/unit/MilliM-PER-HR", - "value": "mm/h" + "value": "mm/h", + "type": "https://qudt.org/vocab/unit/MilliM-PER-HR" } }, "metocean:measurementType": { @@ -655,8 +628,8 @@ "en": "kg/m2" }, "symbol": { - "type": "https://qudt.org/vocab/unit/KiloGM-PER-M2", - "value": "kg/m²" + "value": "kg/m²", + "type": "https://qudt.org/vocab/unit/KiloGM-PER-M2" } }, "metocean:measurementType": { @@ -682,8 +655,8 @@ "en": "kg/m2" }, "symbol": { - "type": "https://qudt.org/vocab/unit/KiloGM-PER-M2", - "value": "kg/m²" + "value": "kg/m²", + "type": "https://qudt.org/vocab/unit/KiloGM-PER-M2" } }, "metocean:measurementType": { @@ -709,8 +682,8 @@ "en": "kg/m2" }, "symbol": { - "type": "https://qudt.org/vocab/unit/KiloGM-PER-M2", - "value": "kg/m²" + "value": "kg/m²", + "type": "https://qudt.org/vocab/unit/KiloGM-PER-M2" } }, "metocean:measurementType": { @@ -736,8 +709,8 @@ "en": "kg/m2" }, "symbol": { - "type": "https://qudt.org/vocab/unit/KiloGM-PER-M2", - "value": "kg/m²" + "value": "kg/m²", + "type": "https://qudt.org/vocab/unit/KiloGM-PER-M2" } }, "metocean:measurementType": { @@ -763,8 +736,8 @@ "en": "mm/h" }, "symbol": { - "type": "https://qudt.org/vocab/unit/MilliM-PER-HR", - "value": "mm/h" + "value": "mm/h", + "type": "https://qudt.org/vocab/unit/MilliM-PER-HR" } }, "metocean:measurementType": { @@ -790,8 +763,8 @@ "en": "%" }, "symbol": { - "type": "https://qudt.org/vocab/unit/PERCENT", - "value": "%" + "value": "%", + "type": "https://qudt.org/vocab/unit/PERCENT" } }, "metocean:measurementType": { @@ -817,8 +790,8 @@ "en": "W/m2" }, "symbol": { - "type": "https://qudt.org/vocab/unit/W-PER-M2", - "value": "W/m²" + "value": "W/m²", + "type": "https://qudt.org/vocab/unit/W-PER-M2" } }, "metocean:measurementType": { @@ -828,33 +801,6 @@ "metocean:standard_name": "surface_downwelling_shortwave_flux_in_air", "metocean:level": 2.0 }, - "visibility_in_air:1.8:mean:PT10M": { - "type": "Parameter", - "description": { - "en": "Visibility in air at 1.8m, aggregated over PT10M with method 'mean'" - }, - "observedProperty": { - "id": "https://vocab.nerc.ac.uk/standard_name/visibility_in_air", - "label": { - "en": "Visibility in air" - } - }, - "unit": { - "label": { - "en": "m" - }, - "symbol": { - "type": "https://qudt.org/vocab/unit/M", - "value": "m" - } - }, - "metocean:measurementType": { - "method": "mean", - "duration": "PT10M" - }, - "metocean:standard_name": "visibility_in_air", - "metocean:level": 1.8 - }, "visibility_in_air:2.5:mean:PT10M": { "type": "Parameter", "description": { @@ -871,8 +817,8 @@ "en": "m" }, "symbol": { - "type": "https://qudt.org/vocab/unit/M", - "value": "m" + "value": "m", + "type": "https://qudt.org/vocab/unit/M" } }, "metocean:measurementType": { @@ -898,8 +844,8 @@ "en": "deg" }, "symbol": { - "type": "https://qudt.org/vocab/unit/DEG", - "value": "°" + "value": "°", + "type": "https://qudt.org/vocab/unit/DEG" } }, "metocean:measurementType": { @@ -925,8 +871,8 @@ "en": "m/s" }, "symbol": { - "type": "https://qudt.org/vocab/unit/M-PER-SEC", - "value": "m/s" + "value": "m/s", + "type": "https://qudt.org/vocab/unit/M-PER-SEC" } }, "metocean:measurementType": { @@ -952,8 +898,8 @@ "en": "m/s" }, "symbol": { - "type": "https://qudt.org/vocab/unit/M-PER-SEC", - "value": "m/s" + "value": "m/s", + "type": "https://qudt.org/vocab/unit/M-PER-SEC" } }, "metocean:measurementType": { diff --git a/datastore/integration-test/response/data_locations_two_points_with_two_parameters.json b/datastore/integration-test/response/data_locations_two_points_with_two_parameters.json deleted file mode 100644 index 01128f65..00000000 --- a/datastore/integration-test/response/data_locations_two_points_with_two_parameters.json +++ /dev/null @@ -1,101 +0,0 @@ -{ - "type": "FeatureCollection", - "features": [ - { - "type": "Feature", - "geometry": { - "type": "Point", - "coordinates": [ - 5.1797058644882, - 52.098821802977 - ] - }, - "properties": { - "name": "DE BILT AWS", - "detail": "https://oscar.wmo.int/surface/rest/api/search/station?wigosId=0-20000-0-06260", - "parameter-name": [ - "air_pressure_at_mean_sea_level:1.0:mean:PT1M", - "air_temperature:0.1:minimum:PT10M" - ], - "timeseries-link": "http://localhost:8008/collections/observations/items?platform=0-20000-0-06260" - }, - "id": "0-20000-0-06260" - }, - { - "type": "Feature", - "geometry": { - "type": "Point", - "coordinates": [ - 5.8723225499118, - 52.0548617826 - ] - }, - "properties": { - "name": "DEELEN", - "detail": "https://oscar.wmo.int/surface/rest/api/search/station?wigosId=0-20000-0-06275", - "parameter-name": [ - "air_pressure_at_mean_sea_level:1.0:mean:PT1M", - "air_temperature:0.1:minimum:PT10M" - ], - "timeseries-link": "http://localhost:8008/collections/observations/items?platform=0-20000-0-06275" - }, - "id": "0-20000-0-06275" - } - ], - "parameters": { - "air_pressure_at_mean_sea_level:1.0:mean:PT1M": { - "type": "Parameter", - "description": { - "en": "Air pressure at mean sea level at 1.0m, aggregated over PT1M with method 'mean'" - }, - "observedProperty": { - "id": "https://vocab.nerc.ac.uk/standard_name/air_pressure_at_mean_sea_level", - "label": { - "en": "Air pressure at mean sea level" - } - }, - "unit": { - "label": { - "en": "hPa" - }, - "symbol": { - "type": "https://qudt.org/vocab/unit/HectoPA", - "value": "hPa" - } - }, - "metocean:measurementType": { - "method": "mean", - "duration": "PT1M" - }, - "metocean:standard_name": "air_pressure_at_mean_sea_level", - "metocean:level": 1.0 - }, - "air_temperature:0.1:minimum:PT10M": { - "type": "Parameter", - "description": { - "en": "Air temperature at 0.1m, aggregated over PT10M with method 'minimum'" - }, - "observedProperty": { - "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", - "label": { - "en": "Air temperature" - } - }, - "unit": { - "label": { - "en": "Cel" - }, - "symbol": { - "type": "https://qudt.org/vocab/unit/DEG_C", - "value": "°C" - } - }, - "metocean:measurementType": { - "method": "minimum", - "duration": "PT10M" - }, - "metocean:standard_name": "air_temperature", - "metocean:level": 0.1 - } - } -} diff --git a/datastore/integration-test/response/metadata_collections_all_collections.json b/datastore/integration-test/response/metadata_collections_all_collections.json index 2ea6187d..6e7bd144 100644 --- a/datastore/integration-test/response/metadata_collections_all_collections.json +++ b/datastore/integration-test/response/metadata_collections_all_collections.json @@ -35,12 +35,12 @@ "temporal": { "interval": [ [ - "2022-12-31T00:00:00Z", + "2022-12-30T00:00:00Z", "2022-12-31T23:50:00Z" ] ], "values": [ - "2022-12-31T00:00:00Z/2022-12-31T23:50:00Z" + "2022-12-30T00:00:00Z/2022-12-31T23:50:00Z" ], "trs": "Gregorian" }, @@ -76,11 +76,12 @@ "id": "level", "interval": [ [ - 0.0, + -9.34, 10.0 ] ], "values": [ + -9.34, 0.0, 0.1, 1.0, @@ -97,14 +98,15 @@ "interval": [ [ "maximum", - "point" + "root_mean_square" ] ], "values": [ "maximum", "mean", "minimum", - "point" + "point", + "root_mean_square" ], "reference": "Time aggregation functions" }, @@ -113,7 +115,7 @@ "interval": [ [ "PT0S", - "PT24H" + "P365D" ] ], "values": [ @@ -123,7 +125,8 @@ "PT6H", "PT12H", "PT14H", - "PT24H" + "PT24H", + "P365D" ], "reference": "https://en.wikipedia.org/wiki/ISO_8601#Durations" } @@ -208,10 +211,10 @@ "metocean:standard_name": "air_pressure_at_mean_sea_level", "metocean:level": 1.0 }, - "air_temperature:0.1:minimum:PT10M": { + "air_temperature:-9.34:root_mean_square:P365D": { "type": "Parameter", "label": "Air temperature", - "description": "Air temperature at 0.1m, aggregated over PT10M with method 'minimum'", + "description": "Air temperature at -9.34m, aggregated over P365D with method 'root_mean_square'", "unit": { "label": "Cel", "symbol": { @@ -224,11 +227,11 @@ "label": "Air temperature" }, "measurementType": { - "method": "minimum", - "duration": "PT10M" + "method": "root_mean_square", + "duration": "P365D" }, "metocean:standard_name": "air_temperature", - "metocean:level": 0.1 + "metocean:level": -9.34 }, "air_temperature:0.0:minimum:PT12H": { "type": "Parameter", @@ -296,6 +299,28 @@ "metocean:standard_name": "air_temperature", "metocean:level": 0.0 }, + "air_temperature:0.1:minimum:PT10M": { + "type": "Parameter", + "label": "Air temperature", + "description": "Air temperature at 0.1m, aggregated over PT10M with method 'minimum'", + "unit": { + "label": "Cel", + "symbol": { + "value": "°C", + "type": "https://qudt.org/vocab/unit/DEG_C" + } + }, + "observedProperty": { + "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", + "label": "Air temperature" + }, + "measurementType": { + "method": "minimum", + "duration": "PT10M" + }, + "metocean:standard_name": "air_temperature", + "metocean:level": 0.1 + }, "air_temperature:1.5:maximum:PT10M": { "type": "Parameter", "label": "Air temperature", diff --git a/datastore/integration-test/test_api.py b/datastore/integration-test/test_api.py index 06f899af..15106ceb 100644 --- a/datastore/integration-test/test_api.py +++ b/datastore/integration-test/test_api.py @@ -105,12 +105,12 @@ def test_get_a_single_existing_collection(): actual_response_is_expected_response(actual_response, expected_json, exclude_regex_paths=r"\['href'\]$") -def test_from_a_single_collection_get_locations_within_a_bbox(): +def test_from_a_single_collection_get_locations_with_duration_range_filtering(): collection_id = "observations" - bbox = "5.0,52.0,6.0,52.1" - actual_response = requests.get(url=BASE_URL + f"/collections/{collection_id}/locations?bbox={bbox}") + duration = "P364D/P365D" + actual_response = requests.get(url=BASE_URL + f"/collections/{collection_id}/locations?duration={duration}") - expected_json = load_json("response/data_locations_two_points_with_multiple_parameters.json") + expected_json = load_json("response/data_locations_one_location_with_one_parameter.json") assert actual_response.status_code == 200 actual_response_is_expected_response( @@ -120,14 +120,12 @@ def test_from_a_single_collection_get_locations_within_a_bbox(): ) -def test_from_a_single_collection_get_locations_within_a_bbox_with_durations_range_filtering(): +def test_from_a_single_collection_get_locations_with_level_range_filtering(): collection_id = "observations" - bbox = "5.0,52.0,6.0,52.1" - actual_response = requests.get( - url=BASE_URL + f"/collections/{collection_id}/locations?bbox={bbox}&durations=PT0S/PT24H" - ) + level = "../-8.0" + actual_response = requests.get(url=BASE_URL + f"/collections/{collection_id}/locations?level={level}") - expected_json = load_json("response/data_locations_two_points_with_multiple_parameters.json") + expected_json = load_json("response/data_locations_one_location_with_one_parameter.json") assert actual_response.status_code == 200 actual_response_is_expected_response( @@ -137,12 +135,12 @@ def test_from_a_single_collection_get_locations_within_a_bbox_with_durations_ran ) -def test_from_a_single_collection_get_locations_within_bbox_with_levels_range_filtering(): +def test_from_a_single_collection_get_locations_with_parameter_name_filtering(): collection_id = "observations" - bbox = "5.0,52.0,6.0,52.1" - actual_response = requests.get(url=BASE_URL + f"/collections/{collection_id}/locations?bbox={bbox}&levels=../10.0") + parameters = "air_temperature:-9.34:root_mean_square:P365D" + actual_response = requests.get(url=BASE_URL + f"/collections/{collection_id}/locations?parameter-name={parameters}") - expected_json = load_json("response/data_locations_two_points_with_multiple_parameters.json") + expected_json = load_json("response/data_locations_one_location_with_one_parameter.json") assert actual_response.status_code == 200 actual_response_is_expected_response( @@ -152,15 +150,12 @@ def test_from_a_single_collection_get_locations_within_bbox_with_levels_range_fi ) -def test_from_a_single_collection_get_locations_within_a_bbox_with_parameter_name_filtering(): +def test_from_a_single_collection_get_locations_before_datetime(): collection_id = "observations" - bbox = "5.0,52.0,6.0,52.1" - parameters = "air_temperature:0.1:minimum:PT10M, air_pressure_at_mean_sea_level:1.0:mean:PT1M" - actual_response = requests.get( - url=BASE_URL + f"/collections/{collection_id}/locations?bbox={bbox}¶meter-name={parameters}" - ) + datetime = "../2022-12-30T00:00:00Z" + actual_response = requests.get(url=BASE_URL + f"/collections/{collection_id}/locations?datetime={datetime}") - expected_json = load_json("response/data_locations_two_points_with_two_parameters.json") + expected_json = load_json("response/data_locations_datetime_filters_where_only_one_parameter_has_data.json") assert actual_response.status_code == 200 actual_response_is_expected_response( @@ -170,40 +165,19 @@ def test_from_a_single_collection_get_locations_within_a_bbox_with_parameter_nam ) -def test_from_a_single_collection_get_locations_within_a_bbox_with_methods_and_levels_filtering(): +def test_from_a_single_collection_get_locations_within_a_bbox_with_multiple_parameters(): collection_id = "observations" - bbox = "5.0,52.0,6.0,52.1" - methods = "minimum, mean" - levels = "0.1, 1.0" - # parameters = "air_temperature:0.1:minimum:PT10M, air_pressure_at_sea_level:1:mean:PT1M" + bbox = "3.3728,50.7155,6.2512,52.5763" + level = "2.5" + datetime = "2022-12-31T00:00:00Z/.." actual_response = requests.get( - url=BASE_URL + f"/collections/{collection_id}/locations?bbox={bbox}" f"&method={methods}&level={levels}" + url=BASE_URL + f"/collections/{collection_id}/locations?bbox={bbox}&level={level}&datetime={datetime}" ) - - expected_json = load_json("response/data_locations_two_points_with_two_parameters.json") - - assert actual_response.status_code == 200 - actual_response_is_expected_response( - actual_response, - expected_json, - exclude_regex_paths=exclude_properties_timeseries, - ) - - -def test_from_a_single_collection_get_locations_within_a_bbox_with_duration_and_standard_name_filtering(): - collection_id = "observations" - bbox = "5.0,52.0,6.0,52.1" - durations = "PT10M, PT1M" - standard_names = "air_temperature, wind_from_direction" - datetime = "2022-12-31T00:50:00Z/2022-12-31T02:10:00Z" - actual_response = requests.get( - url=BASE_URL + f"/collections/{collection_id}/locations?bbox={bbox}&standard_name={standard_names}" - f"&duration={durations}&datetime={datetime}" - ) - - expected_json = load_json("response/data_locations_two_points_with_five_parameters.json") + expected_json = load_json("response/data_locations_two_points_with_multiple_parameters.json") assert actual_response.status_code == 200 + for feature in actual_response.json()["features"]: + assert "air_temperature:-9.34:root_mean_square:P365D" not in feature["properties"] actual_response_is_expected_response( actual_response, expected_json, @@ -233,15 +207,15 @@ def test_from_a_single_collection_get_a_single_location(): def test_from_a_single_collection_get_a_single_location_with_multiple_custom_coverages(): collection_id = "observations" location_id = "0-20000-0-06260" - standard_names = "air_temperature, wind_speed, dew_point_temperature, duration_of_sunshine" - levels = "0/1.8" - methods = "maximum, point" - durations = "PT1M/PT10M" + standard_name = "air_temperature, wind_speed, dew_point_temperature, duration_of_sunshine" + level = "0/1.8" + method = "maximum, point" + duration = "PT1M/PT10M" datetime = "2022-12-31T00:50:00Z/2022-12-31T02:10:00Z" actual_response = requests.get( url=BASE_URL + f"/collections/{collection_id}/locations/{location_id}" - f"?standard_name={standard_names}&level={levels}&method={methods}" - f"&duration={durations}&datetime={datetime}" + f"?standard_name={standard_name}&level={level}&method={method}" + f"&duration={duration}&datetime={datetime}" ) expected_json = load_json("response/data_position_one_location_with_one_parameter.json") @@ -254,11 +228,11 @@ def test_from_a_single_collection_get_a_single_location_with_multiple_custom_cov def test_from_a_single_collection_get_a_single_location_with_duration_range_filtering(): collection_id = "observations" location_id = "0-20000-0-06260" - durations = "PT6H/.." + duration = "PT6H/.." datetime = "2022-12-31T00:00:00Z" actual_response = requests.get( url=BASE_URL + f"/collections/{collection_id}/locations/{location_id}" - f"?duration={durations}&datetime={datetime}" + f"?duration={duration}&datetime={datetime}" ) expected_json = load_json("response/data_locations_one_location_with_multiple_parameters.json") @@ -272,9 +246,9 @@ def test_from_a_single_collection_get_a_single_location_with_standard_name_filte collection_id = "observations" location_id = "0-20000-0-06257" # Use a standard name that exists in the system but not for this location. - standard_names = "wind_from_direction" + standard_name = "wind_from_direction" actual_response = requests.get( - url=BASE_URL + f"/collections/{collection_id}/locations/{location_id}?standard_name={standard_names}" + url=BASE_URL + f"/collections/{collection_id}/locations/{location_id}?standard_name={standard_name}" ) expected_json = load_json("response/404_not_found.json") @@ -342,15 +316,15 @@ def test_from_a_single_collection_get_a_single_position_with_one_parameter(): def test_from_a_single_collection_get_a_single_position_with_all_available_custom_parameters_filtering(): collection_id = "observations" coords = "POINT(5.179705 52.0988218)" - standard_names = "air_temperature" - levels = "1.5" - methods = "maximum" - durations = "PT10M" + standard_name = "air_temperature" + level = "1.5" + method = "maximum" + duration = "PT10M" datetime = "2022-12-31T00:50:00Z/2022-12-31T02:10:00Z" actual_response = requests.get( url=BASE_URL + f"/collections/{collection_id}/position" - f"?coords={coords}&standard_name={standard_names}&level={levels}" - f"&method={methods}&duration={durations}&datetime={datetime}" + f"?coords={coords}&standard_name={standard_name}&level={level}" + f"&method={method}&duration={duration}&datetime={datetime}" ) expected_json = load_json("response/data_position_one_location_with_one_parameter.json") @@ -363,12 +337,12 @@ def test_from_a_single_collection_get_a_single_position_with_all_available_custo def test_from_a_single_collection_get_a_single_position_with_repeating_level_interval_and_duration(): collection_id = "observations" coords = "POINT(5.179705 52.0988218)" - levels = "R6/1.5/0.1" - durations = "PT1M" + level = "R6/1.5/0.1" + duration = "PT1M" datetime = "2022-12-31T00:00:00Z" actual_response = requests.get( url=BASE_URL + f"/collections/{collection_id}/position" - f"?coords={coords}&level={levels}&duration={durations}&datetime={datetime}" + f"?coords={coords}&level={level}&duration={duration}&datetime={datetime}" ) expected_json = load_json("response/data_position_one_location_with_three_parameters.json") @@ -398,11 +372,11 @@ def test_from_a_single_collection_get_an_area_with_two_parameters(): def test_from_a_single_collection_get_an_area_with_standard_name_filtering(): collection_id = "observations" coords = "POLYGON((5.0 52.0, 6.0 52.0,6.0 52.1,5.0 52.1, 5.0 52.0))" - standard_names = "relative_humidity, wind_speed" + standard_name = "relative_humidity, wind_speed" datetime = "2022-12-31T22:50:00Z/.." actual_response = requests.get( url=BASE_URL + f"/collections/{collection_id}/area" - f"?coords={coords}&standard_name={standard_names}&datetime={datetime}" + f"?coords={coords}&standard_name={standard_name}&datetime={datetime}" ) expected_json = load_json("response/data_area_two_locations_with_two_parameters.json") @@ -429,7 +403,7 @@ def test_from_a_single_collection_get_an_area_with_parameter_with_wildcard(): actual_response_is_expected_response(actual_response, expected_json) -def test_from_a_single_collection_get_an_area_with_non_existing_durations(): +def test_from_a_single_collection_get_an_area_with_non_existing_duration(): collection_id = "observations" coords = "POLYGON((4.0 52.4, 4.7 52.4,4.7 52.6,4.0 52.6, 4.0 52.4))" actual_response = requests.get( diff --git a/datastore/integration-test/test_knmi.py b/datastore/integration-test/test_knmi.py index fa786780..1681d3ba 100644 --- a/datastore/integration-test/test_knmi.py +++ b/datastore/integration-test/test_knmi.py @@ -2,12 +2,13 @@ import os from datetime import datetime -import datastore_pb2 as dstore -import datastore_pb2_grpc as dstore_grpc import grpc import pytest from google.protobuf.timestamp_pb2 import Timestamp +import datastore_pb2 as dstore +import datastore_pb2_grpc as dstore_grpc + NUMBER_OF_PARAMETERS = 44 NUMBER_OF_STATIONS = 55 @@ -47,7 +48,7 @@ def test_find_series_single_station_all_parameters(grpc_stub): response = grpc_stub.GetObservations(request) assert ( - len([obs.ts_mdata.parameter_name for obs in response.observations]) == 30 + len([obs.ts_mdata.parameter_name for obs in response.observations]) == 31 ) # Station 06260 doesn't have all parameters diff --git a/datastore/load-test/schedule_write.py b/datastore/load-test/schedule_write.py index 37f4f653..90a54d75 100644 --- a/datastore/load-test/schedule_write.py +++ b/datastore/load-test/schedule_write.py @@ -7,8 +7,6 @@ from datetime import datetime from datetime import UTC -import datastore_pb2 as dstore -import datastore_pb2_grpc as dstore_grpc import grpc from apscheduler.executors.pool import ThreadPoolExecutor from apscheduler.schedulers.blocking import BlockingScheduler @@ -16,6 +14,9 @@ from google.protobuf.timestamp_pb2 import Timestamp from variables import variable_info +import datastore_pb2 as dstore +import datastore_pb2_grpc as dstore_grpc + # from apscheduler.executors.pool import ProcessPoolExecutor @@ -60,7 +61,7 @@ def write_data(station): obs_ts.FromDatetime(obs_time) observations = [] for var in range(0, vars_per_station): - (param_id, long_name, standard_name, unit) = variable_info[var] + param_id, long_name, standard_name, unit = variable_info[var] ts_mdata = dstore.TSMetadata( platform=station.id, instrument=param_id, @@ -89,7 +90,7 @@ def write_data(station): scheduler.add_executor(ThreadPoolExecutor()) print(f"Now: {datetime.now()}", flush=True) for i in range(0, 5000): - (period, cron) = random.choice(crons) + period, cron = random.choice(crons) station_id = f"station{i:04d}" station = Station(station_id, random.uniform(50.0, 55.0), random.uniform(4.0, 8.0), period) # print(station_id, cron, period) diff --git a/justfile b/justfile index de41f806..588a7255 100644 --- a/justfile +++ b/justfile @@ -5,7 +5,7 @@ default: set positional-arguments # Run all docker services. After running the database needs cleanup, run just destroy -all: lint build unit ingest-unit services monitoring load integration performance client +all: lint build unit ingest-unit services monitoring ingest-load integration performance client # Build and run the default docker services up: build services