261: /locations should return all parameters available for a station, even when filtering.

api/main.py

@@ -12,13 +12,13 @@
 from export_metrics import add_metrics
 from fastapi import FastAPI
 from fastapi import Request
+from fastapi.middleware.cors import CORSMiddleware
+from middleware.x_forwarded_headers import ForwardedHostAndPrefixMiddleware
 from openapi.collections_metadata import collections_metadata
 from openapi.openapi_metadata import openapi_metadata
 from routers import edr
 from routers import feature
 from utilities import create_url_from_request
-from fastapi.middleware.cors import CORSMiddleware
-from middleware.x_forwarded_headers import ForwardedHostAndPrefixMiddleware
 
 
 all_collections = collections_metadata.keys()

api/routers/edr.py

@@ -5,11 +5,7 @@
 from typing import Literal
 
 
-import datastore_pb2 as dstore
 import formatters
-from openapi import custom_dimension_examples
-from openapi import openapi_examples
-from openapi import edr_query_parameter_descriptions
 from covjson_pydantic.coverage import Coverage
 from covjson_pydantic.coverage import CoverageCollection
 from covjson_pydantic.parameter import Parameter
@@ -22,9 +18,12 @@
 from formatters.covjson import make_parameter
 from geojson_pydantic import Feature
 from geojson_pydantic import Point
-from grpc_getter import get_obs_request
 from grpc_getter import get_locations_request
+from grpc_getter import get_obs_request
 from grpc_getter import get_ts_ag_request
+from openapi import custom_dimension_examples
+from openapi import edr_query_parameter_descriptions
+from openapi import openapi_examples
 from response_classes import CoverageJsonResponse
 from response_classes import GeoJsonResponse
 from shapely import geometry
@@ -33,6 +32,8 @@
 from utilities import add_request_parameters
 from utilities import validate_bbox
 
+import datastore_pb2 as dstore
+
 router = APIRouter(prefix="/collections/observations")
 
 response_fields_needed_for_data_api = [

datastore/data-loader/client_knmi_station_ingest.py

@@ -2,19 +2,21 @@
 # tested with Python 3.11
 import math
 import os
-import requests
-from multiprocessing import cpu_count, Pool
+from functools import partial
+from multiprocessing import cpu_count
+from multiprocessing import Pool
 from pathlib import Path
 from time import perf_counter
 from typing import List
 from typing import Tuple
-from functools import partial
 
 import pandas as pd
+import requests
 import xarray as xr
 from google.protobuf.timestamp_pb2 import Timestamp
 from parameters import knmi_parameter_names
-from utilities import generate_parameter_name, convert_unit_names
+from utilities import convert_unit_names
+from utilities import generate_parameter_name
 
 
 def netcdf_file_to_requests(file_path: Path | str) -> Tuple[List, List]:
@@ -153,6 +155,46 @@ def main():
     create_requests_start = perf_counter()
     file_path = Path(Path(__file__).parent / "test-data" / "KNMI" / "20221231.nc")
     observation_request_messages = netcdf_file_to_requests(file_path=file_path)
+    observation_request_messages.append(
+        [
+            {
+                "geometry": {"coordinates": {"lat": 52.098821802977, "lon": 5.1797058644882}, "type": "Point"},
+                "links": [{"href": "Insert documentation about E-SOH datastore", "rel": "canonical"}],
+                "properties": {
+                    "Conventions": "CF-1.8",
+                    "content": {
+                        "encoding": "utf-8",
+                        "standard_name": "air_temperature",
+                        "unit": "Cel",
+                        "value": "123.456",
+                    },
+                    "creator_email": "datacentrum@knmi.nl",
+                    "creator_name": "KNMI",
+                    "creator_type": "institution",
+                    "creator_url": "http://data.knmi.nl",
+                    "datetime": "2022-12-30T00:00:00Z",
+                    "function": "root_mean_square",
+                    "institution": "Royal Netherlands Meteorological Institute (KNMI)",
+                    "instrument": "accordion",
+                    "keywords": "temperature, pressure, relative humidity, visibility, wind speed, wind direction, "
+                    "wind gust",
+                    "keywords_vocabulary": "http://data.knmi.nl",
+                    "level": "-9.34",
+                    "license": "CC BY 4.0",
+                    "naming_authority": "nl.knmi",
+                    "parameter_name": "air_temperature:-9.34:point:P1Y",
+                    "period": "P365D",
+                    "platform": "0-20000-0-06260",
+                    "platform_name": "DE BILT AWS",
+                    "source": "Royal Netherlands Meteorological Institute (KNMI)",
+                    "summary": "A test value for the tests.",
+                    "title": "Test Value",
+                },
+                "type": "Feature",
+                "version": "4.0",
+            }
+        ]
+    )
     print("Finished creating the time series and observation requests " f"{perf_counter() - create_requests_start}.")
 
     insert_data(

datastore/datastore/storagebackend/postgresql/getlocations.go

@@ -36,21 +36,33 @@ func getLocs(
 
 	// define and execute query
 	query := fmt.Sprintf(`
-		SELECT DISTINCT ON (ts_id)
-			point,
-			platform,
-			platform_name,
-			parameter_name
-		FROM observation
-		JOIN time_series on observation.ts_id = time_series.id
-		JOIN geo_point ON observation.geo_point_id = geo_point.id
-		WHERE %s AND %s AND %s AND %s
-		ORDER BY ts_id, obstime_instant DESC;
+		WITH platforms AS (
+			SELECT DISTINCT ON (platform)
+				platform
+			FROM time_series
+				JOIN observation ON observation.ts_id = time_series.id
+				JOIN geo_point ON observation.geo_point_id = geo_point.id
+			-- time AND geolocation AND integer parameters AND string parameters
+			WHERE %s AND %s AND %s AND %s
+			ORDER BY platform, obstime_instant DESC
+		)
+		SELECT point, platform, platform_name, parameter_name FROM (  -- just to deal with final sorting
+			SELECT DISTINCT ON (ts_id)
+				point, platform, platform_name, parameter_name
+			FROM time_series
+				JOIN observation ON observation.ts_id = time_series.id
+				JOIN geo_point ON observation.geo_point_id = geo_point.id
+			WHERE platform in (SELECT platform from platforms)
+				AND %s
+			ORDER BY ts_id, platform, obstime_instant DESC
+		) t ORDER BY platform, parameter_name;
 		`,
 		timeFilter,
 		geoFilter,
 		int64MdataFilter,
-		stringMdataFilter)
+		stringMdataFilter,
+		timeFilter,
+	)
 
 	rows, err := db.Query(query, phVals...)
 	if err != nil {

datastore/integration-test/response/data_locations_datetime_filters_where_only_one_parameter_has_data.json

@@ -0,0 +1,53 @@
+{
+    "type": "FeatureCollection",
+    "features": [
+        {
+            "type": "Feature",
+            "geometry": {
+                "type": "Point",
+                "coordinates": [
+                    5.1797058644882,
+                    52.098821802977
+                ]
+            },
+            "properties": {
+                "name": "DE BILT AWS",
+                "detail": "https://oscar.wmo.int/surface/rest/api/search/station?wigosId=0-20000-0-06260",
+                "parameter-name": [
+                    "air_temperature:-9.34:root_mean_square:P365D"
+                ],
+                "timeseries-link": "http://localhost:8008/collections/observations/items?platform=0-20000-0-06260"
+            },
+            "id": "0-20000-0-06260"
+        }
+    ],
+    "parameters": {
+        "air_temperature:-9.34:root_mean_square:P365D": {
+            "type": "Parameter",
+            "description": {
+                "en": "Air temperature at -9.34m, aggregated over P365D with method 'root_mean_square'"
+            },
+            "observedProperty": {
+                "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature",
+                "label": {
+                    "en": "Air temperature"
+                }
+            },
+            "unit": {
+                "label": {
+                    "en": "Cel"
+                },
+                "symbol": {
+                    "value": "°C",
+                    "type": "https://qudt.org/vocab/unit/DEG_C"
+                }
+            },
+            "metocean:measurementType": {
+                "method": "root_mean_square",
+                "duration": "P365D"
+            },
+            "metocean:standard_name": "air_temperature",
+            "metocean:level": -9.34
+        }
+    }
+}