diff --git a/fluent-bit/filesystem/etc/fluent-bit/fluent-bit.yaml b/fluent-bit/filesystem/etc/fluent-bit/fluent-bit.yaml index ea5cb31..aa4b010 100644 --- a/fluent-bit/filesystem/etc/fluent-bit/fluent-bit.yaml +++ b/fluent-bit/filesystem/etc/fluent-bit/fluent-bit.yaml @@ -33,12 +33,9 @@ pipeline: - name: null match: health_check - # Log untagged records so we can respond to them - - name: file + # Log untagged records to stdout (since we don't have disk access on ECS) so we can respond to them + - name: stdout match: untagged - path: /fluent-bit/outputs - file: untagged.log - format: out_file # Send Stalwart telemetry to Posthog - name: http diff --git a/fluent-bit/filesystem/etc/fluent-bit/lua/stalwart_telemetry_events.lua b/fluent-bit/filesystem/etc/fluent-bit/lua/stalwart_telemetry_events.lua index 4ec97ff..6ac464e 100644 --- a/fluent-bit/filesystem/etc/fluent-bit/lua/stalwart_telemetry_events.lua +++ b/fluent-bit/filesystem/etc/fluent-bit/lua/stalwart_telemetry_events.lua @@ -87,7 +87,10 @@ function sanitize_event(event) -- Delete any keys we want totally gone event.data[value] = nil elseif value_in_array(hash_keys, key) then - -- Replace certain other values with hashes + -- Replace certain other values with hashes, ensuring they're strings first + if (type(event.data[key]) ~= 'string') then + event.data[key] = tostring(event.data[key]) + end event.data[key] = md5(event.data[key]) end end @@ -99,7 +102,7 @@ end function normalize_event(event) -- distinct_id is used by Posthog to refer to a unique user. Today we use "from" but we would -- like this to be the Stalwart account ID (or a hash of it) - event.data.distinct_id = event.data.from + event.data.distinct_id = event.data.from or 'n/a' -- These are our internally recognized common fields os_env = os.getenv('ENV') or 'dev' diff --git a/pulumi/config.dev.yaml b/pulumi/config.dev.yaml index 3a5d617..e29a67a 100644 --- a/pulumi/config.dev.yaml +++ b/pulumi/config.dev.yaml @@ -30,7 +30,7 @@ resources: - secretsmanager additional_routes: - destination_cidr_block: 10.2.0.0/16 # mailstrom-dev - vpc_peering_connection_id: pcx-018f09af30c2dce6c + vpc_peering_connection_id: pcx-0d2027442f0e54ca4 tb:fargate:AutoscalingFargateCluster: fluentbit: @@ -78,7 +78,7 @@ resources: secrets: - name: POSTHOG_API_KEY valueFrom: arn:aws:secretsmanager:eu-central-1:768512802988:secret:observability/dev/posthog_api_key-e3UEK4 - image: 768512802988.dkr.ecr.eu-central-1.amazonaws.com/thunderbird/fluent-bit:7b9c3eed77ed + image: 768512802988.dkr.ecr.eu-central-1.amazonaws.com/thunderbird/fluent-bit:fdd1b4748cfaee29553ee2c83fcaa428b68ba8e88c2791e1626e282b48127b9d logConfiguration: logDriver: awslogs options: diff --git a/pulumi/config.stage.yaml b/pulumi/config.stage.yaml index 0dc30ea..9fe4881 100644 --- a/pulumi/config.stage.yaml +++ b/pulumi/config.stage.yaml @@ -78,7 +78,7 @@ resources: secrets: - name: POSTHOG_API_KEY valueFrom: arn:aws:secretsmanager:eu-central-1:768512802988:secret:observability/stage/posthog_api_key-3xsHYd - image: 768512802988.dkr.ecr.eu-central-1.amazonaws.com/thunderbird/fluent-bit:aa968a499d7e + image: 768512802988.dkr.ecr.eu-central-1.amazonaws.com/thunderbird/fluent-bit:fdd1b4748cfaee29553ee2c83fcaa428b68ba8e88c2791e1626e282b48127b9d logConfiguration: logDriver: awslogs options: @@ -87,6 +87,7 @@ resources: awslogs-stream-prefix: observability/stage/fluentbit/ portMappings: - containerPort: 1337 + hostPort: 1337 protocol: tcp restartPolicy: enabled: yes