Fix some small fluent-bit bugs

fluent-bit/filesystem/etc/fluent-bit/fluent-bit.yaml

@@ -33,12 +33,9 @@ pipeline:
     - name: null
       match: health_check
 
-    # Log untagged records so we can respond to them
-    - name: file
+    # Log untagged records to stdout (since we don't have disk access on ECS) so we can respond to them
+    - name: stdout
       match: untagged
-      path: /fluent-bit/outputs
-      file: untagged.log
-      format: out_file
 
     # Send Stalwart telemetry to Posthog
     - name: http

fluent-bit/filesystem/etc/fluent-bit/lua/stalwart_telemetry_events.lua

@@ -87,7 +87,10 @@ function sanitize_event(event)
             -- Delete any keys we want totally gone
             event.data[value] = nil
         elseif value_in_array(hash_keys, key) then
-            -- Replace certain other values with hashes
+            -- Replace certain other values with hashes, ensuring they're strings first
+            if (type(event.data[key]) ~= 'string') then
+                event.data[key] = tostring(event.data[key])
+            end
             event.data[key] = md5(event.data[key])
         end
     end
@@ -99,7 +102,7 @@ end
 function normalize_event(event)
     -- distinct_id is used by Posthog to refer to a unique user. Today we use "from" but we would
     -- like this to be the Stalwart account ID (or a hash of it)
-    event.data.distinct_id = event.data.from
+    event.data.distinct_id = event.data.from or 'n/a'
 
     -- These are our internally recognized common fields
     os_env = os.getenv('ENV') or 'dev'

pulumi/config.dev.yaml

@@ -30,7 +30,7 @@ resources:
         - secretsmanager
       additional_routes:
         - destination_cidr_block: 10.2.0.0/16  # mailstrom-dev
-          vpc_peering_connection_id: pcx-018f09af30c2dce6c
+          vpc_peering_connection_id: pcx-0d2027442f0e54ca4
 
   tb:fargate:AutoscalingFargateCluster:
     fluentbit:
@@ -78,7 +78,7 @@ resources:
               secrets:
                 - name: POSTHOG_API_KEY
                   valueFrom: arn:aws:secretsmanager:eu-central-1:768512802988:secret:observability/dev/posthog_api_key-e3UEK4
-              image: 768512802988.dkr.ecr.eu-central-1.amazonaws.com/thunderbird/fluent-bit:7b9c3eed77ed
+              image: 768512802988.dkr.ecr.eu-central-1.amazonaws.com/thunderbird/fluent-bit:fdd1b4748cfaee29553ee2c83fcaa428b68ba8e88c2791e1626e282b48127b9d
               logConfiguration:
                 logDriver: awslogs
                 options:

pulumi/config.stage.yaml

@@ -78,7 +78,7 @@ resources:
               secrets:
                 - name: POSTHOG_API_KEY
                   valueFrom: arn:aws:secretsmanager:eu-central-1:768512802988:secret:observability/stage/posthog_api_key-3xsHYd
-              image: 768512802988.dkr.ecr.eu-central-1.amazonaws.com/thunderbird/fluent-bit:aa968a499d7e
+              image: 768512802988.dkr.ecr.eu-central-1.amazonaws.com/thunderbird/fluent-bit:fdd1b4748cfaee29553ee2c83fcaa428b68ba8e88c2791e1626e282b48127b9d
               logConfiguration:
                 logDriver: awslogs
                 options:
@@ -87,6 +87,7 @@ resources:
                   awslogs-stream-prefix: observability/stage/fluentbit/
               portMappings:
                 - containerPort: 1337
+                  hostPort: 1337
                   protocol: tcp
               restartPolicy:
                 enabled: yes