Add comprehensive telemetry to all content providers

.github/copilot-instructions.md

@@ -277,6 +277,7 @@ When adding a new social media provider:
 3. **Configuration UI**: Create Blazor component in `TagzApp.Blazor.Client/Components/Admin/[Platform].Config.Ui.razor`
 4. **Reference Documentation**: See `doc/Provider-Configuration-Pattern.md` for complete pattern
 5. **Add Tests**: Include unit tests in `TagzApp.UnitTest` project
+6. **Implement Telemetry**: Add comprehensive logging and metrics (see Telemetry Requirements below)
 
 ### Provider Configuration Pattern
 All providers follow a consistent pattern:
@@ -286,6 +287,69 @@ All providers follow a consistent pattern:
 - Enable/disable toggle for each provider
 - Graceful error handling with detailed logging
 
+### Telemetry Requirements
+**All new providers MUST implement comprehensive telemetry for observability:**
+
+#### Required Logging
+Inject `ILogger<T>` and add structured logging for:
+- **Connection lifecycle**: Log when starting, stopping, or changing configuration
+- **Message discovery**: Log count of new messages retrieved
+- **Error conditions**: Log detailed error messages with context
+- **Log prefix**: All logs must start with provider name (e.g., "Twitter:", "Bluesky:")
+
+Example logging:
+```csharp
+_Logger.LogInformation("Twitter: Provider started");
+_Logger.LogInformation("Twitter: Retrieved {Count} new tweets", tweets.Count);
+_Logger.LogError(ex, "Twitter: Error retrieving tweets");
+_Logger.LogWarning("Twitter: Client is not connected - check credentials");
+```
+
+#### Required Metrics
+Inject `ProviderInstrumentation?` (optional dependency) and report:
+- **Messages received**: Call `_Instrumentation?.AddMessage(providerId, username)` for each message
+- **Connection status changes**: Call `_Instrumentation?.RecordConnectionStatusChange(Id, status)` when status changes
+
+Example metrics:
+```csharp
+public TwitterProvider(IHttpClientFactory httpClientFactory, ILogger<TwitterProvider> logger,
+    TwitterConfiguration configuration, ProviderInstrumentation? instrumentation = null)
+{
+    _Instrumentation = instrumentation;
+    // ...
+}
+
+// In GetContentForHashtag
+if (_Instrumentation is not null && messages.Any())
+{
+    _Logger.LogInformation("Twitter: Retrieved {Count} new tweets", messages.Count);
+    foreach (var tweet in messages)
+    {
+        if (!string.IsNullOrEmpty(tweet.Author?.UserName))
+        {
+            _Instrumentation.AddMessage(Id.ToLowerInvariant(), tweet.Author.UserName);
+        }
+    }
+}
+
+// In StartAsync
+_Logger.LogInformation("Twitter: Provider started");
+_Instrumentation?.RecordConnectionStatusChange(Id, SocialMediaStatus.Healthy);
+
+// In StopAsync
+_Logger.LogInformation("Twitter: Provider stopped");
+_Instrumentation?.RecordConnectionStatusChange(Id, SocialMediaStatus.Disabled);
+```
+
+#### Available Metrics
+The `ProviderInstrumentation` class provides:
+- `MessagesReceivedCounter`: Tracks messages with provider and author tags
+- `ConnectionStatusChangesCounter`: Tracks status transitions with provider and status tags
+- `ConnectionStatusGauge`: Observable gauge showing current health (0=Disabled, 1=Unhealthy, 2=Degraded, 3=Healthy)
+
+#### OpenTelemetry Integration
+Metrics are automatically collected via OpenTelemetry. The meter name is `tagzapp-provider-metrics` and is configured in `TagzApp.ServiceDefaults/Extensions.cs`.
+
 ### Supported Providers (7 platforms)
 - **Blazot**: Developer-focused social platform
 - **Bluesky**: AT Protocol integration

doc/Provider-Configuration-Pattern.md

@@ -206,3 +206,130 @@ The `ToStaticMonitor()` extension method automatically handles the conversion to
 - [ ] Update `GetConfiguration` and `SaveConfiguration` methods
 - [ ] Test reactive configuration updates
 - [ ] Verify unit tests still work with testing constructor
+- [ ] **Implement comprehensive telemetry** (see Telemetry Requirements below)
+
+## Telemetry Requirements
+
+**All providers MUST implement comprehensive telemetry for observability and debugging.**
+
+### Required Dependencies
+
+1. Inject `ILogger<T>` for structured logging
+2. Inject `ProviderInstrumentation?` (optional dependency) for metrics
+
+```csharp
+public class YourProvider : ISocialMediaProvider
+{
+    private readonly ILogger<YourProvider> _Logger;
+    private readonly ProviderInstrumentation? _Instrumentation;
+
+    public YourProvider(IOptionsMonitor<YourProviderConfiguration> configMonitor,
+        ILogger<YourProvider> logger,
+        ProviderInstrumentation? instrumentation = null)
+    {
+        _Logger = logger;
+        _Instrumentation = instrumentation;
+        // ...
+    }
+}
+```
+
+### Required Logging
+
+Add structured logging for:
+
+1. **Connection Lifecycle Events**
+```csharp
+// In StartAsync
+_Logger.LogInformation("YourProvider: Provider started");
+_Instrumentation?.RecordConnectionStatusChange(Id, SocialMediaStatus.Healthy);
+
+// In StopAsync
+_Logger.LogInformation("YourProvider: Provider stopped");
+_Instrumentation?.RecordConnectionStatusChange(Id, SocialMediaStatus.Disabled);
+
+// On configuration changes
+_Logger.LogInformation("YourProvider: Configuration changed - enabling provider");
+```
+
+2. **Message Discovery**
+```csharp
+// In GetContentForHashtag
+if (_Instrumentation is not null && messages.Any())
+{
+    _Logger.LogInformation("YourProvider: Retrieved {Count} new messages", messages.Count);
+    foreach (var msg in messages)
+    {
+        if (!string.IsNullOrEmpty(msg.Author?.UserName))
+        {
+            _Instrumentation.AddMessage(Id.ToLowerInvariant(), msg.Author.UserName);
+        }
+    }
+}
+```
+
+3. **Error Conditions**
+```csharp
+catch (Exception ex)
+{
+    _Logger.LogError(ex, "YourProvider: Error fetching content");
+    _Status = SocialMediaStatus.Unhealthy;
+    _StatusMessage = $"Error: {ex.Message}";
+    _Instrumentation?.RecordConnectionStatusChange(Id, SocialMediaStatus.Unhealthy);
+}
+```
+
+4. **Warning States**
+```csharp
+if (!_Client.IsConnected)
+{
+    _Logger.LogWarning("YourProvider: Client is not connected - check credentials");
+    _Instrumentation?.RecordConnectionStatusChange(Id, SocialMediaStatus.Unhealthy);
+}
+```
+
+### Logging Conventions
+
+- **Always prefix** logs with the provider name followed by a colon (e.g., "Twitter:", "Bluesky:")
+- Use **structured logging** with named parameters: `{Count}`, `{Username}`, etc.
+- Use appropriate **log levels**:
+  - `LogInformation`: Normal operations, message counts, lifecycle events
+  - `LogWarning`: Degraded states, connection issues
+  - `LogError`: Exceptions, critical failures
+  - `LogDebug`: Detailed diagnostic information
+
+### Available Metrics
+
+The `ProviderInstrumentation` class provides:
+
+1. **MessagesReceivedCounter** (`messages-received`)
+   - Tracks individual messages with provider and author tags
+   - Call: `_Instrumentation.AddMessage(providerId, username)`
+
+2. **ConnectionStatusChangesCounter** (`connection-status-changes`)
+   - Tracks status transitions with provider and status tags
+   - Call: `_Instrumentation.RecordConnectionStatusChange(providerId, status)`
+
+3. **ConnectionStatusGauge** (`connection-status`)
+   - Observable gauge showing current provider health
+   - Values: 0=Disabled, 1=Unhealthy, 2=Degraded, 3=Healthy
+   - Updated automatically by `RecordConnectionStatusChange`
+
+### OpenTelemetry Integration
+
+Metrics are automatically collected via the `tagzapp-provider-metrics` meter configured in `TagzApp.ServiceDefaults/Extensions.cs`. No additional setup required in the provider.
+
+### Example Implementation
+
+See existing providers for reference implementations:
+- **TwitchChat**: Most comprehensive with activity tracing
+- **Twitter**: Clean logging and metrics pattern
+- **Bluesky**: Connection lifecycle tracking
+- **Mastodon**: Error handling with telemetry
+
+### Benefits
+
+- **Real-time monitoring**: Track provider health and message throughput
+- **Faster debugging**: Detailed logs help identify issues quickly
+- **Performance insights**: Metrics enable tracking of message volumes
+- **Consistent observability**: All providers follow the same pattern

src/TagzApp.Common/Telemetry/ProviderInstrumentation.cs

@@ -5,16 +5,46 @@ namespace TagzApp.Common.Telemetry;
 public class ProviderInstrumentation
 {
 	public Counter<int> MessagesReceivedCounter { get; set; }
+	public Counter<int> ConnectionStatusChangesCounter { get; set; }
+	public ObservableGauge<int> ConnectionStatusGauge { get; set; }
+
+	private readonly Dictionary<string, int> _ProviderConnectionStatus = new();
+	private readonly object _StatusLock = new();
 
 	public ProviderInstrumentation(IMeterFactory meterFactory)
 	{
 		var meter = meterFactory.Create("tagzapp-provider-metrics");
 
 		MessagesReceivedCounter = meter.CreateCounter<int>("messages-received", "message", "Counter for messages received");
+		ConnectionStatusChangesCounter = meter.CreateCounter<int>("connection-status-changes", "change", "Counter for connection status changes");
+		ConnectionStatusGauge = meter.CreateObservableGauge("connection-status", () => GetConnectionStatusMeasurements(), "status", "Current connection status of providers (0=Disabled, 1=Unhealthy, 2=Degraded, 3=Healthy)");
 	}
 
 	public void AddMessage(string provider, string author) =>
 		MessagesReceivedCounter.Add(1,
 			new KeyValuePair<string, object?>(nameof(provider), provider),
 			new KeyValuePair<string, object?>(nameof(author), author));
+
+	public void RecordConnectionStatusChange(string provider, SocialMediaStatus status)
+	{
+		lock (_StatusLock)
+		{
+			_ProviderConnectionStatus[provider] = (int)status;
+		}
+
+		ConnectionStatusChangesCounter.Add(1,
+			new KeyValuePair<string, object?>(nameof(provider), provider),
+			new KeyValuePair<string, object?>(nameof(status), status.ToString()));
+	}
+
+	private IEnumerable<Measurement<int>> GetConnectionStatusMeasurements()
+	{
+		lock (_StatusLock)
+		{
+			foreach (var kvp in _ProviderConnectionStatus)
+			{
+				yield return new Measurement<int>(kvp.Value, new KeyValuePair<string, object?>("provider", kvp.Key));
+			}
+		}
+	}
 }

src/TagzApp.Providers.AzureQueue/AzureQueueProvider.cs

@@ -1,5 +1,7 @@
 using Azure.Storage.Queues;
+using Microsoft.Extensions.Logging;
 using System.Text.Json;
+using TagzApp.Common.Telemetry;
 
 namespace TagzApp.Providers.AzureQueue;
 
@@ -8,6 +10,8 @@ public class AzureQueueProvider : ISocialMediaProvider
 	private const string QueueName = "tagzapp-content";
 	private AzureQueueConfiguration _Configuration;
 	private QueueClient _Client;
+	private readonly ILogger<AzureQueueProvider>? _Logger;
+	private readonly ProviderInstrumentation? _Instrumentation;
 	private SocialMediaStatus _Status = SocialMediaStatus.Unhealthy;
 	private string _StatusMessage = "Not started";
 	private bool _DisposedValue;
@@ -20,9 +24,12 @@ public class AzureQueueProvider : ISocialMediaProvider
 
 	public bool Enabled { get; private set; }
 
-	public AzureQueueProvider(AzureQueueConfiguration configuration)
+	public AzureQueueProvider(AzureQueueConfiguration configuration, ILogger<AzureQueueProvider>? logger = null,
+		ProviderInstrumentation? instrumentation = null)
 	{
 		_Configuration = configuration;
+		_Logger = logger;
+		_Instrumentation = instrumentation;
 		Enabled = configuration.Enabled;
 	}
 
@@ -49,6 +56,18 @@ public async Task<IEnumerable<Content>> GetContentForHashtag(Hashtag tag, DateTi
 
 		}
 
+		if (_Instrumentation is not null && outList.Any())
+		{
+			_Logger?.LogInformation("AzureQueue: Retrieved {Count} new messages", outList.Count);
+			foreach (var content in outList)
+			{
+				if (!string.IsNullOrEmpty(content.Author?.UserName))
+				{
+					_Instrumentation.AddMessage(Id.ToLowerInvariant(), content.Author.UserName);
+				}
+			}
+		}
+
 		return outList;
 
 
@@ -67,22 +86,26 @@ public async Task StartAsync()
 		try
 		{
 			await _Client.CreateIfNotExistsAsync();
+			_Status = SocialMediaStatus.Healthy;
+			_StatusMessage = "Connected";
+			_Logger?.LogInformation("AzureQueue: Successfully connected to queue");
+			_Instrumentation?.RecordConnectionStatusChange(Id, SocialMediaStatus.Healthy);
 		}
 		catch (Exception ex)
 		{
 			_Status = SocialMediaStatus.Unhealthy;
 			_StatusMessage = $"Unable to start a connection to the Azure Queue: {ex.Message}";
+			_Logger?.LogError(ex, "AzureQueue: Failed to connect to queue");
+			_Instrumentation?.RecordConnectionStatusChange(Id, SocialMediaStatus.Unhealthy);
 			return;
 		}
 
-		_Status = SocialMediaStatus.Healthy;
-		_StatusMessage = "Connected";
-
 	}
 
 	public Task StopAsync()
 	{
-		// do nothing
+		_Logger?.LogInformation("AzureQueue: Provider stopped");
+		_Instrumentation?.RecordConnectionStatusChange(Id, SocialMediaStatus.Disabled);
 		return Task.CompletedTask;
 	}