diff --git a/fleetspeak/src/client/internal/monitoring/resource_usage_fetcher.go b/fleetspeak/src/client/internal/monitoring/resource_usage_fetcher.go index 96fa98b2..4f128141 100644 --- a/fleetspeak/src/client/internal/monitoring/resource_usage_fetcher.go +++ b/fleetspeak/src/client/internal/monitoring/resource_usage_fetcher.go @@ -39,6 +39,12 @@ type ResourceUsage struct { // Resident set size for a process, in bytes. ResidentMemory int64 + + // Client IO data read so far. + IOReadBytes int64 + + // Client IO data written so far. + IOWriteBytes int64 } // ResourceUsageFetcher obtains resource-usage data for a process from the OS. @@ -48,7 +54,8 @@ type ResourceUsageFetcher struct{} // information from exec.Cmd.ProcessState. NOTE that this is only possible // after the process has finished and has been waited for, and will most // probably panic otherwise. -// This function doesn't fill in ResourceUsage.ResidentMemory. +// This function doesn't fill in ResourceUsage.ResidentMemory, ResourceUsage.IORead +// and ResourceUsage.IOWrite. func (f ResourceUsageFetcher) ResourceUsageFromFinishedCmd(cmd *exec.Cmd) *ResourceUsage { return &ResourceUsage{ Timestamp: time.Now(), @@ -78,11 +85,17 @@ func (f ResourceUsageFetcher) ResourceUsageForPID(pid int) (*ResourceUsage, erro return nil, err } + ioCounters, err := process.IOCounters() + if err != nil { + return nil, err + } return &ResourceUsage{ Timestamp: timestamp, UserCPUMillis: times.User * 1e3, SystemCPUMillis: times.System * 1e3, ResidentMemory: int64(memoryInfo.RSS), + IOReadBytes: int64(ioCounters.ReadBytes), + IOWriteBytes: int64(ioCounters.WriteBytes), }, nil } diff --git a/fleetspeak/src/client/internal/monitoring/resource_usage_fetcher_unix.go b/fleetspeak/src/client/internal/monitoring/resource_usage_fetcher_unix.go index 006d9ac8..05235009 100644 --- a/fleetspeak/src/client/internal/monitoring/resource_usage_fetcher_unix.go +++ b/fleetspeak/src/client/internal/monitoring/resource_usage_fetcher_unix.go @@ -25,6 +25,8 @@ import ( "strconv" "strings" "time" + + "github.com/shirou/gopsutil/process" ) // TODO: Support monitoring on other platforms. @@ -42,6 +44,12 @@ type ResourceUsage struct { // Resident set size for a process. ResidentMemory int64 + + // Client IO data read so far. + IOReadBytes int64 + + // Client IO data written so far. + IOWriteBytes int64 } // ResourceUsageFetcher obtains resource-usage data for a process from the OS. @@ -125,11 +133,22 @@ func (f ResourceUsageFetcher) ResourceUsageForPID(pid int) (*ResourceUsage, erro return nil, fmt.Errorf("error while parsing resident from %s: %v", statmFilename, err) } + process, err := process.NewProcess(int32(pid)) + if err != nil { + return nil, err + } + + ioCounters, err := process.IOCounters() + if err != nil { + return nil, err + } return &ResourceUsage{ Timestamp: timestamp, UserCPUMillis: float64((utime + cutime) * 10), // Assume rate of 100 ticks/second SystemCPUMillis: float64((stime + cstime) * 10), // Assume rate of 100 ticks/second ResidentMemory: resident * pageSize, + IOReadBytes: int64(ioCounters.ReadBytes), + IOWriteBytes: int64(ioCounters.WriteBytes), }, nil } diff --git a/fleetspeak/src/client/internal/monitoring/resource_usage_monitor.go b/fleetspeak/src/client/internal/monitoring/resource_usage_monitor.go index 7409082f..d122b97a 100644 --- a/fleetspeak/src/client/internal/monitoring/resource_usage_monitor.go +++ b/fleetspeak/src/client/internal/monitoring/resource_usage_monitor.go @@ -63,6 +63,10 @@ func AggregateResourceUsage(prevRU *ResourceUsage, currRU *ResourceUsage, numRUC } aggRU.MeanResidentMemory = float64(currRU.ResidentMemory) / float64(numRUCalls) aggRU.MaxResidentMemory = currRU.ResidentMemory + aggRU.MeanIoReadBytes = float64(currRU.IOReadBytes) / float64(numRUCalls) + aggRU.MaxIoReadBytes = currRU.IOReadBytes + aggRU.MeanIoWriteBytes = float64(currRU.IOWriteBytes) / float64(numRUCalls) + aggRU.MaxIoWriteBytes = currRU.IOWriteBytes return nil } @@ -80,6 +84,10 @@ func AggregateResourceUsage(prevRU *ResourceUsage, currRU *ResourceUsage, numRUC return nil } + if err := aggregateIOResourceUsage(currRU, numRUCalls, aggRU); err != nil { + return err + } + return aggregateMemoryResourceUsage(currRU, numRUCalls, aggRU) } @@ -119,6 +127,18 @@ func aggregateMemoryResourceUsage(currRU *ResourceUsage, numRUCalls int, aggRU * return nil } +func aggregateIOResourceUsage(currRU *ResourceUsage, numRUCalls int, aggRU *mpb.AggregatedResourceUsage) error { + aggRU.MeanIoReadBytes += float64(currRU.IOReadBytes) / float64(numRUCalls) + if currRU.IOReadBytes > aggRU.MaxIoReadBytes { + aggRU.MaxIoReadBytes = currRU.IOReadBytes + } + aggRU.MeanIoWriteBytes += float64(currRU.IOWriteBytes) / float64(numRUCalls) + if currRU.IOWriteBytes > aggRU.MaxIoWriteBytes { + aggRU.MaxIoWriteBytes = currRU.IOWriteBytes + } + return nil +} + // AggregateResourceUsageForFinishedCmd computes resource-usage for a finished process, given // resource-usage before and after the process ran. func AggregateResourceUsageForFinishedCmd(initialRU, finalRU *ResourceUsage) (*mpb.AggregatedResourceUsage, error) { @@ -132,13 +152,21 @@ func AggregateResourceUsageForFinishedCmd(initialRU, finalRU *ResourceUsage) (*m return nil, err } - // If this field is untouched, we have not aggregated memory resource usage + // If those field are untouched, we have not aggregated memory and IO resources usage // for this process yet. We fill it in with what we have. // TODO if aggRU.MaxResidentMemory == 0 { aggRU.MeanResidentMemory = float64(initialRU.ResidentMemory) aggRU.MaxResidentMemory = initialRU.ResidentMemory } + if aggRU.MaxIoReadBytes == 0 { + aggRU.MeanIoReadBytes = float64(initialRU.IOReadBytes) + aggRU.MaxIoReadBytes = initialRU.IOReadBytes + } + if aggRU.MaxIoWriteBytes == 0 { + aggRU.MeanIoWriteBytes = float64(initialRU.IOWriteBytes) + aggRU.MaxIoWriteBytes = initialRU.IOWriteBytes + } return &aggRU, nil } diff --git a/fleetspeak/src/client/internal/monitoring/resource_usage_monitor_test.go b/fleetspeak/src/client/internal/monitoring/resource_usage_monitor_test.go index efbd8118..76282661 100644 --- a/fleetspeak/src/client/internal/monitoring/resource_usage_monitor_test.go +++ b/fleetspeak/src/client/internal/monitoring/resource_usage_monitor_test.go @@ -75,12 +75,16 @@ func TestResourceUsageMonitor(t *testing.T) { UserCPUMillis: 13.0, SystemCPUMillis: 5.0, ResidentMemory: 10000, + IOReadBytes: 40000, + IOWriteBytes: 50000, } fakeRU1 := ResourceUsage{ Timestamp: start.Add(10 * time.Millisecond), UserCPUMillis: 27.0, SystemCPUMillis: 9.0, ResidentMemory: 30000, + IOReadBytes: 80000, + IOWriteBytes: 100000, } ruf := fakeResourceUsageFetcher{} ruf.setResourceUsageData([]ResourceUsage{fakeRU0, fakeRU1}) @@ -123,6 +127,10 @@ func TestResourceUsageMonitor(t *testing.T) { MaxSystemCpuRate: 400.0, MeanResidentMemory: 20000.0, MaxResidentMemory: 30000, + MeanIoReadBytes: 60000.0, + MaxIoReadBytes: 80000, + MeanIoWriteBytes: 75000.0, + MaxIoWriteBytes: 100000, }, DebugStatus: fmt.Sprintf("Fake Debug Status %d", protosReceived), DataTimestamp: got.DataTimestamp, diff --git a/fleetspeak/src/common/proto/fleetspeak_monitoring/resource.pb.go b/fleetspeak/src/common/proto/fleetspeak_monitoring/resource.pb.go index c73f1d27..b803dc28 100644 --- a/fleetspeak/src/common/proto/fleetspeak_monitoring/resource.pb.go +++ b/fleetspeak/src/common/proto/fleetspeak_monitoring/resource.pb.go @@ -89,6 +89,10 @@ type AggregatedResourceUsage struct { MaxSystemCpuRate float64 `protobuf:"fixed64,4,opt,name=max_system_cpu_rate,json=maxSystemCpuRate,proto3" json:"max_system_cpu_rate,omitempty"` MeanResidentMemory float64 `protobuf:"fixed64,5,opt,name=mean_resident_memory,json=meanResidentMemory,proto3" json:"mean_resident_memory,omitempty"` MaxResidentMemory int64 `protobuf:"varint,6,opt,name=max_resident_memory,json=maxResidentMemory,proto3" json:"max_resident_memory,omitempty"` + MeanIoReadBytes float64 `protobuf:"fixed64,7,opt,name=mean_io_read_bytes,json=meanIoReadBytes,proto3" json:"mean_io_read_bytes,omitempty"` + MaxIoReadBytes int64 `protobuf:"varint,8,opt,name=max_io_read_bytes,json=maxIoReadBytes,proto3" json:"max_io_read_bytes,omitempty"` + MeanIoWriteBytes float64 `protobuf:"fixed64,9,opt,name=mean_io_write_bytes,json=meanIoWriteBytes,proto3" json:"mean_io_write_bytes,omitempty"` + MaxIoWriteBytes int64 `protobuf:"varint,10,opt,name=max_io_write_bytes,json=maxIoWriteBytes,proto3" json:"max_io_write_bytes,omitempty"` } func (x *AggregatedResourceUsage) Reset() { @@ -165,6 +169,34 @@ func (x *AggregatedResourceUsage) GetMaxResidentMemory() int64 { return 0 } +func (x *AggregatedResourceUsage) GetMeanIoReadBytes() float64 { + if x != nil { + return x.MeanIoReadBytes + } + return 0 +} + +func (x *AggregatedResourceUsage) GetMaxIoReadBytes() int64 { + if x != nil { + return x.MaxIoReadBytes + } + return 0 +} + +func (x *AggregatedResourceUsage) GetMeanIoWriteBytes() float64 { + if x != nil { + return x.MeanIoWriteBytes + } + return 0 +} + +func (x *AggregatedResourceUsage) GetMaxIoWriteBytes() int64 { + if x != nil { + return x.MaxIoWriteBytes + } + return 0 +} + // A fleetspeak.Message with message type "ResourceUsage" is sent regularly by // the system and daemon services to the server, to report the performance of // processes. @@ -384,7 +416,7 @@ var file_fleetspeak_src_common_proto_fleetspeak_monitoring_resource_proto_rawDes 0x74, 0x6f, 0x12, 0x15, 0x66, 0x6c, 0x65, 0x65, 0x74, 0x73, 0x70, 0x65, 0x61, 0x6b, 0x2e, 0x6d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, - 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xb3, 0x02, 0x0a, 0x17, 0x41, + 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xe7, 0x03, 0x0a, 0x17, 0x41, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x65, 0x64, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x55, 0x73, 0x61, 0x67, 0x65, 0x12, 0x2b, 0x0a, 0x12, 0x6d, 0x65, 0x61, 0x6e, 0x5f, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x63, 0x70, 0x75, 0x5f, 0x72, 0x61, 0x74, 0x65, 0x18, 0x01, 0x20, 0x01, @@ -404,55 +436,66 @@ var file_fleetspeak_src_common_proto_fleetspeak_monitoring_resource_proto_rawDes 0x12, 0x2e, 0x0a, 0x13, 0x6d, 0x61, 0x78, 0x5f, 0x72, 0x65, 0x73, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x5f, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x11, 0x6d, 0x61, 0x78, 0x52, 0x65, 0x73, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x4d, 0x65, 0x6d, 0x6f, 0x72, 0x79, - 0x22, 0x8b, 0x03, 0x0a, 0x11, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x55, 0x73, 0x61, - 0x67, 0x65, 0x44, 0x61, 0x74, 0x61, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x63, 0x6f, 0x70, 0x65, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x73, 0x63, 0x6f, 0x70, 0x65, 0x12, 0x10, 0x0a, 0x03, - 0x70, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x03, 0x70, 0x69, 0x64, 0x12, 0x18, - 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x48, 0x0a, 0x12, 0x70, 0x72, 0x6f, 0x63, - 0x65, 0x73, 0x73, 0x5f, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, - 0x52, 0x10, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x53, 0x74, 0x61, 0x72, 0x74, 0x54, 0x69, - 0x6d, 0x65, 0x12, 0x41, 0x0a, 0x0e, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x73, - 0x74, 0x61, 0x6d, 0x70, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, - 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0d, 0x64, 0x61, 0x74, 0x61, 0x54, 0x69, 0x6d, 0x65, - 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x55, 0x0a, 0x0e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, - 0x65, 0x5f, 0x75, 0x73, 0x61, 0x67, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, - 0x66, 0x6c, 0x65, 0x65, 0x74, 0x73, 0x70, 0x65, 0x61, 0x6b, 0x2e, 0x6d, 0x6f, 0x6e, 0x69, 0x74, - 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x41, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x65, 0x64, - 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x55, 0x73, 0x61, 0x67, 0x65, 0x52, 0x0d, 0x72, - 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x55, 0x73, 0x61, 0x67, 0x65, 0x12, 0x21, 0x0a, 0x0c, - 0x64, 0x65, 0x62, 0x75, 0x67, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x06, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0b, 0x64, 0x65, 0x62, 0x75, 0x67, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, - 0x2d, 0x0a, 0x12, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x74, 0x65, 0x72, 0x6d, 0x69, - 0x6e, 0x61, 0x74, 0x65, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x11, 0x70, 0x72, 0x6f, - 0x63, 0x65, 0x73, 0x73, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x65, 0x64, 0x22, 0xee, - 0x02, 0x0a, 0x10, 0x4b, 0x69, 0x6c, 0x6c, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x10, 0x0a, - 0x03, 0x70, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x03, 0x70, 0x69, 0x64, 0x12, - 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x48, 0x0a, 0x12, 0x70, 0x72, 0x6f, - 0x63, 0x65, 0x73, 0x73, 0x5f, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, - 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, - 0x70, 0x52, 0x10, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x53, 0x74, 0x61, 0x72, 0x74, 0x54, - 0x69, 0x6d, 0x65, 0x12, 0x3b, 0x0a, 0x0b, 0x6b, 0x69, 0x6c, 0x6c, 0x65, 0x64, 0x5f, 0x77, 0x68, - 0x65, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, + 0x12, 0x2b, 0x0a, 0x12, 0x6d, 0x65, 0x61, 0x6e, 0x5f, 0x69, 0x6f, 0x5f, 0x72, 0x65, 0x61, 0x64, + 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0f, 0x6d, 0x65, + 0x61, 0x6e, 0x49, 0x6f, 0x52, 0x65, 0x61, 0x64, 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, 0x29, 0x0a, + 0x11, 0x6d, 0x61, 0x78, 0x5f, 0x69, 0x6f, 0x5f, 0x72, 0x65, 0x61, 0x64, 0x5f, 0x62, 0x79, 0x74, + 0x65, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0e, 0x6d, 0x61, 0x78, 0x49, 0x6f, 0x52, + 0x65, 0x61, 0x64, 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, 0x2d, 0x0a, 0x13, 0x6d, 0x65, 0x61, 0x6e, + 0x5f, 0x69, 0x6f, 0x5f, 0x77, 0x72, 0x69, 0x74, 0x65, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, + 0x09, 0x20, 0x01, 0x28, 0x01, 0x52, 0x10, 0x6d, 0x65, 0x61, 0x6e, 0x49, 0x6f, 0x57, 0x72, 0x69, + 0x74, 0x65, 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, 0x2b, 0x0a, 0x12, 0x6d, 0x61, 0x78, 0x5f, 0x69, + 0x6f, 0x5f, 0x77, 0x72, 0x69, 0x74, 0x65, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x0a, 0x20, + 0x01, 0x28, 0x03, 0x52, 0x0f, 0x6d, 0x61, 0x78, 0x49, 0x6f, 0x57, 0x72, 0x69, 0x74, 0x65, 0x42, + 0x79, 0x74, 0x65, 0x73, 0x22, 0x8b, 0x03, 0x0a, 0x11, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x55, 0x73, 0x61, 0x67, 0x65, 0x44, 0x61, 0x74, 0x61, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x63, + 0x6f, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x73, 0x63, 0x6f, 0x70, 0x65, + 0x12, 0x10, 0x0a, 0x03, 0x70, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x03, 0x70, + 0x69, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x08, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x48, 0x0a, 0x12, + 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x74, 0x69, + 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, - 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0a, 0x6b, 0x69, 0x6c, 0x6c, 0x65, 0x64, 0x57, 0x68, 0x65, 0x6e, - 0x12, 0x46, 0x0a, 0x06, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0e, - 0x32, 0x2e, 0x2e, 0x66, 0x6c, 0x65, 0x65, 0x74, 0x73, 0x70, 0x65, 0x61, 0x6b, 0x2e, 0x6d, 0x6f, - 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x4b, 0x69, 0x6c, 0x6c, 0x4e, 0x6f, 0x74, - 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, - 0x52, 0x06, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x22, 0x45, 0x0a, 0x06, 0x52, 0x65, 0x61, 0x73, - 0x6f, 0x6e, 0x12, 0x0f, 0x0a, 0x0b, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, - 0x44, 0x10, 0x00, 0x12, 0x15, 0x0a, 0x11, 0x48, 0x45, 0x41, 0x52, 0x54, 0x42, 0x45, 0x41, 0x54, - 0x5f, 0x46, 0x41, 0x49, 0x4c, 0x55, 0x52, 0x45, 0x10, 0x01, 0x12, 0x13, 0x0a, 0x0f, 0x4d, 0x45, - 0x4d, 0x4f, 0x52, 0x59, 0x5f, 0x45, 0x58, 0x43, 0x45, 0x45, 0x44, 0x45, 0x44, 0x10, 0x02, 0x62, - 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x74, 0x61, 0x6d, 0x70, 0x52, 0x10, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x53, 0x74, 0x61, + 0x72, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x41, 0x0a, 0x0e, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x74, + 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, + 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0d, 0x64, 0x61, 0x74, 0x61, + 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x55, 0x0a, 0x0e, 0x72, 0x65, 0x73, + 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x75, 0x73, 0x61, 0x67, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x2e, 0x2e, 0x66, 0x6c, 0x65, 0x65, 0x74, 0x73, 0x70, 0x65, 0x61, 0x6b, 0x2e, 0x6d, + 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x41, 0x67, 0x67, 0x72, 0x65, 0x67, + 0x61, 0x74, 0x65, 0x64, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x55, 0x73, 0x61, 0x67, + 0x65, 0x52, 0x0d, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x55, 0x73, 0x61, 0x67, 0x65, + 0x12, 0x21, 0x0a, 0x0c, 0x64, 0x65, 0x62, 0x75, 0x67, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x65, 0x62, 0x75, 0x67, 0x53, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x12, 0x2d, 0x0a, 0x12, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x74, + 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x65, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, + 0x11, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, + 0x65, 0x64, 0x22, 0xee, 0x02, 0x0a, 0x10, 0x4b, 0x69, 0x6c, 0x6c, 0x4e, 0x6f, 0x74, 0x69, 0x66, + 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x65, 0x12, 0x10, 0x0a, 0x03, 0x70, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x03, + 0x70, 0x69, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x48, 0x0a, + 0x12, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x74, + 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, + 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x10, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x53, 0x74, + 0x61, 0x72, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x3b, 0x0a, 0x0b, 0x6b, 0x69, 0x6c, 0x6c, 0x65, + 0x64, 0x5f, 0x77, 0x68, 0x65, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, + 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0a, 0x6b, 0x69, 0x6c, 0x6c, 0x65, 0x64, + 0x57, 0x68, 0x65, 0x6e, 0x12, 0x46, 0x0a, 0x06, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x06, + 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2e, 0x2e, 0x66, 0x6c, 0x65, 0x65, 0x74, 0x73, 0x70, 0x65, 0x61, + 0x6b, 0x2e, 0x6d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x4b, 0x69, 0x6c, + 0x6c, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x52, 0x65, + 0x61, 0x73, 0x6f, 0x6e, 0x52, 0x06, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x22, 0x45, 0x0a, 0x06, + 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x12, 0x0f, 0x0a, 0x0b, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, + 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x15, 0x0a, 0x11, 0x48, 0x45, 0x41, 0x52, 0x54, + 0x42, 0x45, 0x41, 0x54, 0x5f, 0x46, 0x41, 0x49, 0x4c, 0x55, 0x52, 0x45, 0x10, 0x01, 0x12, 0x13, + 0x0a, 0x0f, 0x4d, 0x45, 0x4d, 0x4f, 0x52, 0x59, 0x5f, 0x45, 0x58, 0x43, 0x45, 0x45, 0x44, 0x45, + 0x44, 0x10, 0x02, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/fleetspeak/src/common/proto/fleetspeak_monitoring/resource.proto b/fleetspeak/src/common/proto/fleetspeak_monitoring/resource.proto index 0c7b69ae..58d85bbb 100644 --- a/fleetspeak/src/common/proto/fleetspeak_monitoring/resource.proto +++ b/fleetspeak/src/common/proto/fleetspeak_monitoring/resource.proto @@ -14,6 +14,10 @@ message AggregatedResourceUsage { double max_system_cpu_rate = 4; double mean_resident_memory = 5; int64 max_resident_memory = 6; + double mean_io_read_bytes = 7; + int64 max_io_read_bytes = 8; + double mean_io_write_bytes = 9; + int64 max_io_write_bytes = 10; } // A fleetspeak.Message with message type "ResourceUsage" is sent regularly by diff --git a/fleetspeak/src/server/dbtesting/clientstore_suite.go b/fleetspeak/src/server/dbtesting/clientstore_suite.go index 8c4db5da..16e10372 100644 --- a/fleetspeak/src/server/dbtesting/clientstore_suite.go +++ b/fleetspeak/src/server/dbtesting/clientstore_suite.go @@ -360,6 +360,8 @@ func fetchResourceUsageRecordsTest(t *testing.T, ds db.Store) { } meanRAM, maxRAM := 190, 200 + meanIoReadData, maxIoReadData := 300, 310 + meanIoWriteData, maxIoWriteData := 400, 410 rud := mpb.ResourceUsageData{ Scope: "test-scope", Pid: 1234, @@ -373,6 +375,10 @@ func fetchResourceUsageRecordsTest(t *testing.T, ds db.Store) { MaxSystemCpuRate: 80.0, MeanResidentMemory: float64(meanRAM) * 1024 * 1024, MaxResidentMemory: int64(maxRAM) * 1024 * 1024, + MeanIoReadBytes: float64(meanIoReadData) * 1024, + MaxIoReadBytes: int64(maxIoReadData) * 1024, + MeanIoWriteBytes: float64(meanIoWriteData) * 1024, + MaxIoWriteBytes: int64(maxIoWriteData) * 1024, }, } @@ -414,6 +420,10 @@ func fetchResourceUsageRecordsTest(t *testing.T, ds db.Store) { MaxSystemCpuRate: 80.0, MeanResidentMemoryMib: int32(meanRAM), MaxResidentMemoryMib: int32(maxRAM), + MeanIoReadKib: int64(meanIoReadData), + MaxIoReadKib: int64(maxIoReadData), + MeanIoWriteKib: int64(meanIoWriteData), + MaxIoWriteKib: int64(maxIoWriteData), } if got, want := record, expected; !proto.Equal(got, want) { diff --git a/fleetspeak/src/server/mysql/clientstore.go b/fleetspeak/src/server/mysql/clientstore.go index 9876edd8..91bd3aa1 100644 --- a/fleetspeak/src/server/mysql/clientstore.go +++ b/fleetspeak/src/server/mysql/clientstore.go @@ -19,6 +19,7 @@ import ( "database/sql" "encoding/binary" "fmt" + "math" "strconv" "time" @@ -35,6 +36,7 @@ import ( ) const ( + bytesToKIB = 1.0 / float64(1<<10) bytesToMIB = 1.0 / float64(1<<20) ) @@ -343,7 +345,7 @@ func (d *Datastore) RecordResourceUsageData(ctx context.Context, id common.Clien return d.runInTx(ctx, false, func(tx *sql.Tx) error { _, err := tx.ExecContext( ctx, - "INSERT INTO client_resource_usage_records VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + "INSERT INTO client_resource_usage_records VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", id.Bytes(), rud.Scope, rud.Pid, @@ -356,7 +358,11 @@ func (d *Datastore) RecordResourceUsageData(ctx context.Context, id common.Clien rud.ResourceUsage.MeanSystemCpuRate, rud.ResourceUsage.MaxSystemCpuRate, int32(rud.ResourceUsage.MeanResidentMemory*bytesToMIB), - int32(float64(rud.ResourceUsage.MaxResidentMemory)*bytesToMIB)) + int32(float64(rud.ResourceUsage.MaxResidentMemory)*bytesToMIB), + int64(math.Ceil(rud.ResourceUsage.MeanIoReadBytes*bytesToKIB)), + int64(math.Ceil(float64(rud.ResourceUsage.MaxIoReadBytes)*bytesToKIB)), + int64(math.Ceil(rud.ResourceUsage.MeanIoWriteBytes*bytesToKIB)), + int64(math.Ceil(float64(rud.ResourceUsage.MaxIoWriteBytes)*bytesToKIB))) return err }) } @@ -381,7 +387,8 @@ func (d *Datastore) FetchResourceUsageRecords(ctx context.Context, id common.Cli "SELECT "+ "scope, pid, process_start_time, client_timestamp, server_timestamp, "+ "process_terminated, mean_user_cpu_rate, max_user_cpu_rate, mean_system_cpu_rate, "+ - "max_system_cpu_rate, mean_resident_memory_mib, max_resident_memory_mib "+ + "max_system_cpu_rate, mean_resident_memory_mib, max_resident_memory_mib, "+ + "mean_io_read_kib, max_io_read_kib, mean_io_write_kib, max_io_write_kib "+ "FROM client_resource_usage_records WHERE client_id=? "+ "AND server_timestamp >= ? AND server_timestamp < ?", id.Bytes(), @@ -400,7 +407,8 @@ func (d *Datastore) FetchResourceUsageRecords(ctx context.Context, id common.Cli err := rows.Scan( &record.Scope, &record.Pid, &processStartTime, &clientTimestamp, &serverTimestamp, &record.ProcessTerminated, &record.MeanUserCpuRate, &record.MaxUserCpuRate, &record.MeanSystemCpuRate, - &record.MaxSystemCpuRate, &record.MeanResidentMemoryMib, &record.MaxResidentMemoryMib) + &record.MaxSystemCpuRate, &record.MeanResidentMemoryMib, &record.MaxResidentMemoryMib, + &record.MeanIoReadKib, &record.MaxIoReadKib, &record.MeanIoWriteKib, &record.MaxIoWriteKib) if err != nil { return err diff --git a/fleetspeak/src/server/mysql/mysql.go b/fleetspeak/src/server/mysql/mysql.go index ec9c03ec..95236927 100644 --- a/fleetspeak/src/server/mysql/mysql.go +++ b/fleetspeak/src/server/mysql/mysql.go @@ -154,6 +154,10 @@ mean_system_cpu_rate REAL, max_system_cpu_rate REAL, mean_resident_memory_mib INT4, max_resident_memory_mib INT4, +mean_io_read_kib INT8, +max_io_read_kib INT8, +mean_io_write_kib INT8, +max_io_write_kib INT8, FOREIGN KEY (client_id) REFERENCES clients(client_id))`, `CREATE TABLE IF NOT EXISTS messages( message_id BINARY(32) NOT NULL, diff --git a/fleetspeak/src/server/proto/fleetspeak_server/resource.pb.go b/fleetspeak/src/server/proto/fleetspeak_server/resource.pb.go index 73129c28..e7ef2b33 100644 --- a/fleetspeak/src/server/proto/fleetspeak_server/resource.pb.go +++ b/fleetspeak/src/server/proto/fleetspeak_server/resource.pb.go @@ -53,6 +53,10 @@ type ClientResourceUsageRecord struct { MaxSystemCpuRate float32 `protobuf:"fixed32,9,opt,name=max_system_cpu_rate,json=maxSystemCpuRate,proto3" json:"max_system_cpu_rate,omitempty"` MeanResidentMemoryMib int32 `protobuf:"varint,10,opt,name=mean_resident_memory_mib,json=meanResidentMemoryMib,proto3" json:"mean_resident_memory_mib,omitempty"` MaxResidentMemoryMib int32 `protobuf:"varint,11,opt,name=max_resident_memory_mib,json=maxResidentMemoryMib,proto3" json:"max_resident_memory_mib,omitempty"` + MeanIoReadKib int64 `protobuf:"varint,13,opt,name=mean_io_read_kib,json=meanIoReadKib,proto3" json:"mean_io_read_kib,omitempty"` + MaxIoReadKib int64 `protobuf:"varint,14,opt,name=max_io_read_kib,json=maxIoReadKib,proto3" json:"max_io_read_kib,omitempty"` + MeanIoWriteKib int64 `protobuf:"varint,15,opt,name=mean_io_write_kib,json=meanIoWriteKib,proto3" json:"mean_io_write_kib,omitempty"` + MaxIoWriteKib int64 `protobuf:"varint,16,opt,name=max_io_write_kib,json=maxIoWriteKib,proto3" json:"max_io_write_kib,omitempty"` } func (x *ClientResourceUsageRecord) Reset() { @@ -171,6 +175,34 @@ func (x *ClientResourceUsageRecord) GetMaxResidentMemoryMib() int32 { return 0 } +func (x *ClientResourceUsageRecord) GetMeanIoReadKib() int64 { + if x != nil { + return x.MeanIoReadKib + } + return 0 +} + +func (x *ClientResourceUsageRecord) GetMaxIoReadKib() int64 { + if x != nil { + return x.MaxIoReadKib + } + return 0 +} + +func (x *ClientResourceUsageRecord) GetMeanIoWriteKib() int64 { + if x != nil { + return x.MeanIoWriteKib + } + return 0 +} + +func (x *ClientResourceUsageRecord) GetMaxIoWriteKib() int64 { + if x != nil { + return x.MaxIoWriteKib + } + return 0 +} + var File_fleetspeak_src_server_proto_fleetspeak_server_resource_proto protoreflect.FileDescriptor var file_fleetspeak_src_server_proto_fleetspeak_server_resource_proto_rawDesc = []byte{ @@ -181,7 +213,7 @@ var file_fleetspeak_src_server_proto_fleetspeak_server_resource_proto_rawDesc = 0x66, 0x6c, 0x65, 0x65, 0x74, 0x73, 0x70, 0x65, 0x61, 0x6b, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x22, 0xf2, 0x04, 0x0a, 0x19, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, + 0x74, 0x6f, 0x22, 0x96, 0x06, 0x0a, 0x19, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x55, 0x73, 0x61, 0x67, 0x65, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x63, 0x6f, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x73, 0x63, 0x6f, 0x70, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x70, 0x69, 0x64, 0x18, 0x02, 0x20, @@ -220,7 +252,18 @@ var file_fleetspeak_src_server_proto_fleetspeak_server_resource_proto_rawDesc = 0x12, 0x35, 0x0a, 0x17, 0x6d, 0x61, 0x78, 0x5f, 0x72, 0x65, 0x73, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x5f, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x5f, 0x6d, 0x69, 0x62, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x05, 0x52, 0x14, 0x6d, 0x61, 0x78, 0x52, 0x65, 0x73, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x4d, 0x65, - 0x6d, 0x6f, 0x72, 0x79, 0x4d, 0x69, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x6d, 0x6f, 0x72, 0x79, 0x4d, 0x69, 0x62, 0x12, 0x27, 0x0a, 0x10, 0x6d, 0x65, 0x61, 0x6e, 0x5f, + 0x69, 0x6f, 0x5f, 0x72, 0x65, 0x61, 0x64, 0x5f, 0x6b, 0x69, 0x62, 0x18, 0x0d, 0x20, 0x01, 0x28, + 0x03, 0x52, 0x0d, 0x6d, 0x65, 0x61, 0x6e, 0x49, 0x6f, 0x52, 0x65, 0x61, 0x64, 0x4b, 0x69, 0x62, + 0x12, 0x25, 0x0a, 0x0f, 0x6d, 0x61, 0x78, 0x5f, 0x69, 0x6f, 0x5f, 0x72, 0x65, 0x61, 0x64, 0x5f, + 0x6b, 0x69, 0x62, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0c, 0x6d, 0x61, 0x78, 0x49, 0x6f, + 0x52, 0x65, 0x61, 0x64, 0x4b, 0x69, 0x62, 0x12, 0x29, 0x0a, 0x11, 0x6d, 0x65, 0x61, 0x6e, 0x5f, + 0x69, 0x6f, 0x5f, 0x77, 0x72, 0x69, 0x74, 0x65, 0x5f, 0x6b, 0x69, 0x62, 0x18, 0x0f, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x0e, 0x6d, 0x65, 0x61, 0x6e, 0x49, 0x6f, 0x57, 0x72, 0x69, 0x74, 0x65, 0x4b, + 0x69, 0x62, 0x12, 0x27, 0x0a, 0x10, 0x6d, 0x61, 0x78, 0x5f, 0x69, 0x6f, 0x5f, 0x77, 0x72, 0x69, + 0x74, 0x65, 0x5f, 0x6b, 0x69, 0x62, 0x18, 0x10, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d, 0x6d, 0x61, + 0x78, 0x49, 0x6f, 0x57, 0x72, 0x69, 0x74, 0x65, 0x4b, 0x69, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x33, } var ( diff --git a/fleetspeak/src/server/proto/fleetspeak_server/resource.proto b/fleetspeak/src/server/proto/fleetspeak_server/resource.proto index 228ef8d5..8165dd44 100644 --- a/fleetspeak/src/server/proto/fleetspeak_server/resource.proto +++ b/fleetspeak/src/server/proto/fleetspeak_server/resource.proto @@ -33,4 +33,9 @@ message ClientResourceUsageRecord { int32 mean_resident_memory_mib = 10; int32 max_resident_memory_mib = 11; + + int64 mean_io_read_kib = 13; + int64 max_io_read_kib = 14; + int64 mean_io_write_kib = 15; + int64 max_io_write_kib = 16; } diff --git a/fleetspeak/src/server/sqlite/clientstore.go b/fleetspeak/src/server/sqlite/clientstore.go index 6d7980c3..dea08146 100644 --- a/fleetspeak/src/server/sqlite/clientstore.go +++ b/fleetspeak/src/server/sqlite/clientstore.go @@ -18,6 +18,7 @@ import ( "context" "database/sql" "fmt" + "math" "strconv" "time" @@ -34,6 +35,7 @@ import ( ) const ( + bytesToKIB = 1.0 / float64(1<<10) bytesToMIB = 1.0 / float64(1<<20) ) @@ -347,7 +349,7 @@ func (d *Datastore) RecordResourceUsageData(ctx context.Context, id common.Clien return d.runInTx(func(tx *sql.Tx) error { _, err := tx.ExecContext( ctx, - "INSERT INTO client_resource_usage_records VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + "INSERT INTO client_resource_usage_records VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", id.String(), rud.Scope, rud.Pid, @@ -360,7 +362,11 @@ func (d *Datastore) RecordResourceUsageData(ctx context.Context, id common.Clien rud.ResourceUsage.MeanSystemCpuRate, rud.ResourceUsage.MaxSystemCpuRate, int32(rud.ResourceUsage.MeanResidentMemory*bytesToMIB), - int32(float64(rud.ResourceUsage.MaxResidentMemory)*bytesToMIB)) + int32(float64(rud.ResourceUsage.MaxResidentMemory)*bytesToMIB), + int64(math.Ceil(rud.ResourceUsage.MeanIoReadBytes*bytesToKIB)), + int64(math.Ceil(float64(rud.ResourceUsage.MaxIoReadBytes)*bytesToKIB)), + int64(math.Ceil(rud.ResourceUsage.MeanIoWriteBytes*bytesToKIB)), + int64(math.Ceil(float64(rud.ResourceUsage.MaxIoWriteBytes)*bytesToKIB))) return err }) } @@ -386,7 +392,8 @@ func (d *Datastore) FetchResourceUsageRecords(ctx context.Context, id common.Cli "SELECT "+ "scope, pid, process_start_time, client_timestamp, server_timestamp, "+ "process_terminated, mean_user_cpu_rate, max_user_cpu_rate, mean_system_cpu_rate, "+ - "max_system_cpu_rate, mean_resident_memory_mib, max_resident_memory_mib "+ + "max_system_cpu_rate, mean_resident_memory_mib, max_resident_memory_mib, "+ + "mean_io_read_kib, max_io_read_kib, mean_io_write_kib, max_io_write_kib "+ "FROM client_resource_usage_records WHERE client_id=? "+ "AND server_timestamp >= ? AND server_timestamp < ?", id.String(), @@ -405,7 +412,8 @@ func (d *Datastore) FetchResourceUsageRecords(ctx context.Context, id common.Cli err := rows.Scan( &record.Scope, &record.Pid, &processStartTime, &clientTimestamp, &serverTimestamp, &record.ProcessTerminated, &record.MeanUserCpuRate, &record.MaxUserCpuRate, &record.MeanSystemCpuRate, - &record.MaxSystemCpuRate, &record.MeanResidentMemoryMib, &record.MaxResidentMemoryMib) + &record.MaxSystemCpuRate, &record.MeanResidentMemoryMib, &record.MaxResidentMemoryMib, + &record.MeanIoReadKib, &record.MaxIoReadKib, &record.MeanIoWriteKib, &record.MaxIoWriteKib) if err != nil { return err diff --git a/fleetspeak/src/server/sqlite/sqlite.go b/fleetspeak/src/server/sqlite/sqlite.go index df9c76b9..6eebe829 100644 --- a/fleetspeak/src/server/sqlite/sqlite.go +++ b/fleetspeak/src/server/sqlite/sqlite.go @@ -135,6 +135,10 @@ mean_system_cpu_rate REAL, max_system_cpu_rate REAL, mean_resident_memory_mib INT4, max_resident_memory_mib INT4, +mean_io_read_kib INT8, +max_io_read_kib INT8, +mean_io_write_kib INT8, +max_io_write_kib INT8, FOREIGN KEY (client_id) REFERENCES clients(client_id))`, `CREATE TABLE IF NOT EXISTS messages( message_id TEXT(64) NOT NULL,