ipng/nginx-logtail
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add aggregator backfill, pulling fine+coarse buckets from collectors

Browse Source
This commit is contained in:
Pim van Pelt
2026-03-25 07:03:46 +01:00
parent d2dcd88c4b
commit eddb04ced4
11 changed files with 419 additions and 1384 deletions
Download Patch File Download Diff File Expand all files Collapse all files

130
proto/logtailpb/logtail.pb.go
View File

@@ -731,7 +731,8 @@ type Snapshot struct {
state protoimpl.MessageState `protogen:"open.v1"`
Source string `protobuf:"bytes,1,opt,name=source,proto3" json:"source,omitempty"`
Timestamp int64 `protobuf:"varint,2,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
Entries []*TopNEntry `protobuf:"bytes,3,rep,name=entries,proto3" json:"entries,omitempty"` // top-50K for this 1-minute bucket, sorted desc
Entries []*TopNEntry `protobuf:"bytes,3,rep,name=entries,proto3" json:"entries,omitempty"` // top-50K for this 1-minute bucket, sorted desc
IsCoarse bool `protobuf:"varint,4,opt,name=is_coarse,json=isCoarse,proto3" json:"is_coarse,omitempty"` // true for coarse-ring (5-min) buckets in DumpSnapshots
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
@@ -787,6 +788,49 @@ func (x *Snapshot) GetEntries() []*TopNEntry {
return nil
}
func (x *Snapshot) GetIsCoarse() bool {
if x != nil {
return x.IsCoarse
}
return false
}
type DumpSnapshotsRequest struct {
state protoimpl.MessageState `protogen:"open.v1"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *DumpSnapshotsRequest) Reset() {
*x = DumpSnapshotsRequest{}
mi := &file_proto_logtail_proto_msgTypes[9]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *DumpSnapshotsRequest) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*DumpSnapshotsRequest) ProtoMessage() {}
func (x *DumpSnapshotsRequest) ProtoReflect() protoreflect.Message {
mi := &file_proto_logtail_proto_msgTypes[9]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use DumpSnapshotsRequest.ProtoReflect.Descriptor instead.
func (*DumpSnapshotsRequest) Descriptor() ([]byte, []int) {
return file_proto_logtail_proto_rawDescGZIP(), []int{9}
}
type ListTargetsRequest struct {
state protoimpl.MessageState `protogen:"open.v1"`
unknownFields protoimpl.UnknownFields
@@ -795,7 +839,7 @@ type ListTargetsRequest struct {
func (x *ListTargetsRequest) Reset() {
*x = ListTargetsRequest{}
mi := &file_proto_logtail_proto_msgTypes[9]
mi := &file_proto_logtail_proto_msgTypes[10]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -807,7 +851,7 @@ func (x *ListTargetsRequest) String() string {
func (*ListTargetsRequest) ProtoMessage() {}
func (x *ListTargetsRequest) ProtoReflect() protoreflect.Message {
mi := &file_proto_logtail_proto_msgTypes[9]
mi := &file_proto_logtail_proto_msgTypes[10]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -820,7 +864,7 @@ func (x *ListTargetsRequest) ProtoReflect() protoreflect.Message {
// Deprecated: Use ListTargetsRequest.ProtoReflect.Descriptor instead.
func (*ListTargetsRequest) Descriptor() ([]byte, []int) {
return file_proto_logtail_proto_rawDescGZIP(), []int{9}
return file_proto_logtail_proto_rawDescGZIP(), []int{10}
}
type TargetInfo struct {
@@ -833,7 +877,7 @@ type TargetInfo struct {
func (x *TargetInfo) Reset() {
*x = TargetInfo{}
mi := &file_proto_logtail_proto_msgTypes[10]
mi := &file_proto_logtail_proto_msgTypes[11]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -845,7 +889,7 @@ func (x *TargetInfo) String() string {
func (*TargetInfo) ProtoMessage() {}
func (x *TargetInfo) ProtoReflect() protoreflect.Message {
mi := &file_proto_logtail_proto_msgTypes[10]
mi := &file_proto_logtail_proto_msgTypes[11]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -858,7 +902,7 @@ func (x *TargetInfo) ProtoReflect() protoreflect.Message {
// Deprecated: Use TargetInfo.ProtoReflect.Descriptor instead.
func (*TargetInfo) Descriptor() ([]byte, []int) {
return file_proto_logtail_proto_rawDescGZIP(), []int{10}
return file_proto_logtail_proto_rawDescGZIP(), []int{11}
}
func (x *TargetInfo) GetName() string {
@@ -884,7 +928,7 @@ type ListTargetsResponse struct {
func (x *ListTargetsResponse) Reset() {
*x = ListTargetsResponse{}
mi := &file_proto_logtail_proto_msgTypes[11]
mi := &file_proto_logtail_proto_msgTypes[12]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -896,7 +940,7 @@ func (x *ListTargetsResponse) String() string {
func (*ListTargetsResponse) ProtoMessage() {}
func (x *ListTargetsResponse) ProtoReflect() protoreflect.Message {
mi := &file_proto_logtail_proto_msgTypes[11]
mi := &file_proto_logtail_proto_msgTypes[12]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -909,7 +953,7 @@ func (x *ListTargetsResponse) ProtoReflect() protoreflect.Message {
// Deprecated: Use ListTargetsResponse.ProtoReflect.Descriptor instead.
func (*ListTargetsResponse) Descriptor() ([]byte, []int) {
return file_proto_logtail_proto_rawDescGZIP(), []int{11}
return file_proto_logtail_proto_rawDescGZIP(), []int{12}
}
func (x *ListTargetsResponse) GetTargets() []*TargetInfo {
@@ -967,11 +1011,13 @@ const file_proto_logtail_proto_rawDesc = "" +
"\rTrendResponse\x12+\n" +
"\x06points\x18\x01 \x03(\v2\x13.logtail.TrendPointR\x06points\x12\x16\n" +
"\x06source\x18\x02 \x01(\tR\x06source\"\x11\n" +
"\x0fSnapshotRequest\"n\n" +
"\x0fSnapshotRequest\"\x8b\x01\n" +
"\bSnapshot\x12\x16\n" +
"\x06source\x18\x01 \x01(\tR\x06source\x12\x1c\n" +
"\ttimestamp\x18\x02 \x01(\x03R\ttimestamp\x12,\n" +
"\aentries\x18\x03 \x03(\v2\x12.logtail.TopNEntryR\aentries\"\x14\n" +
"\aentries\x18\x03 \x03(\v2\x12.logtail.TopNEntryR\aentries\x12\x1b\n" +
"\tis_coarse\x18\x04 \x01(\bR\bisCoarse\"\x16\n" +
"\x14DumpSnapshotsRequest\"\x14\n" +
"\x12ListTargetsRequest\"4\n" +
"\n" +
"TargetInfo\x12\x12\n" +
@@ -1004,12 +1050,13 @@ const file_proto_logtail_proto_rawDesc = "" +
"\x04W15M\x10\x02\x12\b\n" +
"\x04W60M\x10\x03\x12\a\n" +
"\x03W6H\x10\x04\x12\b\n" +
"\x04W24H\x10\x052\x89\x02\n" +
"\x04W24H\x10\x052\xce\x02\n" +
"\x0eLogtailService\x123\n" +
"\x04TopN\x12\x14.logtail.TopNRequest\x1a\x15.logtail.TopNResponse\x126\n" +
"\x05Trend\x12\x15.logtail.TrendRequest\x1a\x16.logtail.TrendResponse\x12@\n" +
"\x0fStreamSnapshots\x12\x18.logtail.SnapshotRequest\x1a\x11.logtail.Snapshot0\x01\x12H\n" +
"\vListTargets\x12\x1b.logtail.ListTargetsRequest\x1a\x1c.logtail.ListTargetsResponseB0Z.git.ipng.ch/ipng/nginx-logtail/proto/logtailpbb\x06proto3"
"\vListTargets\x12\x1b.logtail.ListTargetsRequest\x1a\x1c.logtail.ListTargetsResponse\x12C\n" +
"\rDumpSnapshots\x12\x1d.logtail.DumpSnapshotsRequest\x1a\x11.logtail.Snapshot0\x01B0Z.git.ipng.ch/ipng/nginx-logtail/proto/logtailpbb\x06proto3"
var (
file_proto_logtail_proto_rawDescOnce sync.Once
@@ -1024,24 +1071,25 @@ func file_proto_logtail_proto_rawDescGZIP() []byte {
}
var file_proto_logtail_proto_enumTypes = make([]protoimpl.EnumInfo, 4)
var file_proto_logtail_proto_msgTypes = make([]protoimpl.MessageInfo, 12)
var file_proto_logtail_proto_msgTypes = make([]protoimpl.MessageInfo, 13)
var file_proto_logtail_proto_goTypes = []any{
(TorFilter)(0), // 0: logtail.TorFilter
(StatusOp)(0), // 1: logtail.StatusOp
(GroupBy)(0), // 2: logtail.GroupBy
(Window)(0), // 3: logtail.Window
(*Filter)(nil), // 4: logtail.Filter
(*TopNRequest)(nil), // 5: logtail.TopNRequest
(*TopNEntry)(nil), // 6: logtail.TopNEntry
(*TopNResponse)(nil), // 7: logtail.TopNResponse
(*TrendRequest)(nil), // 8: logtail.TrendRequest
(*TrendPoint)(nil), // 9: logtail.TrendPoint
(*TrendResponse)(nil), // 10: logtail.TrendResponse
(*SnapshotRequest)(nil), // 11: logtail.SnapshotRequest
(*Snapshot)(nil), // 12: logtail.Snapshot
(*ListTargetsRequest)(nil), // 13: logtail.ListTargetsRequest
(*TargetInfo)(nil), // 14: logtail.TargetInfo
(*ListTargetsResponse)(nil), // 15: logtail.ListTargetsResponse
(TorFilter)(0), // 0: logtail.TorFilter
(StatusOp)(0), // 1: logtail.StatusOp
(GroupBy)(0), // 2: logtail.GroupBy
(Window)(0), // 3: logtail.Window
(*Filter)(nil), // 4: logtail.Filter
(*TopNRequest)(nil), // 5: logtail.TopNRequest
(*TopNEntry)(nil), // 6: logtail.TopNEntry
(*TopNResponse)(nil), // 7: logtail.TopNResponse
(*TrendRequest)(nil), // 8: logtail.TrendRequest
(*TrendPoint)(nil), // 9: logtail.TrendPoint
(*TrendResponse)(nil), // 10: logtail.TrendResponse
(*SnapshotRequest)(nil), // 11: logtail.SnapshotRequest
(*Snapshot)(nil), // 12: logtail.Snapshot
(*DumpSnapshotsRequest)(nil), // 13: logtail.DumpSnapshotsRequest
(*ListTargetsRequest)(nil), // 14: logtail.ListTargetsRequest
(*TargetInfo)(nil), // 15: logtail.TargetInfo
(*ListTargetsResponse)(nil), // 16: logtail.ListTargetsResponse
}
var file_proto_logtail_proto_depIdxs = []int32{
1, // 0: logtail.Filter.status_op:type_name -> logtail.StatusOp
@@ -1055,17 +1103,19 @@ var file_proto_logtail_proto_depIdxs = []int32{
3, // 8: logtail.TrendRequest.window:type_name -> logtail.Window
9, // 9: logtail.TrendResponse.points:type_name -> logtail.TrendPoint
6, // 10: logtail.Snapshot.entries:type_name -> logtail.TopNEntry
14, // 11: logtail.ListTargetsResponse.targets:type_name -> logtail.TargetInfo
15, // 11: logtail.ListTargetsResponse.targets:type_name -> logtail.TargetInfo
5, // 12: logtail.LogtailService.TopN:input_type -> logtail.TopNRequest
8, // 13: logtail.LogtailService.Trend:input_type -> logtail.TrendRequest
11, // 14: logtail.LogtailService.StreamSnapshots:input_type -> logtail.SnapshotRequest
13, // 15: logtail.LogtailService.ListTargets:input_type -> logtail.ListTargetsRequest
7, // 16: logtail.LogtailService.TopN:output_type -> logtail.TopNResponse
10, // 17: logtail.LogtailService.Trend:output_type -> logtail.TrendResponse
12, // 18: logtail.LogtailService.StreamSnapshots:output_type -> logtail.Snapshot
15, // 19: logtail.LogtailService.ListTargets:output_type -> logtail.ListTargetsResponse
16, // [16:20] is the sub-list for method output_type
12, // [12:16] is the sub-list for method input_type
14, // 15: logtail.LogtailService.ListTargets:input_type -> logtail.ListTargetsRequest
13, // 16: logtail.LogtailService.DumpSnapshots:input_type -> logtail.DumpSnapshotsRequest
7, // 17: logtail.LogtailService.TopN:output_type -> logtail.TopNResponse
10, // 18: logtail.LogtailService.Trend:output_type -> logtail.TrendResponse
12, // 19: logtail.LogtailService.StreamSnapshots:output_type -> logtail.Snapshot
16, // 20: logtail.LogtailService.ListTargets:output_type -> logtail.ListTargetsResponse
12, // 21: logtail.LogtailService.DumpSnapshots:output_type -> logtail.Snapshot
17, // [17:22] is the sub-list for method output_type
12, // [12:17] is the sub-list for method input_type
12, // [12:12] is the sub-list for extension type_name
12, // [12:12] is the sub-list for extension extendee
0, // [0:12] is the sub-list for field type_name
@@ -1083,7 +1133,7 @@ func file_proto_logtail_proto_init() {
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: unsafe.Slice(unsafe.StringData(file_proto_logtail_proto_rawDesc), len(file_proto_logtail_proto_rawDesc)),
NumEnums: 4,
NumMessages: 12,
NumMessages: 13,
NumExtensions: 0,
NumServices: 1,
},

41
proto/logtailpb/logtail_grpc.pb.go
View File

@@ -23,6 +23,7 @@ const (
LogtailService_Trend_FullMethodName = "/logtail.LogtailService/Trend"
LogtailService_StreamSnapshots_FullMethodName = "/logtail.LogtailService/StreamSnapshots"
LogtailService_ListTargets_FullMethodName = "/logtail.LogtailService/ListTargets"
LogtailService_DumpSnapshots_FullMethodName = "/logtail.LogtailService/DumpSnapshots"
)
// LogtailServiceClient is the client API for LogtailService service.
@@ -33,6 +34,7 @@ type LogtailServiceClient interface {
Trend(ctx context.Context, in *TrendRequest, opts ...grpc.CallOption) (*TrendResponse, error)
StreamSnapshots(ctx context.Context, in *SnapshotRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[Snapshot], error)
ListTargets(ctx context.Context, in *ListTargetsRequest, opts ...grpc.CallOption) (*ListTargetsResponse, error)
DumpSnapshots(ctx context.Context, in *DumpSnapshotsRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[Snapshot], error)
}
type logtailServiceClient struct {
@@ -92,6 +94,25 @@ func (c *logtailServiceClient) ListTargets(ctx context.Context, in *ListTargetsR
return out, nil
}
func (c *logtailServiceClient) DumpSnapshots(ctx context.Context, in *DumpSnapshotsRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[Snapshot], error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
stream, err := c.cc.NewStream(ctx, &LogtailService_ServiceDesc.Streams[1], LogtailService_DumpSnapshots_FullMethodName, cOpts...)
if err != nil {
return nil, err
}
x := &grpc.GenericClientStream[DumpSnapshotsRequest, Snapshot]{ClientStream: stream}
if err := x.ClientStream.SendMsg(in); err != nil {
return nil, err
}
if err := x.ClientStream.CloseSend(); err != nil {
return nil, err
}
return x, nil
}
// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
type LogtailService_DumpSnapshotsClient = grpc.ServerStreamingClient[Snapshot]
// LogtailServiceServer is the server API for LogtailService service.
// All implementations must embed UnimplementedLogtailServiceServer
// for forward compatibility.
@@ -100,6 +121,7 @@ type LogtailServiceServer interface {
Trend(context.Context, *TrendRequest) (*TrendResponse, error)
StreamSnapshots(*SnapshotRequest, grpc.ServerStreamingServer[Snapshot]) error
ListTargets(context.Context, *ListTargetsRequest) (*ListTargetsResponse, error)
DumpSnapshots(*DumpSnapshotsRequest, grpc.ServerStreamingServer[Snapshot]) error
mustEmbedUnimplementedLogtailServiceServer()
}
@@ -122,6 +144,9 @@ func (UnimplementedLogtailServiceServer) StreamSnapshots(*SnapshotRequest, grpc.
func (UnimplementedLogtailServiceServer) ListTargets(context.Context, *ListTargetsRequest) (*ListTargetsResponse, error) {
return nil, status.Error(codes.Unimplemented, "method ListTargets not implemented")
}
func (UnimplementedLogtailServiceServer) DumpSnapshots(*DumpSnapshotsRequest, grpc.ServerStreamingServer[Snapshot]) error {
return status.Error(codes.Unimplemented, "method DumpSnapshots not implemented")
}
func (UnimplementedLogtailServiceServer) mustEmbedUnimplementedLogtailServiceServer() {}
func (UnimplementedLogtailServiceServer) testEmbeddedByValue() {}
@@ -208,6 +233,17 @@ func _LogtailService_ListTargets_Handler(srv interface{}, ctx context.Context, d
return interceptor(ctx, in, info, handler)
}
func _LogtailService_DumpSnapshots_Handler(srv interface{}, stream grpc.ServerStream) error {
m := new(DumpSnapshotsRequest)
if err := stream.RecvMsg(m); err != nil {
return err
}
return srv.(LogtailServiceServer).DumpSnapshots(m, &grpc.GenericServerStream[DumpSnapshotsRequest, Snapshot]{ServerStream: stream})
}
// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
type LogtailService_DumpSnapshotsServer = grpc.ServerStreamingServer[Snapshot]
// LogtailService_ServiceDesc is the grpc.ServiceDesc for LogtailService service.
// It's only intended for direct use with grpc.RegisterService,
// and not to be introspected or modified (even as a copy)
@@ -234,6 +270,11 @@ var LogtailService_ServiceDesc = grpc.ServiceDesc{
Handler: _LogtailService_StreamSnapshots_Handler,
ServerStreams: true,
},
{
StreamName: "DumpSnapshots",
Handler: _LogtailService_DumpSnapshots_Handler,
ServerStreams: true,
},
},
Metadata: "proto/logtail.proto",
}