feat(api): add grouped statistics and request_body to log endpoints

Add group_by parameter to /admin/logs/stats endpoint supporting:
- group_by=model: aggregate stats per model with avg latency
- group_by=day: daily aggregation with token counts
- group_by=month: monthly aggregation with token counts

Also include request_body field in admin ListLogs response for
full visibility into logged requests.
This commit is contained in:
zenfun
2025-12-26 11:05:19 +08:00
parent 30f15a84b5
commit 3f1d006006
2 changed files with 295 additions and 6 deletions

View File

@@ -29,6 +29,7 @@ type LogView struct {
RequestSize int64 `json:"request_size"` RequestSize int64 `json:"request_size"`
ResponseSize int64 `json:"response_size"` ResponseSize int64 `json:"response_size"`
AuditReason string `json:"audit_reason"` AuditReason string `json:"audit_reason"`
RequestBody string `json:"request_body,omitempty"`
} }
func toLogView(r model.LogRecord) LogView { func toLogView(r model.LogRecord) LogView {
@@ -50,6 +51,7 @@ func toLogView(r model.LogRecord) LogView {
RequestSize: r.RequestSize, RequestSize: r.RequestSize,
ResponseSize: r.ResponseSize, ResponseSize: r.ResponseSize,
AuditReason: r.AuditReason, AuditReason: r.AuditReason,
RequestBody: r.RequestBody,
} }
} }
@@ -207,11 +209,12 @@ func (h *Handler) ListLogs(c *gin.Context) {
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to list logs", "details": err.Error()}) c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to list logs", "details": err.Error()})
return return
} }
out := make([]MasterLogView, 0, len(rows)) // Admin sees full LogView including request_body
out := make([]LogView, 0, len(rows))
for _, r := range rows { for _, r := range rows {
out = append(out, toMasterLogView(r)) out = append(out, toLogView(r))
} }
c.JSON(http.StatusOK, ListMasterLogsResponse{Total: total, Limit: limit, Offset: offset, Items: out}) c.JSON(http.StatusOK, ListLogsResponse{Total: total, Limit: limit, Offset: offset, Items: out})
} }
type LogStatsResponse struct { type LogStatsResponse struct {
@@ -222,6 +225,26 @@ type LogStatsResponse struct {
ByStatus map[string]int64 `json:"by_status"` ByStatus map[string]int64 `json:"by_status"`
} }
// GroupedStatsItem represents a single group in grouped statistics
type GroupedStatsItem struct {
// For group_by=model
Model string `json:"model,omitempty"`
// For group_by=day
Date string `json:"date,omitempty"`
// For group_by=month
Month string `json:"month,omitempty"`
Count int64 `json:"count"`
TokensIn int64 `json:"tokens_in"`
TokensOut int64 `json:"tokens_out"`
AvgLatencyMs float64 `json:"avg_latency_ms,omitempty"`
}
// GroupedStatsResponse is returned when group_by is specified
type GroupedStatsResponse struct {
Items []GroupedStatsItem `json:"items"`
}
type DeleteLogsRequest struct { type DeleteLogsRequest struct {
Before string `json:"before"` Before string `json:"before"`
KeyID uint `json:"key_id"` KeyID uint `json:"key_id"`
@@ -323,12 +346,13 @@ func (h *Handler) deleteLogsBefore(cutoff time.Time, keyID uint, modelName strin
// LogStats godoc // LogStats godoc
// @Summary Log stats (admin) // @Summary Log stats (admin)
// @Description Aggregate log stats with basic filtering // @Description Aggregate log stats with basic filtering. Use group_by param for grouped statistics.
// @Tags admin // @Tags admin
// @Produce json // @Produce json
// @Security AdminAuth // @Security AdminAuth
// @Param since query int false "unix seconds" // @Param since query int false "unix seconds"
// @Param until query int false "unix seconds" // @Param until query int false "unix seconds"
// @Param group_by query string false "group by dimension: model, day, month"
// @Success 200 {object} LogStatsResponse // @Success 200 {object} LogStatsResponse
// @Failure 500 {object} gin.H // @Failure 500 {object} gin.H
// @Router /admin/logs/stats [get] // @Router /admin/logs/stats [get]
@@ -341,6 +365,20 @@ func (h *Handler) LogStats(c *gin.Context) {
q = q.Where("created_at <= ?", t) q = q.Where("created_at <= ?", t)
} }
groupBy := strings.TrimSpace(c.Query("group_by"))
switch groupBy {
case "model":
h.logStatsByModel(c, q)
return
case "day":
h.logStatsByDay(c, q)
return
case "month":
h.logStatsByMonth(c, q)
return
}
// Default: aggregated stats (backward compatible)
var total int64 var total int64
if err := q.Count(&total).Error; err != nil { if err := q.Count(&total).Error; err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to count logs", "details": err.Error()}) c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to count logs", "details": err.Error()})
@@ -381,6 +419,94 @@ func (h *Handler) LogStats(c *gin.Context) {
}) })
} }
// logStatsByModel handles group_by=model
func (h *Handler) logStatsByModel(c *gin.Context, q *gorm.DB) {
type modelStats struct {
ModelName string
Cnt int64
TokensIn int64
TokensOut int64
AvgLatencyMs float64
}
var rows []modelStats
if err := q.Select(`model_name, COUNT(*) as cnt, COALESCE(SUM(tokens_in),0) as tokens_in, COALESCE(SUM(tokens_out),0) as tokens_out, COALESCE(AVG(latency_ms),0) as avg_latency_ms`).
Group("model_name").
Order("cnt DESC").
Scan(&rows).Error; err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to aggregate by model", "details": err.Error()})
return
}
items := make([]GroupedStatsItem, 0, len(rows))
for _, r := range rows {
items = append(items, GroupedStatsItem{
Model: r.ModelName,
Count: r.Cnt,
TokensIn: r.TokensIn,
TokensOut: r.TokensOut,
AvgLatencyMs: r.AvgLatencyMs,
})
}
c.JSON(http.StatusOK, GroupedStatsResponse{Items: items})
}
// logStatsByDay handles group_by=day
func (h *Handler) logStatsByDay(c *gin.Context, q *gorm.DB) {
type dayStats struct {
Day string
Cnt int64
TokensIn int64
TokensOut int64
}
var rows []dayStats
// PostgreSQL DATE function; SQLite uses date()
if err := q.Select(`TO_CHAR(created_at, 'YYYY-MM-DD') as day, COUNT(*) as cnt, COALESCE(SUM(tokens_in),0) as tokens_in, COALESCE(SUM(tokens_out),0) as tokens_out`).
Group("day").
Order("day ASC").
Scan(&rows).Error; err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to aggregate by day", "details": err.Error()})
return
}
items := make([]GroupedStatsItem, 0, len(rows))
for _, r := range rows {
items = append(items, GroupedStatsItem{
Date: r.Day,
Count: r.Cnt,
TokensIn: r.TokensIn,
TokensOut: r.TokensOut,
})
}
c.JSON(http.StatusOK, GroupedStatsResponse{Items: items})
}
// logStatsByMonth handles group_by=month
func (h *Handler) logStatsByMonth(c *gin.Context, q *gorm.DB) {
type monthStats struct {
Month string
Cnt int64
TokensIn int64
TokensOut int64
}
var rows []monthStats
// PostgreSQL format
if err := q.Select(`TO_CHAR(created_at, 'YYYY-MM') as month, COUNT(*) as cnt, COALESCE(SUM(tokens_in),0) as tokens_in, COALESCE(SUM(tokens_out),0) as tokens_out`).
Group("month").
Order("month ASC").
Scan(&rows).Error; err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to aggregate by month", "details": err.Error()})
return
}
items := make([]GroupedStatsItem, 0, len(rows))
for _, r := range rows {
items = append(items, GroupedStatsItem{
Month: r.Month,
Count: r.Cnt,
TokensIn: r.TokensIn,
TokensOut: r.TokensOut,
})
}
c.JSON(http.StatusOK, GroupedStatsResponse{Items: items})
}
// ListSelfLogs godoc // ListSelfLogs godoc
// @Summary List logs (master) // @Summary List logs (master)
// @Description List request logs for the authenticated master // @Description List request logs for the authenticated master

View File

@@ -191,3 +191,166 @@ func TestAdmin_DeleteLogs_RequiresBefore(t *testing.T) {
t.Fatalf("expected 400, got %d body=%s", rr.Code, rr.Body.String()) t.Fatalf("expected 400, got %d body=%s", rr.Code, rr.Body.String())
} }
} }
func TestAdmin_ListLogs_IncludesRequestBody(t *testing.T) {
gin.SetMode(gin.TestMode)
dsn := fmt.Sprintf("file:%s?mode=memory&cache=shared", t.Name())
db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{})
if err != nil {
t.Fatalf("open sqlite: %v", err)
}
if err := db.AutoMigrate(&model.LogRecord{}); err != nil {
t.Fatalf("migrate: %v", err)
}
// Create log with request_body
log1 := model.LogRecord{
KeyID: 1,
ModelName: "gpt-4",
StatusCode: 200,
RequestBody: `{"messages":[{"role":"user","content":"hello"}]}`,
}
if err := db.Create(&log1).Error; err != nil {
t.Fatalf("create log: %v", err)
}
h := &Handler{db: db, logDB: db}
r := gin.New()
r.GET("/admin/logs", h.ListLogs)
req := httptest.NewRequest(http.MethodGet, "/admin/logs", nil)
rr := httptest.NewRecorder()
r.ServeHTTP(rr, req)
if rr.Code != http.StatusOK {
t.Fatalf("expected 200, got %d body=%s", rr.Code, rr.Body.String())
}
var resp ListLogsResponse
if err := json.Unmarshal(rr.Body.Bytes(), &resp); err != nil {
t.Fatalf("unmarshal: %v", err)
}
if resp.Total != 1 || len(resp.Items) != 1 {
t.Fatalf("expected 1 item, got total=%d len=%d", resp.Total, len(resp.Items))
}
if resp.Items[0].RequestBody != log1.RequestBody {
t.Fatalf("expected request_body=%q, got %q", log1.RequestBody, resp.Items[0].RequestBody)
}
}
func TestLogStats_GroupByModel(t *testing.T) {
gin.SetMode(gin.TestMode)
dsn := fmt.Sprintf("file:%s?mode=memory&cache=shared", t.Name())
db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{})
if err != nil {
t.Fatalf("open sqlite: %v", err)
}
if err := db.AutoMigrate(&model.LogRecord{}); err != nil {
t.Fatalf("migrate: %v", err)
}
// Create logs with different models
logs := []model.LogRecord{
{KeyID: 1, ModelName: "gpt-4", TokensIn: 100, TokensOut: 200, StatusCode: 200, LatencyMs: 100},
{KeyID: 1, ModelName: "gpt-4", TokensIn: 150, TokensOut: 300, StatusCode: 200, LatencyMs: 200},
{KeyID: 1, ModelName: "claude-3", TokensIn: 80, TokensOut: 160, StatusCode: 200, LatencyMs: 150},
}
for _, log := range logs {
if err := db.Create(&log).Error; err != nil {
t.Fatalf("create log: %v", err)
}
}
h := &Handler{db: db, logDB: db}
r := gin.New()
r.GET("/admin/logs/stats", h.LogStats)
req := httptest.NewRequest(http.MethodGet, "/admin/logs/stats?group_by=model", nil)
rr := httptest.NewRecorder()
r.ServeHTTP(rr, req)
if rr.Code != http.StatusOK {
t.Fatalf("expected 200, got %d body=%s", rr.Code, rr.Body.String())
}
var resp GroupedStatsResponse
if err := json.Unmarshal(rr.Body.Bytes(), &resp); err != nil {
t.Fatalf("unmarshal: %v", err)
}
if len(resp.Items) != 2 {
t.Fatalf("expected 2 groups, got %d: %+v", len(resp.Items), resp.Items)
}
// Should be sorted by count DESC, so gpt-4 first (2 logs) then claude-3 (1 log)
if resp.Items[0].Model != "gpt-4" {
t.Fatalf("expected first group to be gpt-4, got %s", resp.Items[0].Model)
}
if resp.Items[0].Count != 2 {
t.Fatalf("expected gpt-4 count=2, got %d", resp.Items[0].Count)
}
if resp.Items[0].TokensIn != 250 {
t.Fatalf("expected gpt-4 tokens_in=250, got %d", resp.Items[0].TokensIn)
}
if resp.Items[1].Model != "claude-3" {
t.Fatalf("expected second group to be claude-3, got %s", resp.Items[1].Model)
}
if resp.Items[1].Count != 1 {
t.Fatalf("expected claude-3 count=1, got %d", resp.Items[1].Count)
}
}
func TestLogStats_DefaultBehavior(t *testing.T) {
gin.SetMode(gin.TestMode)
dsn := fmt.Sprintf("file:%s?mode=memory&cache=shared", t.Name())
db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{})
if err != nil {
t.Fatalf("open sqlite: %v", err)
}
if err := db.AutoMigrate(&model.LogRecord{}); err != nil {
t.Fatalf("migrate: %v", err)
}
// Create some logs
logs := []model.LogRecord{
{KeyID: 1, ModelName: "gpt-4", TokensIn: 100, TokensOut: 200, StatusCode: 200, LatencyMs: 100},
{KeyID: 1, ModelName: "gpt-4", TokensIn: 150, TokensOut: 300, StatusCode: 500, LatencyMs: 200},
}
for _, log := range logs {
if err := db.Create(&log).Error; err != nil {
t.Fatalf("create log: %v", err)
}
}
h := &Handler{db: db, logDB: db}
r := gin.New()
r.GET("/admin/logs/stats", h.LogStats)
// Without group_by, should return aggregated stats
req := httptest.NewRequest(http.MethodGet, "/admin/logs/stats", nil)
rr := httptest.NewRecorder()
r.ServeHTTP(rr, req)
if rr.Code != http.StatusOK {
t.Fatalf("expected 200, got %d body=%s", rr.Code, rr.Body.String())
}
var resp LogStatsResponse
if err := json.Unmarshal(rr.Body.Bytes(), &resp); err != nil {
t.Fatalf("unmarshal: %v", err)
}
if resp.Total != 2 {
t.Fatalf("expected total=2, got %d", resp.Total)
}
if resp.TokensIn != 250 {
t.Fatalf("expected tokens_in=250, got %d", resp.TokensIn)
}
if resp.TokensOut != 500 {
t.Fatalf("expected tokens_out=500, got %d", resp.TokensOut)
}
if len(resp.ByStatus) != 2 {
t.Fatalf("expected 2 status buckets, got %d", len(resp.ByStatus))
}
if resp.ByStatus["200"] != 1 || resp.ByStatus["500"] != 1 {
t.Fatalf("unexpected by_status: %+v", resp.ByStatus)
}
}