ruochen/ez-api
1
0
Fork 0
mirror of https://github.com/EZ-Api/ez-api.git synced 2026-01-13 17:47:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(arch): add log partitioning and provider delete sync

Browse Source
This commit is contained in:
zenfun
2025-12-21 20:45:16 +08:00
parent f819f89ba2
commit 816ea93339
23 changed files with 582 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
internal/api/admin_handler.go
View File

@@ -13,17 +13,18 @@ import (
)
type AdminHandler struct {
db *gorm.DB
logDB *gorm.DB
masterService *service.MasterService
syncService *service.SyncService
db *gorm.DB
logDB *gorm.DB
masterService *service.MasterService
syncService *service.SyncService
logPartitioner *service.LogPartitioner
}
func NewAdminHandler(db *gorm.DB, logDB *gorm.DB, masterService *service.MasterService, syncService *service.SyncService) *AdminHandler {
func NewAdminHandler(db *gorm.DB, logDB *gorm.DB, masterService *service.MasterService, syncService *service.SyncService, partitioner *service.LogPartitioner) *AdminHandler {
if logDB == nil {
logDB = db
}
return &AdminHandler{db: db, logDB: logDB, masterService: masterService, syncService: syncService}
return &AdminHandler{db: db, logDB: logDB, masterService: masterService, syncService: syncService, logPartitioner: partitioner}
}
func (h *AdminHandler) logDBConn() *gorm.DB {
@@ -33,6 +34,10 @@ func (h *AdminHandler) logDBConn() *gorm.DB {
return h.logDB
}
func (h *AdminHandler) logBaseQuery() *gorm.DB {
return logBaseQuery(h.logDBConn(), h.logPartitioner)
}
type CreateMasterRequest struct {
Name string `json:"name" binding:"required"`
Group string `json:"group" binding:"required"`

2
internal/api/admin_issue_key_test.go
View File

@@ -34,7 +34,7 @@ func TestAdmin_IssueChildKeyForMaster_IssuedByAdminAndSynced(t *testing.T) {
syncService := service.NewSyncService(rdb)
masterService := service.NewMasterService(db)
adminHandler := NewAdminHandler(db, db, masterService, syncService)
adminHandler := NewAdminHandler(db, db, masterService, syncService, nil)
m, _, err := masterService.CreateMaster("m1", "default", 5, 10)
if err != nil {

32
internal/api/handler.go
View File

@@ -17,25 +17,27 @@ import (
)
type Handler struct {
db *gorm.DB
logDB *gorm.DB
sync *service.SyncService
logger *service.LogWriter
rdb *redis.Client
logWebhook *service.LogWebhookService
db *gorm.DB
logDB *gorm.DB
sync *service.SyncService
logger *service.LogWriter
rdb *redis.Client
logWebhook *service.LogWebhookService
logPartitioner *service.LogPartitioner
}
func NewHandler(db *gorm.DB, logDB *gorm.DB, sync *service.SyncService, logger *service.LogWriter, rdb *redis.Client) *Handler {
func NewHandler(db *gorm.DB, logDB *gorm.DB, sync *service.SyncService, logger *service.LogWriter, rdb *redis.Client, partitioner *service.LogPartitioner) *Handler {
if logDB == nil {
logDB = db
}
return &Handler{
db: db,
logDB: logDB,
sync: sync,
logger: logger,
rdb: rdb,
logWebhook: service.NewLogWebhookService(rdb),
db: db,
logDB: logDB,
sync: sync,
logger: logger,
rdb: rdb,
logWebhook: service.NewLogWebhookService(rdb),
logPartitioner: partitioner,
}
}
@@ -46,6 +48,10 @@ func (h *Handler) logDBConn() *gorm.DB {
return h.logDB
}
func (h *Handler) logBaseQuery() *gorm.DB {
return logBaseQuery(h.logDBConn(), h.logPartitioner)
}
// CreateKey is now handled by MasterHandler
// CreateProvider godoc

78
internal/api/log_handler.go
View File

@@ -132,8 +132,9 @@ func parseUnixSeconds(raw string) (time.Time, bool) {
func (h *MasterHandler) masterLogBase(masterID uint) (*gorm.DB, error) {
logDB := h.logDBConn()
base := h.logBaseQuery()
if logDB == h.db {
return logDB.Model(&model.LogRecord{}).
return base.
Joins("JOIN keys ON keys.id = log_records.key_id").
Where("keys.master_id = ?", masterID), nil
}
@@ -144,9 +145,9 @@ func (h *MasterHandler) masterLogBase(masterID uint) (*gorm.DB, error) {
return nil, err
}
if len(keyIDs) == 0 {
return logDB.Model(&model.LogRecord{}).Where("1 = 0"), nil
return base.Where("1 = 0"), nil
}
return logDB.Model(&model.LogRecord{}).
return base.
Where("log_records.key_id IN ?", keyIDs), nil
}
@@ -170,7 +171,7 @@ func (h *MasterHandler) masterLogBase(masterID uint) (*gorm.DB, error) {
func (h *Handler) ListLogs(c *gin.Context) {
limit, offset := parseLimitOffset(c)
q := h.logDBConn().Model(&model.LogRecord{})
q := h.logBaseQuery()
if t, ok := parseUnixSeconds(c.Query("since")); ok {
q = q.Where("created_at >= ?", t)
@@ -261,20 +262,63 @@ func (h *Handler) DeleteLogs(c *gin.Context) {
return
}
q := h.logDBConn().Unscoped().Where("created_at < ?", ts.UTC())
if req.KeyID > 0 {
q = q.Where("key_id = ?", req.KeyID)
}
if model := strings.TrimSpace(req.Model); model != "" {
q = q.Where("model_name = ?", model)
}
res := q.Delete(&model.LogRecord{})
if res.Error != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to delete logs", "details": res.Error.Error()})
deleted, err := h.deleteLogsBefore(ts.UTC(), req.KeyID, req.Model)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to delete logs", "details": err.Error()})
return
}
c.JSON(http.StatusOK, DeleteLogsResponse{DeletedCount: res.RowsAffected})
c.JSON(http.StatusOK, DeleteLogsResponse{DeletedCount: deleted})
}
func (h *Handler) deleteLogsBefore(cutoff time.Time, keyID uint, modelName string) (int64, error) {
modelName = strings.TrimSpace(modelName)
if h == nil || h.logPartitioner == nil || !h.logPartitioner.Enabled() {
q := h.logBaseQuery().Unscoped().Where("created_at < ?", cutoff)
if keyID > 0 {
q = q.Where("key_id = ?", keyID)
}
if modelName != "" {
q = q.Where("model_name = ?", modelName)
}
res := q.Delete(&model.LogRecord{})
return res.RowsAffected, res.Error
}
partitions, err := h.logPartitioner.ListPartitions()
if err != nil {
return 0, err
}
if len(partitions) == 0 {
q := h.logDBConn().Table("log_records").Unscoped().Where("created_at < ?", cutoff)
if keyID > 0 {
q = q.Where("key_id = ?", keyID)
}
if modelName != "" {
q = q.Where("model_name = ?", modelName)
}
res := q.Delete(&model.LogRecord{})
return res.RowsAffected, res.Error
}
var deleted int64
for _, part := range partitions {
if !part.Start.Before(cutoff) {
continue
}
q := h.logDBConn().Table(part.Table).Unscoped().Where("created_at < ?", cutoff)
if keyID > 0 {
q = q.Where("key_id = ?", keyID)
}
if modelName != "" {
q = q.Where("model_name = ?", modelName)
}
res := q.Delete(&model.LogRecord{})
if res.Error != nil {
return deleted, res.Error
}
deleted += res.RowsAffected
}
return deleted, nil
}
// LogStats godoc
@@ -289,7 +333,7 @@ func (h *Handler) DeleteLogs(c *gin.Context) {
// @Failure 500 {object} gin.H
// @Router /admin/logs/stats [get]
func (h *Handler) LogStats(c *gin.Context) {
q := h.logDBConn().Model(&model.LogRecord{})
q := h.logBaseQuery()
if t, ok := parseUnixSeconds(c.Query("since")); ok {
q = q.Where("created_at >= ?", t)
}

17
internal/api/log_query.go Normal file
View File

@@ -0,0 +1,17 @@
package api
import (
"github.com/ez-api/ez-api/internal/model"
"github.com/ez-api/ez-api/internal/service"
"gorm.io/gorm"
)
func logBaseQuery(db *gorm.DB, partitioner *service.LogPartitioner) *gorm.DB {
if db == nil {
return db
}
if partitioner != nil && partitioner.Enabled() {
return db.Table(partitioner.ViewName() + " as log_records")
}
return db.Model(&model.LogRecord{})
}

2
internal/api/log_webhook_handler_test.go
View File

@@ -33,7 +33,7 @@ func newTestHandlerWithWebhook(t *testing.T) (*Handler, *miniredis.Miniredis) {
mr := miniredis.RunT(t)
rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})
sync := service.NewSyncService(rdb)
return NewHandler(db, db, sync, nil, rdb), mr
return NewHandler(db, db, sync, nil, rdb, nil), mr
}
func TestLogWebhookConfigCRUD(t *testing.T) {

17
internal/api/master_handler.go
View File

@@ -14,17 +14,18 @@ import (
)
type MasterHandler struct {
db *gorm.DB
logDB *gorm.DB
masterService *service.MasterService
syncService *service.SyncService
db *gorm.DB
logDB *gorm.DB
masterService *service.MasterService
syncService *service.SyncService
logPartitioner *service.LogPartitioner
}
func NewMasterHandler(db *gorm.DB, logDB *gorm.DB, masterService *service.MasterService, syncService *service.SyncService) *MasterHandler {
func NewMasterHandler(db *gorm.DB, logDB *gorm.DB, masterService *service.MasterService, syncService *service.SyncService, partitioner *service.LogPartitioner) *MasterHandler {
if logDB == nil {
logDB = db
}
return &MasterHandler{db: db, logDB: logDB, masterService: masterService, syncService: syncService}
return &MasterHandler{db: db, logDB: logDB, masterService: masterService, syncService: syncService, logPartitioner: partitioner}
}
func (h *MasterHandler) logDBConn() *gorm.DB {
@@ -34,6 +35,10 @@ func (h *MasterHandler) logDBConn() *gorm.DB {
return h.logDB
}
func (h *MasterHandler) logBaseQuery() *gorm.DB {
return logBaseQuery(h.logDBConn(), h.logPartitioner)
}
type IssueChildKeyRequest struct {
Group string `json:"group"`
Scopes string `json:"scopes"`

2
internal/api/master_tokens_handler_test.go
View File

@@ -42,7 +42,7 @@ func TestMaster_ListTokens_AndUpdateToken(t *testing.T) {
rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})
syncSvc := service.NewSyncService(rdb)
masterSvc := service.NewMasterService(db)
h := NewMasterHandler(db, db, masterSvc, syncSvc)
h := NewMasterHandler(db, db, masterSvc, syncSvc, nil)
withMaster := func(next gin.HandlerFunc) gin.HandlerFunc {
return func(c *gin.Context) {

2
internal/api/model_handler_test.go
View File

@@ -33,7 +33,7 @@ func newTestHandlerWithRedis(t *testing.T) (*Handler, *gorm.DB, *miniredis.Minir
mr := miniredis.RunT(t)
rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})
sync := service.NewSyncService(rdb)
return NewHandler(db, db, sync, nil, rdb), db, mr
return NewHandler(db, db, sync, nil, rdb, nil), db, mr
}
func TestCreateModel_DefaultsKindChat_AndWritesModelsMeta(t *testing.T) {

10
internal/api/provider_admin_handler.go
View File

@@ -86,10 +86,12 @@ func (h *Handler) DeleteProvider(c *gin.Context) {
return
}
// Full sync is the simplest safe option because provider deletion needs to remove
// stale entries in Redis snapshots (config:providers) and refresh bindings.
if err := h.sync.SyncAll(h.db); err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to sync snapshots", "details": err.Error()})
if err := h.sync.SyncProviderDelete(&p); err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to sync provider delete", "details": err.Error()})
return
}
if err := h.sync.SyncBindings(h.db); err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to sync bindings", "details": err.Error()})
return
}

2
internal/api/provider_handler_test.go
View File

@@ -36,7 +36,7 @@ func newTestHandler(t *testing.T) (*Handler, *gorm.DB) {
rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})
sync := service.NewSyncService(rdb)
return NewHandler(db, db, sync, nil, rdb), db
return NewHandler(db, db, sync, nil, rdb, nil), db
}
func TestCreateProvider_DefaultsVertexLocationGlobal(t *testing.T) {

4
internal/api/stats_handler.go
View File

@@ -61,7 +61,7 @@ func (h *MasterHandler) GetSelfStats(c *gin.Context) {
}
logDB := h.logDBConn()
base := logDB.Model(&model.LogRecord{})
base := h.logBaseQuery()
if logDB == h.db {
base = base.Joins("JOIN keys ON keys.id = log_records.key_id").
Where("keys.master_id = ?", m.ID)
@@ -170,7 +170,7 @@ func (h *AdminHandler) GetAdminStats(c *gin.Context) {
}
logDB := h.logDBConn()
base := logDB.Model(&model.LogRecord{})
base := h.logBaseQuery()
base = applyStatsRange(base, rng)
totalRequests, totalTokens, err := aggregateTotals(base)

4
internal/api/stats_handler_test.go
View File

@@ -72,7 +72,7 @@ func TestMasterStats_AggregatesByKeyAndModel(t *testing.T) {
rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})
masterSvc := service.NewMasterService(db)
syncSvc := service.NewSyncService(rdb)
h := NewMasterHandler(db, db, masterSvc, syncSvc)
h := NewMasterHandler(db, db, masterSvc, syncSvc, nil)
withMaster := func(next gin.HandlerFunc) gin.HandlerFunc {
return func(c *gin.Context) {
@@ -163,7 +163,7 @@ func TestAdminStats_AggregatesByProvider(t *testing.T) {
rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})
masterSvc := service.NewMasterService(db)
syncSvc := service.NewSyncService(rdb)
adminHandler := NewAdminHandler(db, db, masterSvc, syncSvc)
adminHandler := NewAdminHandler(db, db, masterSvc, syncSvc, nil)
r := gin.New()
r.GET("/admin/stats", adminHandler.GetAdminStats)