feat(arch): add log partitioning and provider delete sync

This commit is contained in:
zenfun
2025-12-21 20:45:16 +08:00
parent f819f89ba2
commit 816ea93339
23 changed files with 582 additions and 69 deletions

View File

@@ -132,8 +132,9 @@ func parseUnixSeconds(raw string) (time.Time, bool) {
func (h *MasterHandler) masterLogBase(masterID uint) (*gorm.DB, error) {
logDB := h.logDBConn()
base := h.logBaseQuery()
if logDB == h.db {
return logDB.Model(&model.LogRecord{}).
return base.
Joins("JOIN keys ON keys.id = log_records.key_id").
Where("keys.master_id = ?", masterID), nil
}
@@ -144,9 +145,9 @@ func (h *MasterHandler) masterLogBase(masterID uint) (*gorm.DB, error) {
return nil, err
}
if len(keyIDs) == 0 {
return logDB.Model(&model.LogRecord{}).Where("1 = 0"), nil
return base.Where("1 = 0"), nil
}
return logDB.Model(&model.LogRecord{}).
return base.
Where("log_records.key_id IN ?", keyIDs), nil
}
@@ -170,7 +171,7 @@ func (h *MasterHandler) masterLogBase(masterID uint) (*gorm.DB, error) {
func (h *Handler) ListLogs(c *gin.Context) {
limit, offset := parseLimitOffset(c)
q := h.logDBConn().Model(&model.LogRecord{})
q := h.logBaseQuery()
if t, ok := parseUnixSeconds(c.Query("since")); ok {
q = q.Where("created_at >= ?", t)
@@ -261,20 +262,63 @@ func (h *Handler) DeleteLogs(c *gin.Context) {
return
}
q := h.logDBConn().Unscoped().Where("created_at < ?", ts.UTC())
if req.KeyID > 0 {
q = q.Where("key_id = ?", req.KeyID)
}
if model := strings.TrimSpace(req.Model); model != "" {
q = q.Where("model_name = ?", model)
}
res := q.Delete(&model.LogRecord{})
if res.Error != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to delete logs", "details": res.Error.Error()})
deleted, err := h.deleteLogsBefore(ts.UTC(), req.KeyID, req.Model)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to delete logs", "details": err.Error()})
return
}
c.JSON(http.StatusOK, DeleteLogsResponse{DeletedCount: res.RowsAffected})
c.JSON(http.StatusOK, DeleteLogsResponse{DeletedCount: deleted})
}
func (h *Handler) deleteLogsBefore(cutoff time.Time, keyID uint, modelName string) (int64, error) {
modelName = strings.TrimSpace(modelName)
if h == nil || h.logPartitioner == nil || !h.logPartitioner.Enabled() {
q := h.logBaseQuery().Unscoped().Where("created_at < ?", cutoff)
if keyID > 0 {
q = q.Where("key_id = ?", keyID)
}
if modelName != "" {
q = q.Where("model_name = ?", modelName)
}
res := q.Delete(&model.LogRecord{})
return res.RowsAffected, res.Error
}
partitions, err := h.logPartitioner.ListPartitions()
if err != nil {
return 0, err
}
if len(partitions) == 0 {
q := h.logDBConn().Table("log_records").Unscoped().Where("created_at < ?", cutoff)
if keyID > 0 {
q = q.Where("key_id = ?", keyID)
}
if modelName != "" {
q = q.Where("model_name = ?", modelName)
}
res := q.Delete(&model.LogRecord{})
return res.RowsAffected, res.Error
}
var deleted int64
for _, part := range partitions {
if !part.Start.Before(cutoff) {
continue
}
q := h.logDBConn().Table(part.Table).Unscoped().Where("created_at < ?", cutoff)
if keyID > 0 {
q = q.Where("key_id = ?", keyID)
}
if modelName != "" {
q = q.Where("model_name = ?", modelName)
}
res := q.Delete(&model.LogRecord{})
if res.Error != nil {
return deleted, res.Error
}
deleted += res.RowsAffected
}
return deleted, nil
}
// LogStats godoc
@@ -289,7 +333,7 @@ func (h *Handler) DeleteLogs(c *gin.Context) {
// @Failure 500 {object} gin.H
// @Router /admin/logs/stats [get]
func (h *Handler) LogStats(c *gin.Context) {
q := h.logDBConn().Model(&model.LogRecord{})
q := h.logBaseQuery()
if t, ok := parseUnixSeconds(c.Query("since")); ok {
q = q.Where("created_at >= ?", t)
}