feat(arch): add log partitioning and provider delete sync

This commit is contained in:
zenfun
2025-12-21 20:45:16 +08:00
parent f819f89ba2
commit 816ea93339
23 changed files with 582 additions and 69 deletions

View File

@@ -9,6 +9,7 @@ import (
"time"
"github.com/ez-api/ez-api/internal/model"
"github.com/ez-api/ez-api/internal/service"
"github.com/redis/go-redis/v9"
"gorm.io/gorm"
)
@@ -25,9 +26,10 @@ type LogCleaner struct {
retentionDays int
maxRecords int64
interval time.Duration
partitioner *service.LogPartitioner
}
func NewLogCleaner(db *gorm.DB, rdb *redis.Client, retentionDays int, maxRecords int64, interval time.Duration) *LogCleaner {
func NewLogCleaner(db *gorm.DB, rdb *redis.Client, retentionDays int, maxRecords int64, interval time.Duration, partitioner *service.LogPartitioner) *LogCleaner {
if interval <= 0 {
interval = time.Hour
}
@@ -37,6 +39,7 @@ func NewLogCleaner(db *gorm.DB, rdb *redis.Client, retentionDays int, maxRecords
retentionDays: retentionDays,
maxRecords: maxRecords,
interval: interval,
partitioner: partitioner,
}
}
@@ -74,6 +77,10 @@ func (c *LogCleaner) cleanOnce(ctx context.Context) error {
return nil
}
if c.partitioner != nil && c.partitioner.Enabled() {
return c.cleanPartitioned(retentionDays, maxRecords)
}
var deleted int64
if retentionDays > 0 {
cutoff := time.Now().UTC().AddDate(0, 0, -retentionDays)
@@ -116,6 +123,64 @@ func (c *LogCleaner) cleanOnce(ctx context.Context) error {
return nil
}
func (c *LogCleaner) cleanPartitioned(retentionDays int, maxRecords int64) error {
var deleted int64
if retentionDays > 0 {
cutoff := time.Now().UTC().AddDate(0, 0, -retentionDays)
dropped, err := c.partitioner.DropPartitionsBefore(cutoff)
if err != nil {
return err
}
if dropped > 0 {
slog.Default().Info("log partition cleanup completed", "dropped_tables", dropped, "retention_days", retentionDays)
}
table, err := c.partitioner.EnsurePartitionFor(time.Now().UTC())
if err != nil {
return err
}
res := c.db.Table(table).Unscoped().Where("created_at < ?", cutoff).Delete(&model.LogRecord{})
if res.Error != nil {
return res.Error
}
deleted += res.RowsAffected
}
if maxRecords > 0 {
if maxRecords > int64(math.MaxInt) {
maxRecords = int64(math.MaxInt)
}
table, err := c.partitioner.EnsurePartitionFor(time.Now().UTC())
if err != nil {
return err
}
var cutoff struct {
ID uint
}
if err := c.db.Unscoped().
Table(table).
Select("id").
Order("id desc").
Offset(int(maxRecords - 1)).
Limit(1).
Scan(&cutoff).Error; err != nil {
return err
}
if cutoff.ID > 0 {
res := c.db.Table(table).Unscoped().Where("id < ?", cutoff.ID).Delete(&model.LogRecord{})
if res.Error != nil {
return res.Error
}
deleted += res.RowsAffected
}
}
if deleted > 0 {
slog.Default().Info("log cleanup completed", "deleted", deleted, "retention_days", retentionDays, "max_records", maxRecords)
}
return nil
}
func (c *LogCleaner) resolveRetentionDays(ctx context.Context) int {
days := c.retentionDays
if days < 0 {

View File

@@ -34,7 +34,7 @@ func TestLogCleanerRetentionDeletesOld(t *testing.T) {
t.Fatalf("create fresh: %v", err)
}
cleaner := NewLogCleaner(db, nil, 1, 0, time.Minute)
cleaner := NewLogCleaner(db, nil, 1, 0, time.Minute, nil)
if err := cleaner.cleanOnce(context.Background()); err != nil {
t.Fatalf("clean once: %v", err)
}
@@ -64,7 +64,7 @@ func TestLogCleanerMaxRecordsKeepsLatest(t *testing.T) {
}
}
cleaner := NewLogCleaner(db, nil, 0, 3, time.Minute)
cleaner := NewLogCleaner(db, nil, 0, 3, time.Minute, nil)
if err := cleaner.cleanOnce(context.Background()); err != nil {
t.Fatalf("clean once: %v", err)
}