mirror of
https://github.com/EZ-Api/ez-api.git
synced 2026-01-13 17:47:51 +00:00
feat(arch): add log partitioning and provider delete sync
This commit is contained in:
@@ -9,6 +9,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/ez-api/ez-api/internal/model"
|
||||
"github.com/ez-api/ez-api/internal/service"
|
||||
"github.com/redis/go-redis/v9"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
@@ -25,9 +26,10 @@ type LogCleaner struct {
|
||||
retentionDays int
|
||||
maxRecords int64
|
||||
interval time.Duration
|
||||
partitioner *service.LogPartitioner
|
||||
}
|
||||
|
||||
func NewLogCleaner(db *gorm.DB, rdb *redis.Client, retentionDays int, maxRecords int64, interval time.Duration) *LogCleaner {
|
||||
func NewLogCleaner(db *gorm.DB, rdb *redis.Client, retentionDays int, maxRecords int64, interval time.Duration, partitioner *service.LogPartitioner) *LogCleaner {
|
||||
if interval <= 0 {
|
||||
interval = time.Hour
|
||||
}
|
||||
@@ -37,6 +39,7 @@ func NewLogCleaner(db *gorm.DB, rdb *redis.Client, retentionDays int, maxRecords
|
||||
retentionDays: retentionDays,
|
||||
maxRecords: maxRecords,
|
||||
interval: interval,
|
||||
partitioner: partitioner,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -74,6 +77,10 @@ func (c *LogCleaner) cleanOnce(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
if c.partitioner != nil && c.partitioner.Enabled() {
|
||||
return c.cleanPartitioned(retentionDays, maxRecords)
|
||||
}
|
||||
|
||||
var deleted int64
|
||||
if retentionDays > 0 {
|
||||
cutoff := time.Now().UTC().AddDate(0, 0, -retentionDays)
|
||||
@@ -116,6 +123,64 @@ func (c *LogCleaner) cleanOnce(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *LogCleaner) cleanPartitioned(retentionDays int, maxRecords int64) error {
|
||||
var deleted int64
|
||||
|
||||
if retentionDays > 0 {
|
||||
cutoff := time.Now().UTC().AddDate(0, 0, -retentionDays)
|
||||
dropped, err := c.partitioner.DropPartitionsBefore(cutoff)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if dropped > 0 {
|
||||
slog.Default().Info("log partition cleanup completed", "dropped_tables", dropped, "retention_days", retentionDays)
|
||||
}
|
||||
table, err := c.partitioner.EnsurePartitionFor(time.Now().UTC())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
res := c.db.Table(table).Unscoped().Where("created_at < ?", cutoff).Delete(&model.LogRecord{})
|
||||
if res.Error != nil {
|
||||
return res.Error
|
||||
}
|
||||
deleted += res.RowsAffected
|
||||
}
|
||||
|
||||
if maxRecords > 0 {
|
||||
if maxRecords > int64(math.MaxInt) {
|
||||
maxRecords = int64(math.MaxInt)
|
||||
}
|
||||
table, err := c.partitioner.EnsurePartitionFor(time.Now().UTC())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var cutoff struct {
|
||||
ID uint
|
||||
}
|
||||
if err := c.db.Unscoped().
|
||||
Table(table).
|
||||
Select("id").
|
||||
Order("id desc").
|
||||
Offset(int(maxRecords - 1)).
|
||||
Limit(1).
|
||||
Scan(&cutoff).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if cutoff.ID > 0 {
|
||||
res := c.db.Table(table).Unscoped().Where("id < ?", cutoff.ID).Delete(&model.LogRecord{})
|
||||
if res.Error != nil {
|
||||
return res.Error
|
||||
}
|
||||
deleted += res.RowsAffected
|
||||
}
|
||||
}
|
||||
|
||||
if deleted > 0 {
|
||||
slog.Default().Info("log cleanup completed", "deleted", deleted, "retention_days", retentionDays, "max_records", maxRecords)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *LogCleaner) resolveRetentionDays(ctx context.Context) int {
|
||||
days := c.retentionDays
|
||||
if days < 0 {
|
||||
|
||||
Reference in New Issue
Block a user