feat(arch): add log partitioning and provider delete sync

This commit is contained in:
zenfun
2025-12-21 20:45:16 +08:00
parent f819f89ba2
commit 816ea93339
23 changed files with 582 additions and 69 deletions

View File

@@ -21,9 +21,10 @@ type LogWriter struct {
batchSize int
flushInterval time.Duration
db *gorm.DB
partitioner *LogPartitioner
}
func NewLogWriter(db *gorm.DB, queueCapacity, batchSize int, flushInterval time.Duration) *LogWriter {
func NewLogWriter(db *gorm.DB, queueCapacity, batchSize int, flushInterval time.Duration, partitioner *LogPartitioner) *LogWriter {
if batchSize <= 0 {
batchSize = 10
}
@@ -38,6 +39,7 @@ func NewLogWriter(db *gorm.DB, queueCapacity, batchSize int, flushInterval time.
batchSize: batchSize,
flushInterval: flushInterval,
db: db,
partitioner: partitioner,
}
}
@@ -52,9 +54,33 @@ func (w *LogWriter) Start(ctx context.Context) {
if len(buf) == 0 {
return
}
if err := w.db.Create(&buf).Error; err != nil {
slog.Default().Error("log batch insert failed", "err", err)
} else {
if w.partitioner == nil || !w.partitioner.Enabled() {
if err := w.db.Create(&buf).Error; err != nil {
slog.Default().Error("log batch insert failed", "err", err)
} else {
logBatchWriteTotal.Add(1)
}
buf = buf[:0]
return
}
byTable := make(map[string][]model.LogRecord)
for _, rec := range buf {
t := rec.CreatedAt
if t.IsZero() {
t = time.Now().UTC()
}
table, err := w.partitioner.EnsurePartitionFor(t)
if err != nil {
slog.Default().Error("log partition ensure failed", "err", err)
table = "log_records"
}
byTable[table] = append(byTable[table], rec)
}
for table, records := range byTable {
if err := w.db.Table(table).Create(&records).Error; err != nil {
slog.Default().Error("log batch insert failed", "table", table, "err", err)
continue
}
logBatchWriteTotal.Add(1)
}
buf = buf[:0]