mirror of
https://github.com/EZ-Api/ez-api.git
synced 2026-01-13 17:47:51 +00:00
feat(arch): add log partitioning and provider delete sync
This commit is contained in:
@@ -21,9 +21,10 @@ type LogWriter struct {
|
||||
batchSize int
|
||||
flushInterval time.Duration
|
||||
db *gorm.DB
|
||||
partitioner *LogPartitioner
|
||||
}
|
||||
|
||||
func NewLogWriter(db *gorm.DB, queueCapacity, batchSize int, flushInterval time.Duration) *LogWriter {
|
||||
func NewLogWriter(db *gorm.DB, queueCapacity, batchSize int, flushInterval time.Duration, partitioner *LogPartitioner) *LogWriter {
|
||||
if batchSize <= 0 {
|
||||
batchSize = 10
|
||||
}
|
||||
@@ -38,6 +39,7 @@ func NewLogWriter(db *gorm.DB, queueCapacity, batchSize int, flushInterval time.
|
||||
batchSize: batchSize,
|
||||
flushInterval: flushInterval,
|
||||
db: db,
|
||||
partitioner: partitioner,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -52,9 +54,33 @@ func (w *LogWriter) Start(ctx context.Context) {
|
||||
if len(buf) == 0 {
|
||||
return
|
||||
}
|
||||
if err := w.db.Create(&buf).Error; err != nil {
|
||||
slog.Default().Error("log batch insert failed", "err", err)
|
||||
} else {
|
||||
if w.partitioner == nil || !w.partitioner.Enabled() {
|
||||
if err := w.db.Create(&buf).Error; err != nil {
|
||||
slog.Default().Error("log batch insert failed", "err", err)
|
||||
} else {
|
||||
logBatchWriteTotal.Add(1)
|
||||
}
|
||||
buf = buf[:0]
|
||||
return
|
||||
}
|
||||
byTable := make(map[string][]model.LogRecord)
|
||||
for _, rec := range buf {
|
||||
t := rec.CreatedAt
|
||||
if t.IsZero() {
|
||||
t = time.Now().UTC()
|
||||
}
|
||||
table, err := w.partitioner.EnsurePartitionFor(t)
|
||||
if err != nil {
|
||||
slog.Default().Error("log partition ensure failed", "err", err)
|
||||
table = "log_records"
|
||||
}
|
||||
byTable[table] = append(byTable[table], rec)
|
||||
}
|
||||
for table, records := range byTable {
|
||||
if err := w.db.Table(table).Create(&records).Error; err != nil {
|
||||
slog.Default().Error("log batch insert failed", "table", table, "err", err)
|
||||
continue
|
||||
}
|
||||
logBatchWriteTotal.Add(1)
|
||||
}
|
||||
buf = buf[:0]
|
||||
|
||||
Reference in New Issue
Block a user