feat: 配置列表分页与钉钉机器人分钟级排队限流

统一 sources/templates/channels/rules 列表为分页响应,避免配置增多时全量返回;按钉钉 access_token 限制每分钟发送并在超限时等待下一分钟,降低触发官方封禁风险。

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
2026-08-10 00:34:26 +08:00
parent 009694fee7
commit 39f3774940
27 changed files with 1447 additions and 60 deletions
+2 -2
View File
@@ -13,10 +13,10 @@ type ChannelSender interface {
Send(title, content string, config json.RawMessage) error
}
func NewSender(channelType string, smtpCfg *config.SMTPConfig) (ChannelSender, error) {
func NewSender(channelType string, smtpCfg *config.SMTPConfig, dingtalkLimiter *DingTalkLimiter) (ChannelSender, error) {
switch channelType {
case "dingtalk":
return &DingTalkSender{}, nil
return &DingTalkSender{limiter: dingtalkLimiter}, nil
case "wecom":
return &WeComSender{}, nil
case "bark":
+10 -1
View File
@@ -2,6 +2,7 @@ package adapter
import (
"bytes"
"context"
"crypto/hmac"
"crypto/sha256"
"encoding/base64"
@@ -27,7 +28,9 @@ type dingtalkMD struct {
Text string `json:"text"`
}
type DingTalkSender struct{}
type DingTalkSender struct {
limiter *DingTalkLimiter
}
func (s *DingTalkSender) Type() string { return "dingtalk" }
@@ -37,6 +40,12 @@ func (s *DingTalkSender) Send(title, content string, config json.RawMessage) err
return fmt.Errorf("parse dingtalk config: %w", err)
}
if s.limiter != nil {
if err := s.limiter.Acquire(context.Background(), DingTalkLimitKey(cfg.WebhookURL)); err != nil {
return fmt.Errorf("dingtalk rate limit: %w", err)
}
}
reqURL := cfg.WebhookURL
if cfg.Secret != "" {
timestamp := time.Now().UnixMilli()
+141
View File
@@ -0,0 +1,141 @@
package adapter
import (
"context"
"fmt"
"log/slog"
"net/url"
"sync"
"time"
)
const DefaultDingTalkPerMin = 18
// MinuteWindowStore tracks per-key counters in a fixed UTC-minute window.
type MinuteWindowStore interface {
// TryIncr increments if under limit; returns (count, true) when acquired,
// or (currentOrLimit, false) when the window is full.
TryIncr(ctx context.Context, key string, minute int64, limit int) (count int64, ok bool, err error)
}
// DingTalkLimiter enforces per-robot (access_token) send quota and waits for the next minute when full.
type DingTalkLimiter struct {
store MinuteWindowStore
limit int
now func() time.Time
sleep func(time.Duration)
}
func NewDingTalkLimiter(store MinuteWindowStore, limitPerMin int) *DingTalkLimiter {
if limitPerMin <= 0 {
limitPerMin = DefaultDingTalkPerMin
}
return &DingTalkLimiter{
store: store,
limit: limitPerMin,
now: time.Now,
sleep: time.Sleep,
}
}
// DingTalkLimitKey returns the rate-limit key for a webhook URL (access_token).
func DingTalkLimitKey(webhookURL string) string {
if webhookURL == "" {
return "unknown"
}
u, err := url.Parse(webhookURL)
if err != nil {
return webhookURL
}
if tok := u.Query().Get("access_token"); tok != "" {
return tok
}
return webhookURL
}
// Acquire blocks until a send slot is available for key in the current minute window.
func (l *DingTalkLimiter) Acquire(ctx context.Context, key string) error {
if l == nil || l.store == nil {
return nil
}
for {
if err := ctx.Err(); err != nil {
return err
}
now := l.now()
minute := now.Unix() / 60
count, ok, err := l.store.TryIncr(ctx, key, minute, l.limit)
if err != nil {
return fmt.Errorf("dingtalk rate limit: %w", err)
}
if ok {
return nil
}
next := time.Unix((minute+1)*60, 0)
wait := next.Sub(now)
if wait < time.Millisecond {
wait = time.Millisecond
}
slog.Info("dingtalk rate limited, waiting for next minute",
"key_suffix", maskKey(key),
"count", count,
"limit", l.limit,
"wait", wait,
)
if err := l.sleepCtx(ctx, wait); err != nil {
return err
}
}
}
func (l *DingTalkLimiter) sleepCtx(ctx context.Context, d time.Duration) error {
// Prefer injectable sleep for tests; also honor context cancel via polling when possible.
done := make(chan struct{})
go func() {
l.sleep(d)
close(done)
}()
select {
case <-ctx.Done():
return ctx.Err()
case <-done:
return nil
}
}
func maskKey(key string) string {
if len(key) <= 8 {
return "****"
}
return key[:4] + "…" + key[len(key)-4:]
}
// --- in-memory store (fallback / tests) ---
type memoryMinuteStore struct {
mu sync.Mutex
data map[string]int64 // key:minute -> count
}
func newMemoryMinuteStore() *memoryMinuteStore {
return &memoryMinuteStore{data: make(map[string]int64)}
}
func (m *memoryMinuteStore) TryIncr(_ context.Context, key string, minute int64, limit int) (int64, bool, error) {
m.mu.Lock()
defer m.mu.Unlock()
k := fmt.Sprintf("%s:%d", key, minute)
cur := m.data[k]
if cur >= int64(limit) {
return cur, false, nil
}
cur++
m.data[k] = cur
return cur, true, nil
}
// NewMemoryDingTalkLimiter builds a process-local limiter (single-instance only).
func NewMemoryDingTalkLimiter(limitPerMin int) *DingTalkLimiter {
return NewDingTalkLimiter(newMemoryMinuteStore(), limitPerMin)
}
+99
View File
@@ -0,0 +1,99 @@
package adapter
import (
"context"
"testing"
"time"
)
func TestDingTalkLimitKey(t *testing.T) {
tests := []struct {
name string
url string
want string
}{
{
name: "token only",
url: "https://oapi.dingtalk.com/robot/send?access_token=abc123",
want: "abc123",
},
{
name: "token with other params",
url: "https://oapi.dingtalk.com/robot/send?access_token=tok&foo=1",
want: "tok",
},
{
name: "missing token falls back to url",
url: "https://oapi.dingtalk.com/robot/send?foo=1",
want: "https://oapi.dingtalk.com/robot/send?foo=1",
},
{
name: "empty",
url: "",
want: "unknown",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := DingTalkLimitKey(tt.url); got != tt.want {
t.Fatalf("DingTalkLimitKey(%q) = %q, want %q", tt.url, got, tt.want)
}
})
}
}
func TestDingTalkLimiterAcquireWaitsForNextWindow(t *testing.T) {
now := time.Unix(1000, 0) // fixed
var slept time.Duration
store := newMemoryMinuteStore()
lim := &DingTalkLimiter{
store: store,
limit: 2,
now: func() time.Time { return now },
sleep: func(d time.Duration) {
slept += d
now = now.Add(d)
},
}
ctx := context.Background()
key := "tok-1"
if err := lim.Acquire(ctx, key); err != nil {
t.Fatal(err)
}
if err := lim.Acquire(ctx, key); err != nil {
t.Fatal(err)
}
if slept != 0 {
t.Fatalf("unexpected sleep before limit: %v", slept)
}
// third should wait until next minute boundary (60 - 1000%60 = 20s? 1000/60=16 rem 40, next at 17*60=1020, wait 20s)
if err := lim.Acquire(ctx, key); err != nil {
t.Fatal(err)
}
if slept != 20*time.Second {
t.Fatalf("slept = %v, want 20s", slept)
}
}
func TestDingTalkLimiterSameKeyShared(t *testing.T) {
now := time.Unix(0, 0)
store := newMemoryMinuteStore()
lim := &DingTalkLimiter{
store: store,
limit: 1,
now: func() time.Time { return now },
sleep: func(d time.Duration) { now = now.Add(d) },
}
ctx := context.Background()
if err := lim.Acquire(ctx, "shared"); err != nil {
t.Fatal(err)
}
if err := lim.Acquire(ctx, "shared"); err != nil {
t.Fatal(err)
}
if now.Unix() != 60 {
t.Fatalf("expected wait to next minute boundary, now=%v", now)
}
}
+60
View File
@@ -0,0 +1,60 @@
package cache
import (
"context"
"fmt"
"github.com/redis/go-redis/v9"
)
// TryIncr implements adapter.MinuteWindowStore for DingTalk per-minute quotas.
func (c *Cache) TryIncr(ctx context.Context, key string, minute int64, limit int) (int64, bool, error) {
if c == nil || c.rdb == nil {
return 0, false, fmt.Errorf("redis unavailable")
}
redisKey := fmt.Sprintf("dingtalk:rl:%s:%d", key, minute)
script := redis.NewScript(`
local key = KEYS[1]
local limit = tonumber(ARGV[1])
local cur = tonumber(redis.call('GET', key) or '0')
if cur >= limit then
return {cur, 0}
end
local n = redis.call('INCR', key)
if n == 1 then
redis.call('EXPIRE', key, 120)
end
return {n, 1}
`)
res, err := script.Run(ctx, c.rdb, []string{redisKey}, limit).Slice()
if err != nil {
return 0, false, fmt.Errorf("dingtalk TryIncr: %w", err)
}
if len(res) != 2 {
return 0, false, fmt.Errorf("dingtalk TryIncr: unexpected result %#v", res)
}
count, err := toInt64(res[0])
if err != nil {
return 0, false, err
}
okFlag, err := toInt64(res[1])
if err != nil {
return 0, false, err
}
return count, okFlag == 1, nil
}
func toInt64(v interface{}) (int64, error) {
switch n := v.(type) {
case int64:
return n, nil
case int:
return int64(n), nil
case string:
var x int64
_, err := fmt.Sscan(n, &x)
return x, err
default:
return 0, fmt.Errorf("cannot convert %T to int64", v)
}
}
+2 -1
View File
@@ -55,7 +55,8 @@ type SMTPConfig struct {
}
type RateLimitConfig struct {
Default int `mapstructure:"default"`
Default int `mapstructure:"default"`
DingTalkPerMin int `mapstructure:"dingtalk_per_min"` // per robot webhook; 0 => 18
}
type LogbullConfig struct {
+9 -2
View File
@@ -47,12 +47,19 @@ func (h *ChannelHandler) Create(c *gin.Context) {
}
func (h *ChannelHandler) List(c *gin.Context) {
channels, err := h.store.ListChannels(c.Request.Context())
var page store.PageFilter
if err := c.ShouldBindQuery(&page); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
page.Normalize()
channels, total, err := h.store.ListChannels(c.Request.Context(), page)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
c.JSON(http.StatusOK, channels)
c.JSON(http.StatusOK, gin.H{"data": channels, "total": total, "page": page.Page})
}
func (h *ChannelHandler) Get(c *gin.Context) {
+1
View File
@@ -22,6 +22,7 @@ func (h *MessageLogHandler) List(c *gin.Context) {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
filter.Normalize()
logs, total, err := h.store.ListMessageLogs(c.Request.Context(), filter)
if err != nil {
+31 -2
View File
@@ -81,12 +81,19 @@ func (h *RuleHandler) Create(c *gin.Context) {
}
func (h *RuleHandler) List(c *gin.Context) {
rules, err := h.store.ListRules(c.Request.Context())
var page store.PageFilter
if err := c.ShouldBindQuery(&page); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
page.Normalize()
rules, total, err := h.store.ListRules(c.Request.Context(), page)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
c.JSON(http.StatusOK, rules)
c.JSON(http.StatusOK, gin.H{"data": rules, "total": total, "page": page.Page})
}
func (h *RuleHandler) Get(c *gin.Context) {
@@ -163,19 +170,35 @@ func (h *RuleHandler) Delete(c *gin.Context) {
func (h *RuleHandler) Enable(c *gin.Context) {
id, _ := strconv.Atoi(c.Param("id"))
rule, err := h.store.GetRule(c.Request.Context(), id)
if err != nil {
c.JSON(http.StatusNotFound, gin.H{"error": "rule not found"})
return
}
if err := h.store.SetRuleEnabled(c.Request.Context(), id, true); err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
if h.cache != nil {
_ = h.cache.InvalidateRule(c.Request.Context(), rule.SourceID, rule.Event)
}
c.JSON(http.StatusOK, gin.H{"ok": true})
}
func (h *RuleHandler) Disable(c *gin.Context) {
id, _ := strconv.Atoi(c.Param("id"))
rule, err := h.store.GetRule(c.Request.Context(), id)
if err != nil {
c.JSON(http.StatusNotFound, gin.H{"error": "rule not found"})
return
}
if err := h.store.SetRuleEnabled(c.Request.Context(), id, false); err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
if h.cache != nil {
_ = h.cache.InvalidateRule(c.Request.Context(), rule.SourceID, rule.Event)
}
c.JSON(http.StatusOK, gin.H{"ok": true})
}
@@ -186,6 +209,9 @@ func (h *RuleHandler) EnableChannel(c *gin.Context) {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
if h.cache != nil {
_ = h.cache.InvalidateChannels(c.Request.Context(), ruleID)
}
c.JSON(http.StatusOK, gin.H{"ok": true})
}
@@ -196,6 +222,9 @@ func (h *RuleHandler) DisableChannel(c *gin.Context) {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
if h.cache != nil {
_ = h.cache.InvalidateChannels(c.Request.Context(), ruleID)
}
c.JSON(http.StatusOK, gin.H{"ok": true})
}
+9 -2
View File
@@ -54,12 +54,19 @@ func (h *SourceHandler) Create(c *gin.Context) {
}
func (h *SourceHandler) List(c *gin.Context) {
sources, err := h.store.ListSources(c.Request.Context())
var page store.PageFilter
if err := c.ShouldBindQuery(&page); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
page.Normalize()
sources, total, err := h.store.ListSources(c.Request.Context(), page)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
c.JSON(http.StatusOK, sources)
c.JSON(http.StatusOK, gin.H{"data": sources, "total": total, "page": page.Page})
}
func (h *SourceHandler) Get(c *gin.Context) {
+9 -2
View File
@@ -40,12 +40,19 @@ func (h *TemplateHandler) Create(c *gin.Context) {
}
func (h *TemplateHandler) List(c *gin.Context) {
templates, err := h.store.ListTemplates(c.Request.Context())
var page store.PageFilter
if err := c.ShouldBindQuery(&page); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
page.Normalize()
templates, total, err := h.store.ListTemplates(c.Request.Context(), page)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
c.JSON(http.StatusOK, templates)
c.JSON(http.StatusOK, gin.H{"data": templates, "total": total, "page": page.Page})
}
func (h *TemplateHandler) Get(c *gin.Context) {
+11 -5
View File
@@ -47,10 +47,16 @@ func (s *Store) GetChannelByName(ctx context.Context, name string) (*model.Chann
return &ch, nil
}
func (s *Store) ListChannels(ctx context.Context) ([]model.Channel, error) {
rows, err := s.DB.QueryContext(ctx, `SELECT id, name, type, config, status, created_at, updated_at FROM notification_channel ORDER BY id`)
func (s *Store) ListChannels(ctx context.Context, page PageFilter) ([]model.Channel, int, error) {
var count int
if err := s.DB.GetContext(ctx, &count, `SELECT COUNT(*) FROM notification_channel`); err != nil {
return nil, 0, fmt.Errorf("count channels: %w", err)
}
page.Normalize()
rows, err := s.DB.QueryContext(ctx, `SELECT id, name, type, config, status, created_at, updated_at FROM notification_channel ORDER BY id LIMIT ? OFFSET ?`, page.PageSize, page.Offset())
if err != nil {
return nil, fmt.Errorf("list channels: %w", err)
return nil, 0, fmt.Errorf("list channels: %w", err)
}
defer rows.Close()
@@ -59,13 +65,13 @@ func (s *Store) ListChannels(ctx context.Context) ([]model.Channel, error) {
var ch model.Channel
var configBytes []byte
if err := rows.Scan(&ch.ID, &ch.Name, &ch.Type, &configBytes, &ch.Status, &ch.CreatedAt, &ch.UpdatedAt); err != nil {
return nil, fmt.Errorf("scan channel: %w", err)
return nil, 0, fmt.Errorf("scan channel: %w", err)
}
raw := json.RawMessage(configBytes)
ch.Config = &raw
channels = append(channels, ch)
}
return channels, rows.Err()
return channels, count, rows.Err()
}
func (s *Store) UpdateChannel(ctx context.Context, id int, ch *model.Channel) error {
+6 -13
View File
@@ -8,11 +8,10 @@ import (
)
type MessageLogFilter struct {
Source string `form:"source"`
Event string `form:"event"`
Status string `form:"status"`
Page int `form:"page"`
PageSize int `form:"page_size"`
Source string `form:"source"`
Event string `form:"event"`
Status string `form:"status"`
PageFilter
}
func (s *Store) CreateMessageLog(ctx context.Context, ml *model.MessageLog) error {
@@ -54,17 +53,11 @@ func (s *Store) ListMessageLogs(ctx context.Context, filter MessageLogFilter) ([
return nil, 0, err
}
if filter.Page <= 0 {
filter.Page = 1
}
if filter.PageSize <= 0 {
filter.PageSize = 20
}
offset := (filter.Page - 1) * filter.PageSize
filter.Normalize()
logs := make([]model.MessageLog, 0)
query := "SELECT * FROM notification_message_log " + where + " ORDER BY id DESC LIMIT ? OFFSET ?"
args = append(args, filter.PageSize, offset)
args = append(args, filter.PageSize, filter.Offset())
if err := s.DB.SelectContext(ctx, &logs, query, args...); err != nil {
return nil, 0, err
}
+22
View File
@@ -0,0 +1,22 @@
package store
// PageFilter is the shared pagination query for list endpoints.
type PageFilter struct {
Page int `form:"page"`
PageSize int `form:"page_size"`
}
// Normalize applies defaults: page=1, page_size=20.
func (p *PageFilter) Normalize() {
if p.Page <= 0 {
p.Page = 1
}
if p.PageSize <= 0 {
p.PageSize = 20
}
}
// Offset returns the SQL OFFSET for the current page.
func (p PageFilter) Offset() int {
return (p.Page - 1) * p.PageSize
}
+30
View File
@@ -0,0 +1,30 @@
package store
import "testing"
func TestPageFilterNormalize(t *testing.T) {
tests := []struct {
name string
in PageFilter
wantPage int
wantPageSize int
wantOffset int
}{
{name: "defaults", in: PageFilter{}, wantPage: 1, wantPageSize: 20, wantOffset: 0},
{name: "negative", in: PageFilter{Page: -1, PageSize: -5}, wantPage: 1, wantPageSize: 20, wantOffset: 0},
{name: "page2", in: PageFilter{Page: 2, PageSize: 10}, wantPage: 2, wantPageSize: 10, wantOffset: 10},
{name: "zero page size", in: PageFilter{Page: 3, PageSize: 0}, wantPage: 3, wantPageSize: 20, wantOffset: 40},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
p := tt.in
p.Normalize()
if p.Page != tt.wantPage || p.PageSize != tt.wantPageSize {
t.Fatalf("Normalize() = %+v, want page=%d page_size=%d", p, tt.wantPage, tt.wantPageSize)
}
if got := p.Offset(); got != tt.wantOffset {
t.Fatalf("Offset() = %d, want %d", got, tt.wantOffset)
}
})
}
}
+14 -4
View File
@@ -66,13 +66,23 @@ func (s *Store) GetRuleBySourceEvent(ctx context.Context, sourceID int, event st
return &r, nil
}
func (s *Store) ListRules(ctx context.Context) ([]model.Rule, error) {
rows, err := s.DB.QueryContext(ctx, `SELECT id, source_id, event, template_id, conditions, enabled, created_at, updated_at FROM notification_rule ORDER BY id`)
func (s *Store) ListRules(ctx context.Context, page PageFilter) ([]model.Rule, int, error) {
var count int
if err := s.DB.GetContext(ctx, &count, `SELECT COUNT(*) FROM notification_rule`); err != nil {
return nil, 0, fmt.Errorf("count rules: %w", err)
}
page.Normalize()
rows, err := s.DB.QueryContext(ctx, `SELECT id, source_id, event, template_id, conditions, enabled, created_at, updated_at FROM notification_rule ORDER BY id LIMIT ? OFFSET ?`, page.PageSize, page.Offset())
if err != nil {
return nil, fmt.Errorf("list rules: %w", err)
return nil, 0, fmt.Errorf("list rules: %w", err)
}
defer rows.Close()
return scanRules(rows)
rules, err := scanRules(rows)
if err != nil {
return nil, 0, err
}
return rules, count, nil
}
func (s *Store) UpdateRule(ctx context.Context, id int, r *model.Rule, channelIDs []int) error {
+12 -6
View File
@@ -54,13 +54,19 @@ func (s *Store) GetSourceByName(ctx context.Context, name string) (*model.Source
return &src, nil
}
func (s *Store) ListSources(ctx context.Context) ([]model.Source, error) {
sources := make([]model.Source, 0)
err := s.DB.SelectContext(ctx, &sources, `SELECT * FROM notification_source ORDER BY id`)
if err != nil {
return nil, fmt.Errorf("list sources: %w", err)
func (s *Store) ListSources(ctx context.Context, page PageFilter) ([]model.Source, int, error) {
var count int
if err := s.DB.GetContext(ctx, &count, `SELECT COUNT(*) FROM notification_source`); err != nil {
return nil, 0, fmt.Errorf("count sources: %w", err)
}
return sources, nil
page.Normalize()
sources := make([]model.Source, 0)
err := s.DB.SelectContext(ctx, &sources, `SELECT * FROM notification_source ORDER BY id LIMIT ? OFFSET ?`, page.PageSize, page.Offset())
if err != nil {
return nil, 0, fmt.Errorf("list sources: %w", err)
}
return sources, count, nil
}
func (s *Store) UpdateSource(ctx context.Context, id int, src *model.Source) error {
+12 -6
View File
@@ -36,13 +36,19 @@ func (s *Store) GetTemplateByName(ctx context.Context, name string) (*model.Temp
return &t, nil
}
func (s *Store) ListTemplates(ctx context.Context) ([]model.Template, error) {
templates := make([]model.Template, 0)
err := s.DB.SelectContext(ctx, &templates, `SELECT * FROM notification_template ORDER BY id`)
if err != nil {
return nil, fmt.Errorf("list templates: %w", err)
func (s *Store) ListTemplates(ctx context.Context, page PageFilter) ([]model.Template, int, error) {
var count int
if err := s.DB.GetContext(ctx, &count, `SELECT COUNT(*) FROM notification_template`); err != nil {
return nil, 0, fmt.Errorf("count templates: %w", err)
}
return templates, nil
page.Normalize()
templates := make([]model.Template, 0)
err := s.DB.SelectContext(ctx, &templates, `SELECT * FROM notification_template ORDER BY id LIMIT ? OFFSET ?`, page.PageSize, page.Offset())
if err != nil {
return nil, 0, fmt.Errorf("list templates: %w", err)
}
return templates, count, nil
}
func (s *Store) UpdateTemplate(ctx context.Context, id int, t *model.Template) error {