comparing f94da9b737e4a4394fd45643b21bd8e70b0e1ae0 and main on seiso.moe/alky

+1 -1

docker-compose.yml

···

       17
       17
        
               soft: 262144

     

       18
       18
        
               hard: 262144

     

       19
       19
        
           healthcheck:

     

       20
       20
       -
             test: wget --no-verbose --tries=1 --spider http://localhost:8123/ping || exit 1

     

       20
       20
       +
             test: curl http://localhost:8123/ping || exit 1

     

       21
       21
        
             interval: 30s

     

       22
       22
        
             timeout: 5s

     

       23
       23
        
             retries: 3

+60

migrations/00002_modified_metrics.sql

···

       1
       1
       +
       -- +goose Up

     

       2
       2
       +
       ALTER TABLE alky_dns_queries

     

       3
       3
       +
           MODIFY COLUMN timestamp DateTime CODEC(Delta, ZSTD(1)),

     

       4
       4
       +
           MODIFY COLUMN instance_id String CODEC(ZSTD(1)),

     

       5
       5
       +
           MODIFY COLUMN query_name String CODEC(ZSTD(1)),

     

       6
       6
       +
           MODIFY COLUMN query_type LowCardinality(String) CODEC(ZSTD(1)),

     

       7
       7
       +
           MODIFY COLUMN query_class LowCardinality(String) CODEC(ZSTD(1)),

     

       8
       8
       +
           MODIFY COLUMN remote_addr String CODEC(ZSTD(1)),

     

       9
       9
       +
           MODIFY COLUMN response_code LowCardinality(String) CODEC(ZSTD(1)),

     

       10
       10
       +
           MODIFY COLUMN duration Int64 CODEC(T64, ZSTD(1)),

     

       11
       11
       +
           MODIFY COLUMN cache_hit Bool CODEC(ZSTD(1));

     

       12
       12
       +
       

     

       13
       13
       +
       ALTER TABLE alky_dns_queries MODIFY TTL timestamp + INTERVAL 30 DAY;

     

       14
       14
       +
       

     

       15
       15
       +
       ALTER TABLE alky_dns_cache_metrics 

     

       16
       16
       +
           DROP COLUMN IF EXISTS total_queries,

     

       17
       17
       +
           MODIFY COLUMN timestamp DateTime CODEC(Delta, ZSTD(1)),

     

       18
       18
       +
           MODIFY COLUMN instance_id String CODEC(ZSTD(1)),

     

       19
       19
       +
           MODIFY COLUMN cache_hits Int64 CODEC(T64, ZSTD(1)),

     

       20
       20
       +
           MODIFY COLUMN cache_misses Int64 CODEC(T64, ZSTD(1)),

     

       21
       21
       +
           MODIFY COLUMN negative_hits Int64 CODEC(T64, ZSTD(1)),

     

       22
       22
       +
           MODIFY COLUMN positive_hits Int64 CODEC(T64, ZSTD(1)),

     

       23
       23
       +
           MODIFY COLUMN evictions Int64 CODEC(T64, ZSTD(1)),

     

       24
       24
       +
           MODIFY COLUMN size Int64 CODEC(T64, ZSTD(1));

     

       25
       25
       +
       

     

       26
       26
       +
       ALTER TABLE alky_dns_cache_metrics 

     

       27
       27
       +
           ADD COLUMN IF NOT EXISTS expired_count Int64 CODEC(T64, ZSTD(1));

     

       28
       28
       +
       

     

       29
       29
       +
       ALTER TABLE alky_dns_cache_metrics MODIFY TTL timestamp + INTERVAL 30 DAY;

     

       30
       30
       +
       

     

       31
       31
       +
       -- +goose Down

     

       32
       32
       +
       ALTER TABLE alky_dns_queries

     

       33
       33
       +
           MODIFY COLUMN timestamp DateTime,

     

       34
       34
       +
           MODIFY COLUMN instance_id String,

     

       35
       35
       +
           MODIFY COLUMN query_name String,

     

       36
       36
       +
           MODIFY COLUMN query_type String,

     

       37
       37
       +
           MODIFY COLUMN query_class String,

     

       38
       38
       +
           MODIFY COLUMN remote_addr String,

     

       39
       39
       +
           MODIFY COLUMN response_code String,

     

       40
       40
       +
           MODIFY COLUMN duration Int64,

     

       41
       41
       +
           MODIFY COLUMN cache_hit Bool;

     

       42
       42
       +
       

     

       43
       43
       +
       ALTER TABLE alky_dns_queries MODIFY TTL timestamp + toIntervalDay(30);

     

       44
       44
       +
       

     

       45
       45
       +
       ALTER TABLE alky_dns_cache_metrics 

     

       46
       46
       +
           ADD COLUMN IF NOT EXISTS total_queries Int64 AFTER instance_id;

     

       47
       47
       +
       

     

       48
       48
       +
       ALTER TABLE alky_dns_cache_metrics 

     

       49
       49
       +
           MODIFY COLUMN timestamp DateTime,

     

       50
       50
       +
           MODIFY COLUMN instance_id String,

     

       51
       51
       +
           MODIFY COLUMN cache_hits Int64,

     

       52
       52
       +
           MODIFY COLUMN cache_misses Int64,

     

       53
       53
       +
           MODIFY COLUMN negative_hits Int64,

     

       54
       54
       +
           MODIFY COLUMN positive_hits Int64,

     

       55
       55
       +
           MODIFY COLUMN evictions Int64,

     

       56
       56
       +
           MODIFY COLUMN size Int;

     

       57
       57
       +
       

     

       58
       58
       +
       ALTER TABLE alky_dns_cache_metrics DROP COLUMN IF EXISTS expired_count;

     

       59
       59
       +
       

     

       60
       60
       +
       ALTER TABLE alky_dns_cache_metrics MODIFY TTL timestamp + toIntervalDay(30);

+2 -97

pkg/metrics/clickhouse.go

···

       17
       17
        
       	db          *sql.DB

     

       18
       18
        
       	config      *config.MetricsConfig

     

       19
       19
        
       	queryBuffer []QueryMetric

     

       20
       20
       -
       	cacheBuffer []CacheMetric

     

       21
       20
        
       	mu          sync.Mutex

     

       22
       21
        
       	stopChan    chan struct{}

     

       23
       22
        
       	wg          sync.WaitGroup

     
···

       33
       32
        
       	RemoteAddr   string

     

       34
       33
        
       	ResponseCode string

     

       35
       34
        
       	Duration     int64

     

       36
       36
       -
       	CacheHit     bool

     

       37
       37
       -
       }

     

       38
       38
       -
       

     

       39
       39
       -
       type CacheMetric struct {

     

       40
       40
       -
       	Timestamp    time.Time

     

       41
       41
       -
       	CacheHits    int64

     

       42
       42
       -
       	CacheMisses  int64

     

       43
       43
       -
       	NegativeHits int64

     

       44
       44
       -
       	PositiveHits int64

     

       45
       45
       -
       	Evictions    int64

     

       46
       46
       -
       	Size         int64

     

       47
       35
        
       }

     

       48
       36
        
       

     

       49
       37
        
       func NewClickHouseMetrics(config *config.MetricsConfig, logger *slog.Logger) (*ClickHouseMetrics, error) {

     
···

       67
       55
        
       		db:          db,

     

       68
       56
        
       		config:      config,

     

       69
       57
        
       		queryBuffer: make([]QueryMetric, 0, config.BatchSize),

     

       70
       70
       -
       		cacheBuffer: make([]CacheMetric, 0, config.BatchSize),

     

       71
       58
        
       		stopChan:    make(chan struct{}),

     

       72
       59
        
       		logger:      logger,

     

       73
       60
        
       	}

     
···

       93
       80
        
       	}

     

       94
       81
        
       }

     

       95
       82
        
       

     

       96
       96
       -
       func (m *ClickHouseMetrics) RecordCacheStats(metric CacheMetric) {

     

       97
       97
       -
       	m.mu.Lock()

     

       98
       98
       -
       	defer m.mu.Unlock()

     

       99
       99
       -
       

     

       100
       100
       -
       	m.cacheBuffer = append(m.cacheBuffer, metric)

     

       101
       101
       -
       	if len(m.cacheBuffer) >= m.config.BatchSize {

     

       102
       102
       -
       		m.flushCacheMetricsLocked()

     

       103
       103
       -
       	}

     

       104
       104
       -
       }

     

       105
       105
       -
       

     

       106
       83
        
       func (m *ClickHouseMetrics) flushLoop() {

     

       107
       84
        
       	defer m.wg.Done()

     

       108
       85
        
       	ticker := time.NewTicker(m.config.FlushInterval.Duration)

     
···

       131
       108
        
       	if len(m.queryBuffer) > 0 {

     

       132
       109
        
       		m.flushQueriesLocked()

     

       133
       110
        
       	}

     

       134
       134
       -
       	if len(m.cacheBuffer) > 0 {

     

       135
       135
       -
       		m.flushCacheMetricsLocked()

     

       136
       136
       -
       	}

     

       137
       111
        
       }

     

       138
       112
        
       

     

       139
       113
        
       func (m *ClickHouseMetrics) checkAndUpdateTTL() error {

     
···

       162
       136
        
       	if err := updateTableTTL("alky_dns_queries"); err != nil {

     

       163
       137
        
       		return err

     

       164
       138
        
       	}

     

       165
       165
       -
       	if err := updateTableTTL("alky_dns_cache_metrics"); err != nil {

     

       166
       166
       -
       		return err

     

       167
       167
       -
       	}

     

       168
       139
        
       

     

       169
       140
        
       	return nil

     

       170
       141
        
       }

     
···

       195
       166
        
       		INSERT INTO alky_dns_queries (

     

       196
       167
        
       			timestamp, instance_id, query_name, query_type, query_class,

     

       197
       168
        
       			remote_addr, response_code, duration, cache_hit

     

       198
       198
       -
       		) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)

     

       169
       169
       +
       		) VALUES (?, ?, ?, ?, ?, ?, ?, ?)

     

       199
       170
        
       	`)

     

       200
       171
        
       	if err != nil {

     

       201
       172
        
       		m.logger.Error("Failed to prepare statement for query metrics", "error", err)

     
···

       215
       186
        
       			metric.RemoteAddr,

     

       216
       187
        
       			metric.ResponseCode,

     

       217
       188
        
       			metric.Duration,

     

       218
       218
       -
       			metric.CacheHit,

     

       189
       189
       +
       			false,

     

       219
       190
        
       		)

     

       220
       191
        
       		if err != nil {

     

       221
       192
        
       			m.logger.Error("Failed to execute statement for query metric", "error", err, "metric_index", count)

     
···

       234
       205
        
       	m.queryBuffer = m.queryBuffer[:0]

     

       235
       206
        
       }

     

       236
       207
        
       

     

       237
       237
       -
       func (m *ClickHouseMetrics) flushCacheMetricsLocked() {

     

       238
       238
       -
       	if len(m.cacheBuffer) == 0 {

     

       239
       239
       -
       		return

     

       240
       240
       -
       	}

     

       241
       241
       -
       

     

       242
       242
       -
       	m.logger.Debug("Flushing cache metrics", "count", len(m.cacheBuffer))

     

       243
       243
       -
       	start := time.Now()

     

       244
       244
       -
       

     

       245
       245
       -
       	tx, err := m.db.Begin()

     

       246
       246
       -
       	if err != nil {

     

       247
       247
       -
       		m.logger.Error("Failed to begin transaction for cache metrics", "error", err)

     

       248
       248
       -
       		m.cacheBuffer = m.cacheBuffer[:0]

     

       249
       249
       -
       		return

     

       250
       250
       -
       	}

     

       251
       251
       -
       	defer func() {

     

       252
       252
       -
       		if err != nil {

     

       253
       253
       -
       			if rbErr := tx.Rollback(); rbErr != nil {

     

       254
       254
       -
       				m.logger.Error("Failed to rollback transaction for cache metrics", "error", rbErr)

     

       255
       255
       -
       			}

     

       256
       256
       -
       		}

     

       257
       257
       -
       	}()

     

       258
       258
       -
       

     

       259
       259
       -
       	stmt, err := tx.Prepare(`

     

       260
       260
       -
       		INSERT INTO alky_dns_cache_metrics (

     

       261
       261
       -
       			timestamp, instance_id, cache_hits, cache_misses,

     

       262
       262
       -
       			negative_hits, positive_hits, evictions,  size

     

       263
       263
       -
       		) VALUES (?, ?, ?, ?, ?, ?, ?, ?)

     

       264
       264
       -
       	`)

     

       265
       265
       -
       	if err != nil {

     

       266
       266
       -
       		m.logger.Error("Failed to prepare statement for cache metrics", "error", err)

     

       267
       267
       -
       		m.cacheBuffer = m.cacheBuffer[:0]

     

       268
       268
       -
       		return

     

       269
       269
       -
       	}

     

       270
       270
       -
       	defer stmt.Close()

     

       271
       271
       -
       

     

       272
       272
       -
       	count := 0

     

       273
       273
       -
       	for _, metric := range m.cacheBuffer {

     

       274
       274
       -
       		instanceID := GetInstanceID()

     

       275
       275
       -
       

     

       276
       276
       -
       		_, err = stmt.Exec(

     

       277
       277
       -
       			metric.Timestamp,

     

       278
       278
       -
       			instanceID,

     

       279
       279
       -
       			metric.CacheHits,

     

       280
       280
       -
       			metric.CacheMisses,

     

       281
       281
       -
       			metric.NegativeHits,

     

       282
       282
       -
       			metric.PositiveHits,

     

       283
       283
       -
       			metric.Evictions,

     

       284
       284
       -
       			metric.Size,

     

       285
       285
       -
       		)

     

       286
       286
       -
       		if err != nil {

     

       287
       287
       -
       			m.logger.Error("Failed to execute statement for cache metric", "error", err, "metric_index", count)

     

       288
       288
       -
       			return

     

       289
       289
       -
       		}

     

       290
       290
       -
       		count++

     

       291
       291
       -
       	}

     

       292
       292
       -
       

     

       293
       293
       -
       	err = tx.Commit()

     

       294
       294
       -
       	if err != nil {

     

       295
       295
       -
       		m.logger.Error("Failed to commit transaction for cache metrics", "error", err)

     

       296
       296
       -
       		return

     

       297
       297
       -
       	}

     

       298
       298
       -
       

     

       299
       299
       -
       	m.logger.Debug("Successfully flushed cache metrics", "count", count, "duration", time.Since(start))

     

       300
       300
       -
       	m.cacheBuffer = m.cacheBuffer[:0]

     

       301
       301
       -
       }

     

       302
       302
       -
       

     

       303
       208
        
       func (m *ClickHouseMetrics) Close() error {

     

       304
       209
        
       	m.logger.Info("Closing metrics client...")

     

       305
       210
        
       	close(m.stopChan)

pkg/rootservers/loader.go

···

       16
       16
        
       	}

     

       17
       17
        
       

     

       18
       18
        
       	for _, line := range bytes.Split(data, []byte{'\n'}) {

     

       19
       19
       +
       		if len(line) == 0 {

     

       20
       20
       +
       			continue

     

       21
       21
       +
       		}

     

       22
       22
       +
       

     

       19
       23
        
       		// skip comments

     

       20
       24
        
       		if line[0] == ';' {

     

       21
       25
        
       			continue

-7

docs/alky.toml

···

       49
       49
        
       # This uses time.ParseDuration semantics

     

       50
       50
        
       retention_period = "720h"

     

       51
       51
        
       

     

       52
       52
       -
       [cache]

     

       53
       53
       -
       # The maximum number of items to store in the cache.

     

       54
       54
       -
       max_items = 5000

     

       55
       55
       -
       

     

       56
       56
       -
       # How often the cache will evict items.

     

       57
       57
       -
       cleanup_interval = "5m"

     

       58
       58
       -
       

     

       59
       52
        
       [advanced]

     

       60
       53
        
       # Timeout (in milliseconds) for outgoing queries before being cancelled.

     

       61
       54
        
       query_timeout = 100

+1 -1

go.mod

···

       9
       9
        
       	github.com/ClickHouse/clickhouse-go/v2 v2.34.0

     

       10
       10
        
       	github.com/stretchr/testify v1.10.0

     

       11
       11
        
       	golang.org/x/time v0.11.0

     

       12
       12
       -
       	tangled.sh/seiso.moe/magna v0.0.1

     

       12
       12
       +
       	tangled.sh/seiso.moe/magna v0.0.2

     

       13
       13
        
       )

     

       14
       14
        
       

     

       15
       15
        
       require (

go.sum

···

       117
       117
        
       gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

     

       118
       118
        
       tangled.sh/seiso.moe/magna v0.0.1 h1:v8GM2y3xEinc0jGVxYf/33xtWJ74ES9EuTaMxXL8zxo=

     

       119
       119
        
       tangled.sh/seiso.moe/magna v0.0.1/go.mod h1:bqm+DTo2Pv4ITT0EnR079l++BJgoChBswSB/3KeijUk=

     

       120
       120
       +
       tangled.sh/seiso.moe/magna v0.0.2 h1:4VGPlqv/7tVyTtsR4Qkk8ZNypuNbmaeLogWzkpbHrRs=

     

       121
       121
       +
       tangled.sh/seiso.moe/magna v0.0.2/go.mod h1:bqm+DTo2Pv4ITT0EnR079l++BJgoChBswSB/3KeijUk=

+7 -7

pkg/dns/ratelimit.go

···

       17
       17
        
       }

     

       18
       18
        
       

     

       19
       19
        
       type rateLimiter struct {

     

       20
       20
       -
       	config RateLimitConfig

     

       21
       21
       -
       	limiters map[string]*ipRateLimiterEntry

     

       22
       22
       -
       	mu     sync.RWMutex

     

       20
       20
       +
       	config      RateLimitConfig

     

       21
       21
       +
       	limiters    map[string]*ipRateLimiterEntry

     

       22
       22
       +
       	mu          sync.RWMutex

     

       23
       23
        
       	stopCleanup chan struct{}

     

       24
       24
        
       }

     

       25
       25
        
       

     
···

       39
       39
        
       

     

       40
       40
        
       func newRateLimiter(config RateLimitConfig) *rateLimiter {

     

       41
       41
        
       	rl := &rateLimiter{

     

       42
       42
       -
       		config: config,

     

       43
       43
       -
       		limiters: make(map[string]*ipRateLimiterEntry),

     

       42
       42
       +
       		config:      config,

     

       43
       43
       +
       		limiters:    make(map[string]*ipRateLimiterEntry),

     

       44
       44
        
       		stopCleanup: make(chan struct{}),

     

       45
       45
        
       	}

     

       46
       46
        
       

     
···

       49
       49
        
       }

     

       50
       50
        
       

     

       51
       51
        
       func (rl *rateLimiter) allow(ip string) bool {

     

       52
       52
       -
       	rl.mu.Lock() 

     

       52
       52
       +
       	rl.mu.Lock()

     

       53
       53
        
       	defer rl.mu.Unlock()

     

       54
       54
        
       

     

       55
       55
        
       	entry, exists := rl.limiters[ip]

     
···

       58
       58
        
       	if !exists {

     

       59
       59
        
       		limiter := rate.NewLimiter(rate.Limit(rl.config.Rate), rl.config.Burst)

     

       60
       60
        
       		entry := &ipRateLimiterEntry{

     

       61
       61
       -
       			limiter: limiter,

     

       61
       61
       +
       			limiter:    limiter,

     

       62
       62
        
       			lastAccess: now,

     

       63
       63
        
       		}

     

       64
       64

+2 -2

pkg/dns/server.go

···

       21
       21
        
       )

     

       22
       22
        
       

     

       23
       23
        
       var (

     

       24
       24
       -
       	serverUDPBufferPool = sync.Pool {

     

       24
       24
       +
       	serverUDPBufferPool = sync.Pool{

     

       25
       25
        
       		New: func() any {

     

       26
       26
        
       			b := make([]byte, maxUDPBufferSize)

     

       27
       27
        
       			return &b

     

       28
       28
        
       		},

     

       29
       29
        
       	}

     

       30
       30
        
       

     

       31
       31
       -
       	resolverUDPBufferPool = sync.Pool {

     

       31
       31
       +
       	resolverUDPBufferPool = sync.Pool{

     

       32
       32
        
       		New: func() any {

     

       33
       33
        
       			b := make([]byte, maxResolverUDPBufferSize)

     

       34
       34
        
       			return &b

+2 -4

pkg/metrics/middleware.go

···

       23
       23
        
       		hostname = "unknown-host"

     

       24
       24
        
       	}

     

       25
       25
        
       

     

       26
       26
       -
       	pid := os.Getpid()

     

       27
       27
       -
       

     

       28
       26
        
       	if version != "" {

     

       29
       29
       -
       		instanceID = fmt.Sprintf("%s-%d-%s", hostname, pid, version)

     

       27
       27
       +
       		instanceID = fmt.Sprintf("%s-%s", hostname, version)

     

       30
       28
        
       	} else {

     

       31
       31
       -
       		instanceID = fmt.Sprintf("%s-%d-dev", hostname, pid)

     

       29
       29
       +
       		instanceID = fmt.Sprintf("%s-dev", hostname)

     

       32
       30
        
       	}

     

       33
       31
        
       }

     

       34
       32

Compare changes