feat(receiver): storage layer and webhook payload compat
Dual-dialect store (SQLite via modernc.org, MySQL via go-sql-driver, both pure Go) with order-tolerant start/complete upserts, filtered and paginated listing, and aggregate queries (per-day, top domains, query types, statuses, duration percentiles, top clients). Sender gains optional EXPLOREDNS_WEBHOOK_TOKEN bearer auth; a round-trip test pins receiver structs byte-compatible with the sender payloads. Note: go directive moves to 1.25.0, required by modernc.org/sqlite. CI reads the version from go.mod so GOTOOLCHAIN=local stays satisfied. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Gary Hansen
co-authored by
Claude Fable 5
Claude Fable 5
parent
d96f25b1d6
commit
e8f56a1ef4
@@ -0,0 +1,322 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"math"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Traversal is one stored traversal row. Pointer fields are NULL until the
|
||||
// complete event arrives.
|
||||
type Traversal struct {
|
||||
ID string
|
||||
Domain string
|
||||
QueryType string
|
||||
AllRoots bool
|
||||
ClientIP string
|
||||
StartedAt time.Time
|
||||
DoneAt *time.Time
|
||||
DurationMS *int64
|
||||
Status string
|
||||
Error string
|
||||
ResultCount *int
|
||||
Summary string
|
||||
FirstSeen time.Time
|
||||
LastSeen time.Time
|
||||
}
|
||||
|
||||
// ListFilter narrows ListTraversals. Zero values mean "no constraint";
|
||||
// From/To bound started_at inclusively.
|
||||
type ListFilter struct {
|
||||
Domain string // substring match
|
||||
Status string
|
||||
From time.Time
|
||||
To time.Time
|
||||
}
|
||||
|
||||
const defaultListLimit = 50
|
||||
|
||||
// where renders the filter as a WHERE clause (or "") plus bind args.
|
||||
func (f ListFilter) where(d dialect) (string, []any) {
|
||||
var preds []string
|
||||
var args []any
|
||||
if f.Domain != "" {
|
||||
p, arg := d.likeContains("domain", f.Domain)
|
||||
preds = append(preds, p)
|
||||
args = append(args, arg)
|
||||
}
|
||||
if f.Status != "" {
|
||||
preds = append(preds, "status = ?")
|
||||
args = append(args, f.Status)
|
||||
}
|
||||
if !f.From.IsZero() {
|
||||
preds = append(preds, "started_at >= ?")
|
||||
args = append(args, fmtTime(f.From))
|
||||
}
|
||||
if !f.To.IsZero() {
|
||||
preds = append(preds, "started_at <= ?")
|
||||
args = append(args, fmtTime(f.To))
|
||||
}
|
||||
if len(preds) == 0 {
|
||||
return "", nil
|
||||
}
|
||||
return " WHERE " + strings.Join(preds, " AND "), args
|
||||
}
|
||||
|
||||
// ListTraversals returns one page of matching traversals ordered newest
|
||||
// first, plus the total match count. limit <= 0 selects a default page size.
|
||||
func (s *Store) ListTraversals(ctx context.Context, f ListFilter, limit, offset int) ([]Traversal, int, error) {
|
||||
where, args := f.where(s.d)
|
||||
|
||||
var total int
|
||||
if err := s.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM traversals"+where, args...).Scan(&total); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
if limit <= 0 {
|
||||
limit = defaultListLimit
|
||||
}
|
||||
if offset < 0 {
|
||||
offset = 0
|
||||
}
|
||||
q := `SELECT id, domain, query_type, all_roots, client_ip, started_at, done_at,
|
||||
duration_ms, status, error, result_count, summary, first_seen, last_seen
|
||||
FROM traversals` + where + ` ORDER BY started_at DESC, id LIMIT ? OFFSET ?`
|
||||
rows, err := s.db.QueryContext(ctx, q, append(args, limit, offset)...)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []Traversal
|
||||
for rows.Next() {
|
||||
var (
|
||||
tr Traversal
|
||||
startedAt, doneAt sql.NullString
|
||||
errMsg, summary sql.NullString
|
||||
durationMS sql.NullInt64
|
||||
resultCount sql.NullInt64
|
||||
firstSeen, lastSeen string
|
||||
)
|
||||
if err := rows.Scan(&tr.ID, &tr.Domain, &tr.QueryType, &tr.AllRoots, &tr.ClientIP,
|
||||
&startedAt, &doneAt, &durationMS, &tr.Status, &errMsg, &resultCount,
|
||||
&summary, &firstSeen, &lastSeen); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
tr.Error = errMsg.String
|
||||
tr.Summary = summary.String
|
||||
if durationMS.Valid {
|
||||
v := durationMS.Int64
|
||||
tr.DurationMS = &v
|
||||
}
|
||||
if resultCount.Valid {
|
||||
v := int(resultCount.Int64)
|
||||
tr.ResultCount = &v
|
||||
}
|
||||
if startedAt.Valid {
|
||||
if tr.StartedAt, err = parseDBTime(startedAt.String); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
}
|
||||
if doneAt.Valid {
|
||||
t, err := parseDBTime(doneAt.String)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
tr.DoneAt = &t
|
||||
}
|
||||
if tr.FirstSeen, err = parseDBTime(firstSeen); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
if tr.LastSeen, err = parseDBTime(lastSeen); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
out = append(out, tr)
|
||||
}
|
||||
return out, total, rows.Err()
|
||||
}
|
||||
|
||||
// Totals summarises whole-table traversal volume. Errors counts all-time
|
||||
// error-status rows so callers can derive an error rate.
|
||||
type Totals struct {
|
||||
AllTime int
|
||||
Last24h int
|
||||
Last7d int
|
||||
DistinctDomains int
|
||||
DistinctClients int
|
||||
Errors int
|
||||
}
|
||||
|
||||
// Totals returns whole-table totals with the recent windows measured back
|
||||
// from now. COALESCE covers the empty table, where SUM yields NULL.
|
||||
func (s *Store) Totals(ctx context.Context, now time.Time) (Totals, error) {
|
||||
var t Totals
|
||||
err := s.db.QueryRowContext(ctx, `SELECT COUNT(*),
|
||||
COALESCE(SUM(CASE WHEN started_at >= ? THEN 1 ELSE 0 END), 0),
|
||||
COALESCE(SUM(CASE WHEN started_at >= ? THEN 1 ELSE 0 END), 0),
|
||||
COUNT(DISTINCT domain),
|
||||
COUNT(DISTINCT client_ip),
|
||||
COALESCE(SUM(CASE WHEN status = ? THEN 1 ELSE 0 END), 0)
|
||||
FROM traversals`,
|
||||
fmtTime(now.Add(-24*time.Hour)), fmtTime(now.AddDate(0, 0, -7)), StatusError).
|
||||
Scan(&t.AllTime, &t.Last24h, &t.Last7d, &t.DistinctDomains, &t.DistinctClients, &t.Errors)
|
||||
return t, err
|
||||
}
|
||||
|
||||
// DayStats is one calendar day's traversal volume.
|
||||
type DayStats struct {
|
||||
Day string // "2006-01-02" (UTC)
|
||||
Total int
|
||||
Errors int
|
||||
}
|
||||
|
||||
// StatsPerDay returns per-day totals and error counts for the last days
|
||||
// calendar days (UTC), oldest first. Days with no traffic are omitted.
|
||||
func (s *Store) StatsPerDay(ctx context.Context, days int) ([]DayStats, error) {
|
||||
if days <= 0 {
|
||||
days = 1
|
||||
}
|
||||
midnight := time.Now().UTC().Truncate(24 * time.Hour)
|
||||
cutoff := midnight.AddDate(0, 0, -(days - 1))
|
||||
// substr on the stored value yields "YYYY-MM-DD" in both backends
|
||||
// (MySQL casts DATETIME to its string form implicitly).
|
||||
rows, err := s.db.QueryContext(ctx, `SELECT substr(started_at, 1, 10) AS day,
|
||||
COUNT(*), SUM(CASE WHEN status = ? THEN 1 ELSE 0 END)
|
||||
FROM traversals WHERE started_at >= ? GROUP BY day ORDER BY day`,
|
||||
StatusError, fmtTime(cutoff))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []DayStats
|
||||
for rows.Next() {
|
||||
var d DayStats
|
||||
if err := rows.Scan(&d.Day, &d.Total, &d.Errors); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, d)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// NameCount is a generic (value, count) aggregate row.
|
||||
type NameCount struct {
|
||||
Name string
|
||||
Count int
|
||||
}
|
||||
|
||||
// TopDomains returns the most-queried domains since the given time (zero =
|
||||
// all time), most frequent first, at most limit rows.
|
||||
func (s *Store) TopDomains(ctx context.Context, since time.Time, limit int) ([]NameCount, error) {
|
||||
return s.countBy(ctx, "domain", since, limit)
|
||||
}
|
||||
|
||||
// TopClientIPs returns the most active client IPs since the given time.
|
||||
func (s *Store) TopClientIPs(ctx context.Context, since time.Time, limit int) ([]NameCount, error) {
|
||||
return s.countBy(ctx, "client_ip", since, limit)
|
||||
}
|
||||
|
||||
// QueryTypeCounts returns traversal counts per query type since the given
|
||||
// time, most frequent first.
|
||||
func (s *Store) QueryTypeCounts(ctx context.Context, since time.Time) ([]NameCount, error) {
|
||||
return s.countBy(ctx, "query_type", since, 0)
|
||||
}
|
||||
|
||||
// StatusCounts returns traversal counts per status since the given time,
|
||||
// most frequent first.
|
||||
func (s *Store) StatusCounts(ctx context.Context, since time.Time) ([]NameCount, error) {
|
||||
return s.countBy(ctx, "status", since, 0)
|
||||
}
|
||||
|
||||
// countBy groups rows by col and counts them. col is always one of the
|
||||
// fixed column names above, never user input.
|
||||
func (s *Store) countBy(ctx context.Context, col string, since time.Time, limit int) ([]NameCount, error) {
|
||||
q := "SELECT " + col + ", COUNT(*) AS n FROM traversals"
|
||||
var args []any
|
||||
if !since.IsZero() {
|
||||
q += " WHERE started_at >= ?"
|
||||
args = append(args, fmtTime(since))
|
||||
}
|
||||
q += " GROUP BY " + col + " ORDER BY n DESC, " + col
|
||||
if limit > 0 {
|
||||
q += " LIMIT ?"
|
||||
args = append(args, limit)
|
||||
}
|
||||
rows, err := s.db.QueryContext(ctx, q, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []NameCount
|
||||
for rows.Next() {
|
||||
var nc NameCount
|
||||
if err := rows.Scan(&nc.Name, &nc.Count); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, nc)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// DurationStats summarises completed-traversal durations.
|
||||
type DurationStats struct {
|
||||
Count int
|
||||
AvgMS float64
|
||||
P50MS int64
|
||||
P95MS int64
|
||||
}
|
||||
|
||||
// Durations returns duration statistics for traversals started since the
|
||||
// given time (zero = all time). Percentiles are computed in Go (nearest
|
||||
// rank) because SQL percentile support differs between the backends.
|
||||
func (s *Store) Durations(ctx context.Context, since time.Time) (DurationStats, error) {
|
||||
q := "SELECT duration_ms FROM traversals WHERE duration_ms IS NOT NULL"
|
||||
var args []any
|
||||
if !since.IsZero() {
|
||||
q += " AND started_at >= ?"
|
||||
args = append(args, fmtTime(since))
|
||||
}
|
||||
rows, err := s.db.QueryContext(ctx, q, args...)
|
||||
if err != nil {
|
||||
return DurationStats{}, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var values []int64
|
||||
var sum int64
|
||||
for rows.Next() {
|
||||
var v int64
|
||||
if err := rows.Scan(&v); err != nil {
|
||||
return DurationStats{}, err
|
||||
}
|
||||
values = append(values, v)
|
||||
sum += v
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return DurationStats{}, err
|
||||
}
|
||||
if len(values) == 0 {
|
||||
return DurationStats{}, nil
|
||||
}
|
||||
sort.Slice(values, func(i, j int) bool { return values[i] < values[j] })
|
||||
return DurationStats{
|
||||
Count: len(values),
|
||||
AvgMS: float64(sum) / float64(len(values)),
|
||||
P50MS: percentile(values, 0.50),
|
||||
P95MS: percentile(values, 0.95),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// percentile returns the nearest-rank percentile of sorted values.
|
||||
func percentile(sorted []int64, q float64) int64 {
|
||||
rank := int(math.Ceil(q * float64(len(sorted))))
|
||||
if rank < 1 {
|
||||
rank = 1
|
||||
}
|
||||
return sorted[rank-1]
|
||||
}
|
||||
Reference in New Issue
Block a user