feat(receiver): storage layer and webhook payload compat
Dual-dialect store (SQLite via modernc.org, MySQL via go-sql-driver, both pure Go) with order-tolerant start/complete upserts, filtered and paginated listing, and aggregate queries (per-day, top domains, query types, statuses, duration percentiles, top clients). Sender gains optional EXPLOREDNS_WEBHOOK_TOKEN bearer auth; a round-trip test pins receiver structs byte-compatible with the sender payloads. Note: go directive moves to 1.25.0, required by modernc.org/sqlite. CI reads the version from go.mod so GOTOOLCHAIN=local stays satisfied. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
d96f25b1d6
commit
e8f56a1ef4
@@ -0,0 +1,103 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// dialect abstracts the SQL syntax differences between the sqlite and
|
||||
// mysql backends. Both drivers use '?' placeholders, and all datetimes are
|
||||
// bound and scanned as "2006-01-02 15:04:05.000000" UTC strings so value
|
||||
// handling stays identical; only the constructs below diverge.
|
||||
type dialect interface {
|
||||
name() string
|
||||
|
||||
// datetimeType is the column type for datetime values. sqlite has no
|
||||
// datetime type (TEXT affinity stores our formatted strings verbatim,
|
||||
// which sort and compare lexically); mysql needs DATETIME(6) because
|
||||
// plain DATETIME truncates the sub-second precision we write.
|
||||
datetimeType() string
|
||||
|
||||
// upsert appends the insert-or-update clause to an INSERT statement,
|
||||
// overwriting cols with the values from the attempted insert. sqlite
|
||||
// spells this ON CONFLICT(id) DO UPDATE SET col=excluded.col; mysql
|
||||
// spells it ON DUPLICATE KEY UPDATE col=VALUES(col) (VALUES() is
|
||||
// deprecated in MySQL 8.0.20+ but is the only form MariaDB and older
|
||||
// MySQL also accept).
|
||||
upsert(insert string, cols []string) string
|
||||
|
||||
// createIndex creates an index if it does not exist. sqlite supports
|
||||
// CREATE INDEX IF NOT EXISTS; mysql has no IF NOT EXISTS for indexes,
|
||||
// so existence is checked via information_schema first.
|
||||
createIndex(ctx context.Context, db *sql.DB, index, table, column string) error
|
||||
|
||||
// likeContains returns a "col contains substr" predicate plus its
|
||||
// bind argument, with LIKE wildcards in substr escaped. The ESCAPE
|
||||
// literal differs: mysql string literals treat backslash as an escape
|
||||
// character (so the SQL needs '\\'), sqlite ones do not (so it needs
|
||||
// '\', and has no default escape character at all).
|
||||
likeContains(col, substr string) (predicate string, arg string)
|
||||
}
|
||||
|
||||
// escapeLike backslash-escapes LIKE pattern metacharacters in s.
|
||||
func escapeLike(s string) string {
|
||||
r := strings.NewReplacer(`\`, `\\`, `%`, `\%`, `_`, `\_`)
|
||||
return r.Replace(s)
|
||||
}
|
||||
|
||||
type sqliteDialect struct{}
|
||||
|
||||
func (sqliteDialect) name() string { return "sqlite" }
|
||||
func (sqliteDialect) datetimeType() string { return "TEXT" }
|
||||
|
||||
func (sqliteDialect) upsert(insert string, cols []string) string {
|
||||
sets := make([]string, len(cols))
|
||||
for i, c := range cols {
|
||||
sets[i] = c + "=excluded." + c
|
||||
}
|
||||
return insert + " ON CONFLICT(id) DO UPDATE SET " + strings.Join(sets, ", ")
|
||||
}
|
||||
|
||||
func (sqliteDialect) createIndex(ctx context.Context, db *sql.DB, index, table, column string) error {
|
||||
_, err := db.ExecContext(ctx,
|
||||
"CREATE INDEX IF NOT EXISTS "+index+" ON "+table+" ("+column+")")
|
||||
return err
|
||||
}
|
||||
|
||||
func (sqliteDialect) likeContains(col, substr string) (string, string) {
|
||||
return col + ` LIKE ? ESCAPE '\'`, "%" + escapeLike(substr) + "%"
|
||||
}
|
||||
|
||||
type mysqlDialect struct{}
|
||||
|
||||
func (mysqlDialect) name() string { return "mysql" }
|
||||
func (mysqlDialect) datetimeType() string { return "DATETIME(6)" }
|
||||
|
||||
func (mysqlDialect) upsert(insert string, cols []string) string {
|
||||
sets := make([]string, len(cols))
|
||||
for i, c := range cols {
|
||||
sets[i] = c + "=VALUES(" + c + ")"
|
||||
}
|
||||
return insert + " ON DUPLICATE KEY UPDATE " + strings.Join(sets, ", ")
|
||||
}
|
||||
|
||||
func (mysqlDialect) createIndex(ctx context.Context, db *sql.DB, index, table, column string) error {
|
||||
var n int
|
||||
err := db.QueryRowContext(ctx,
|
||||
`SELECT COUNT(*) FROM information_schema.statistics
|
||||
WHERE table_schema = DATABASE() AND table_name = ? AND index_name = ?`,
|
||||
table, index).Scan(&n)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n > 0 {
|
||||
return nil
|
||||
}
|
||||
_, err = db.ExecContext(ctx, "CREATE INDEX "+index+" ON "+table+" ("+column+")")
|
||||
return err
|
||||
}
|
||||
|
||||
func (mysqlDialect) likeContains(col, substr string) (string, string) {
|
||||
return col + ` LIKE ? ESCAPE '\\'`, "%" + escapeLike(substr) + "%"
|
||||
}
|
||||
Reference in New Issue
Block a user