feat(receiver): ingest server and admin dashboard

POST /webhook (bearer-token auth, strict validation, 1MB cap) and
/healthz stay open; everything under /admin requires basic auth
(RECEIVER_ADMIN_USER/PASSWORD, refuses to start without a password,
constant-time compares). Admin JSON APIs for the traversal log
(filters, pagination) and stats, plus an embedded dashboard: stat
cards, four Chart.js charts (lazy CDN load with graceful degradation),
filterable log with expandable summaries, auto-refresh. Fetches
resolve against location.origin so credentialed bookmark URLs work.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Gary Hansen
2026-07-08 02:43:51 +10:00
co-authored by Claude Fable 5
parent e8f56a1ef4
commit beb595442f
6 changed files with 1765 additions and 0 deletions
+482
View File
@@ -0,0 +1,482 @@
package server
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"
"gitea.hansenits.com.au/hits/ExploreDNS/internal/receiver/store"
)
const (
testAdminUser = "admin"
testAdminPass = "swordfish"
)
// get performs a GET with optional basic-auth credentials.
func get(h http.Handler, path, user, pass string) *httptest.ResponseRecorder {
req := httptest.NewRequest(http.MethodGet, path, nil)
if user != "" || pass != "" {
req.SetBasicAuth(user, pass)
}
w := httptest.NewRecorder()
h.ServeHTTP(w, req)
return w
}
func TestStartRequiresAdminPassword(t *testing.T) {
st, err := store.OpenSQLite(filepath.Join(t.TempDir(), "receiver.db"))
if err != nil {
t.Fatalf("OpenSQLite: %v", err)
}
defer st.Close()
srv := New("127.0.0.1:0", st)
err = srv.Start()
if err == nil {
srv.Shutdown(time.Second) //nolint:errcheck
t.Fatal("Start succeeded without an admin password")
}
if !strings.Contains(err.Error(), "RECEIVER_ADMIN_PASSWORD") {
t.Errorf("error = %q, want mention of RECEIVER_ADMIN_PASSWORD", err)
}
}
func TestAdminAuthMatrix(t *testing.T) {
paths := []string{"/admin", "/admin/api/traversals", "/admin/api/stats"}
creds := []struct {
name string
user, pass string
want int
}{
{"no credentials", "", "", http.StatusUnauthorized},
{"wrong user", "root", testAdminPass, http.StatusUnauthorized},
{"wrong password", testAdminUser, "nope", http.StatusUnauthorized},
{"correct credentials", testAdminUser, testAdminPass, http.StatusOK},
}
h, _ := newTestHandler(t, "")
for _, path := range paths {
for _, tc := range creds {
t.Run(path+" "+tc.name, func(t *testing.T) {
w := get(h, path, tc.user, tc.pass)
if w.Code != tc.want {
t.Fatalf("status = %d, want %d (%s)", w.Code, tc.want, w.Body)
}
if tc.want == http.StatusUnauthorized {
if got := w.Header().Get("WWW-Authenticate"); !strings.Contains(got, `Basic realm=`) {
t.Errorf("WWW-Authenticate = %q, want Basic realm", got)
}
}
})
}
}
}
// TestAdminSubtreeAuthenticated pins that unknown and unauthenticated paths
// under /admin still 401 rather than falling through to a handler.
func TestAdminSubtreeAuthenticated(t *testing.T) {
h, _ := newTestHandler(t, "")
for _, path := range []string{"/admin/", "/admin/secret", "/admin/api/other"} {
if got := get(h, path, "", "").Code; got != http.StatusUnauthorized {
t.Errorf("GET %s without creds = %d, want 401", path, got)
}
}
}
// TestAdminLockedWithoutPassword pins that a handler built without an admin
// password rejects everything, even blank credentials.
func TestAdminLockedWithoutPassword(t *testing.T) {
st, err := store.OpenSQLite(filepath.Join(t.TempDir(), "receiver.db"))
if err != nil {
t.Fatalf("OpenSQLite: %v", err)
}
defer st.Close()
h := newHandler(st, "test-version", "", "admin", "")
req := httptest.NewRequest(http.MethodGet, "/admin", nil)
req.SetBasicAuth("admin", "")
w := httptest.NewRecorder()
h.ServeHTTP(w, req)
if w.Code != http.StatusUnauthorized {
t.Errorf("status = %d, want 401", w.Code)
}
}
// TestWebhookAndHealthzStayOpen asserts the sender and k8s probe endpoints
// need no admin credentials.
func TestWebhookAndHealthzStayOpen(t *testing.T) {
h, _ := newTestHandler(t, "")
if got := post(h, startJSON, nil).Code; got != http.StatusNoContent {
t.Errorf("POST /webhook without creds = %d, want 204", got)
}
if got := get(h, "/healthz", "", "").Code; got != http.StatusOK {
t.Errorf("GET /healthz without creds = %d, want 200", got)
}
}
// seedList inserts the deterministic fixed-date dataset used by the
// traversals endpoint tests (same shape as the store package's seed).
func seedList(t *testing.T, st *store.Store) {
t.Helper()
ctx := context.Background()
base := time.Date(2026, 7, 1, 0, 0, 0, 0, time.UTC)
rows := []struct {
id, domain, qtype, ip, status string
day int
durMS int64
}{
{"a1", "example.com", "A", "203.0.113.1", store.StatusComplete, 0, 100},
{"a2", "example.com", "AAAA", "203.0.113.1", store.StatusComplete, 0, 200},
{"a3", "sub.example.com", "A", "203.0.113.2", store.StatusError, 1, 300},
{"a4", "other.net", "MX", "203.0.113.3", store.StatusComplete, 1, 400},
{"a5", "other.net", "A", "203.0.113.1", store.StatusComplete, 2, 500},
{"a6", "under_score.org", "A", "203.0.113.4", store.StatusComplete, 2, 600},
}
for i, r := range rows {
started := base.AddDate(0, 0, r.day).Add(time.Duration(i) * time.Minute)
ev := store.CompleteEvent{
ID: r.id, Domain: r.domain, QueryType: r.qtype, ClientIP: r.ip,
StartedAt: started, DoneAt: started.Add(time.Duration(r.durMS) * time.Millisecond),
DurationMS: r.durMS, Status: r.status, ResultCount: 1,
Summary: json.RawMessage(`{"answers":[{"probability":1}]}`),
}
if r.status == store.StatusError {
ev.Error = "lookup failed"
}
if err := st.RecordComplete(ctx, ev); err != nil {
t.Fatalf("seed %s: %v", r.id, err)
}
}
if err := st.RecordStart(ctx, store.StartEvent{
ID: "a7", Domain: "running.io", QueryType: "A", ClientIP: "203.0.113.5",
StartedAt: base.AddDate(0, 0, 2).Add(time.Hour),
}); err != nil {
t.Fatalf("seed a7: %v", err)
}
}
type listResponse struct {
Total int `json:"total"`
Items []struct {
ID string `json:"id"`
Domain string `json:"domain"`
QueryType string `json:"query_type"`
ClientIP string `json:"client_ip"`
StartedAt string `json:"started_at"`
DoneAt string `json:"done_at"`
DurationMS *int64 `json:"duration_ms"`
Status string `json:"status"`
Error string `json:"error"`
ResultCount *int `json:"result_count"`
Summary json.RawMessage `json:"summary"`
ReceivedAt string `json:"received_at"`
} `json:"items"`
}
func listGET(t *testing.T, h http.Handler, query string) listResponse {
t.Helper()
w := get(h, "/admin/api/traversals"+query, testAdminUser, testAdminPass)
if w.Code != http.StatusOK {
t.Fatalf("GET %s = %d (%s)", query, w.Code, w.Body)
}
var resp listResponse
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatalf("decode %s: %v", query, err)
}
return resp
}
func TestAdminTraversalsEndpoint(t *testing.T) {
h, st := newTestHandler(t, "")
seedList(t, st)
// Default page: everything, newest first.
resp := listGET(t, h, "")
if resp.Total != 7 || len(resp.Items) != 7 {
t.Fatalf("total = %d, len = %d, want 7 and 7", resp.Total, len(resp.Items))
}
first := resp.Items[0]
if first.ID != "a7" || first.Status != store.StatusRunning {
t.Errorf("first item = %s/%s, want a7/running", first.ID, first.Status)
}
if first.DurationMS != nil || first.DoneAt != "" || len(first.Summary) != 0 {
t.Errorf("running item has completion fields: %+v", first)
}
if first.StartedAt != "2026-07-03T01:00:00Z" {
t.Errorf("started_at = %q, want 2026-07-03T01:00:00Z", first.StartedAt)
}
if first.ReceivedAt == "" {
t.Error("received_at empty")
}
// Completed rows carry duration, result count, and raw summary JSON.
last := resp.Items[6]
if last.ID != "a1" || last.DurationMS == nil || *last.DurationMS != 100 ||
last.ResultCount == nil || *last.ResultCount != 1 {
t.Errorf("oldest item = %+v, want a1 with duration 100 and 1 result", last)
}
if string(last.Summary) != `{"answers":[{"probability":1}]}` {
t.Errorf("summary = %s", last.Summary)
}
// Pagination.
resp = listGET(t, h, "?limit=3&offset=3")
if resp.Total != 7 || len(resp.Items) != 3 {
t.Fatalf("page 2 total = %d, len = %d, want 7 and 3", resp.Total, len(resp.Items))
}
if resp.Items[0].ID != "a4" || resp.Items[1].ID != "a3" || resp.Items[2].ID != "a2" {
t.Errorf("page 2 order = %s,%s,%s, want a4,a3,a2",
resp.Items[0].ID, resp.Items[1].ID, resp.Items[2].ID)
}
// Filters.
if resp := listGET(t, h, "?domain=example"); resp.Total != 3 {
t.Errorf("domain filter total = %d, want 3", resp.Total)
}
resp = listGET(t, h, "?status=error")
if resp.Total != 1 || resp.Items[0].ID != "a3" || resp.Items[0].Error != "lookup failed" {
t.Errorf("status filter = %+v, want a3 with error", resp)
}
resp = listGET(t, h, "?from=2026-07-02T00:00:00Z&to=2026-07-02T23:59:59Z")
if resp.Total != 2 {
t.Errorf("time window total = %d, want 2", resp.Total)
}
// Limit above the cap succeeds and is clamped rather than rejected.
if resp := listGET(t, h, "?limit=1000"); resp.Total != 7 {
t.Errorf("clamped limit total = %d, want 7", resp.Total)
}
}
func TestAdminTraversalsBadParams(t *testing.T) {
h, _ := newTestHandler(t, "")
for _, q := range []string{"?limit=abc", "?limit=0", "?offset=-1", "?from=notatime", "?to=2026-13-99"} {
if got := get(h, "/admin/api/traversals"+q, testAdminUser, testAdminPass).Code; got != http.StatusBadRequest {
t.Errorf("GET %s = %d, want 400", q, got)
}
}
if got := get(h, "/admin/api/stats?days=x", testAdminUser, testAdminPass).Code; got != http.StatusBadRequest {
t.Errorf("GET stats?days=x = %d, want 400", got)
}
}
type statsBody struct {
Days int `json:"days"`
Totals struct {
AllTime int `json:"all_time"`
Last24h int `json:"last_24h"`
Last7d int `json:"last_7d"`
DistinctDomains int `json:"distinct_domains"`
DistinctClients int `json:"distinct_clients"`
ErrorRate float64 `json:"error_rate"`
} `json:"totals"`
PerDay []struct {
Day string `json:"day"`
Total int `json:"total"`
Errors int `json:"errors"`
} `json:"per_day"`
TopDomains []struct {
Name string `json:"name"`
Count int `json:"count"`
} `json:"top_domains"`
QueryTypes []struct {
Name string `json:"name"`
Count int `json:"count"`
} `json:"query_types"`
Statuses []struct {
Name string `json:"name"`
Count int `json:"count"`
} `json:"statuses"`
Durations struct {
Count int `json:"count"`
AvgMS float64 `json:"avg_ms"`
P50MS int64 `json:"p50_ms"`
P95MS int64 `json:"p95_ms"`
} `json:"durations"`
TopClients []struct {
Name string `json:"name"`
Count int `json:"count"`
} `json:"top_clients"`
}
func statsGET(t *testing.T, h http.Handler, query string) statsBody {
t.Helper()
w := get(h, "/admin/api/stats"+query, testAdminUser, testAdminPass)
if w.Code != http.StatusOK {
t.Fatalf("GET stats%s = %d (%s)", query, w.Code, w.Body)
}
var resp statsBody
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatalf("decode stats: %v", err)
}
return resp
}
func TestAdminStatsEndpoint(t *testing.T) {
h, st := newTestHandler(t, "")
ctx := context.Background()
now := time.Now().UTC()
complete := func(id, domain, qtype, ip, status string, ago time.Duration, durMS int64) {
t.Helper()
ev := store.CompleteEvent{
ID: id, Domain: domain, QueryType: qtype, ClientIP: ip,
StartedAt: now.Add(-ago), DoneAt: now.Add(-ago).Add(time.Duration(durMS) * time.Millisecond),
DurationMS: durMS, Status: status, ResultCount: 1,
}
if status == store.StatusError {
ev.Error = "boom"
}
if err := st.RecordComplete(ctx, ev); err != nil {
t.Fatalf("seed %s: %v", id, err)
}
}
complete("r1", "a.com", "A", "203.0.113.1", store.StatusComplete, time.Hour, 100)
complete("r2", "a.com", "A", "203.0.113.2", store.StatusError, 2*time.Hour, 200)
complete("r3", "b.net", "A", "203.0.113.1", store.StatusComplete, 30*time.Hour, 300)
complete("r4", "c.org", "MX", "203.0.113.3", store.StatusComplete, 8*24*time.Hour, 400)
if err := st.RecordStart(ctx, store.StartEvent{
ID: "r5", Domain: "a.com", QueryType: "A", ClientIP: "203.0.113.1",
StartedAt: now.Add(-time.Hour),
}); err != nil {
t.Fatalf("seed r5: %v", err)
}
s := statsGET(t, h, "?days=30")
if s.Days != 30 {
t.Errorf("days = %d, want 30", s.Days)
}
tot := s.Totals
if tot.AllTime != 5 || tot.Last24h != 3 || tot.Last7d != 4 ||
tot.DistinctDomains != 3 || tot.DistinctClients != 3 {
t.Errorf("totals = %+v, want all 5, 24h 3, 7d 4, domains 3, clients 3", tot)
}
if tot.ErrorRate != 0.2 {
t.Errorf("error_rate = %v, want 0.2", tot.ErrorRate)
}
var perDayTotal, perDayErrors int
for _, d := range s.PerDay {
perDayTotal += d.Total
perDayErrors += d.Errors
}
if perDayTotal != 5 || perDayErrors != 1 {
t.Errorf("per_day sums = %d/%d, want 5/1 (%+v)", perDayTotal, perDayErrors, s.PerDay)
}
if len(s.TopDomains) != 3 || s.TopDomains[0].Name != "a.com" || s.TopDomains[0].Count != 3 {
t.Errorf("top_domains = %+v, want a.com x3 leading 3", s.TopDomains)
}
if len(s.QueryTypes) != 2 || s.QueryTypes[0].Name != "A" || s.QueryTypes[0].Count != 4 ||
s.QueryTypes[1].Name != "MX" || s.QueryTypes[1].Count != 1 {
t.Errorf("query_types = %+v, want A x4, MX x1", s.QueryTypes)
}
if len(s.Statuses) != 3 || s.Statuses[0].Name != store.StatusComplete || s.Statuses[0].Count != 3 {
t.Errorf("statuses = %+v, want complete x3 leading", s.Statuses)
}
d := s.Durations
if d.Count != 4 || d.AvgMS != 250 || d.P50MS != 200 || d.P95MS != 400 {
t.Errorf("durations = %+v, want count 4 avg 250 p50 200 p95 400", d)
}
if len(s.TopClients) != 3 || s.TopClients[0].Name != "203.0.113.1" || s.TopClients[0].Count != 3 {
t.Errorf("top_clients = %+v, want 203.0.113.1 x3 leading 3", s.TopClients)
}
// A 7-day window drops r4 from the windowed aggregates but not from the
// fixed totals.
s = statsGET(t, h, "?days=7")
if s.Totals.AllTime != 5 {
t.Errorf("7d all_time = %d, want 5", s.Totals.AllTime)
}
if s.Durations.Count != 3 || s.Durations.AvgMS != 200 {
t.Errorf("7d durations = %+v, want count 3 avg 200", s.Durations)
}
if len(s.QueryTypes) != 1 || s.QueryTypes[0].Count != 4 {
t.Errorf("7d query_types = %+v, want A x4 only", s.QueryTypes)
}
// days above the cap clamps to 365.
if s := statsGET(t, h, "?days=9999"); s.Days != 365 {
t.Errorf("days = %d, want 365", s.Days)
}
// default is 30.
if s := statsGET(t, h, ""); s.Days != 30 {
t.Errorf("default days = %d, want 30", s.Days)
}
}
func TestAdminStatsEmptyStore(t *testing.T) {
h, _ := newTestHandler(t, "")
s := statsGET(t, h, "")
if s.Totals.AllTime != 0 || s.Totals.ErrorRate != 0 {
t.Errorf("empty totals = %+v, want zeros", s.Totals)
}
// Arrays must be present (possibly empty), never null.
w := get(h, "/admin/api/stats", testAdminUser, testAdminPass)
body := w.Body.String()
for _, key := range []string{`"per_day":[]`, `"top_domains":[]`, `"query_types":[]`, `"statuses":[]`, `"top_clients":[]`} {
if !strings.Contains(body, key) {
t.Errorf("stats body missing %s: %s", key, body)
}
}
}
// TestStaticAdmin_Markup asserts the admin page ships the stat cards,
// charts, log table, and controls the API is built for.
func TestStaticAdmin_Markup(t *testing.T) {
h, _ := newTestHandler(t, "")
w := get(h, "/admin", testAdminUser, testAdminPass)
if w.Code != http.StatusOK {
t.Fatalf("GET /admin = %d", w.Code)
}
if ct := w.Header().Get("Content-Type"); !strings.HasPrefix(ct, "text/html") {
t.Errorf("Content-Type = %q, want text/html", ct)
}
page := w.Body.String()
for _, want := range []string{
`id="statAllTime"`, `id="statLast24h"`, `id="statLast7d"`,
`id="statDomains"`, `id="statClients"`, `id="statErrorRate"`,
`id="perDayChart"`, `id="statusChart"`, `id="domainsChart"`, `id="typesChart"`,
`cdn.jsdelivr.net/npm/chart.js@4`, // Chart.js 4, lazy-loaded
`id="logTable"`, `id="logBody"`,
`<th>Received</th><th>Domain</th><th>Type</th><th>Status</th><th>Duration</th><th>Client IP</th><th>Results</th>`,
`id="filterDomain"`, `id="filterStatus"`, `id="filterFrom"`, `id="filterTo"`, `id="applyFilters"`,
`id="prevPage"`, `id="nextPage"`, `id="pageInfo"`,
`id="autoRefresh"`, `id="daysSelect"`,
`/admin/api/stats`, `/admin/api/traversals`,
} {
if !strings.Contains(page, want) {
t.Errorf("admin.html missing %q", want)
}
}
}
// TestStaticAdmin_ScriptSyntax runs node --check over the inline admin
// script so syntax errors fail in CI, not in the browser.
func TestStaticAdmin_ScriptSyntax(t *testing.T) {
nodeBin, err := exec.LookPath("node")
if err != nil {
t.Skip("node not installed")
}
page := string(adminHTML)
start := strings.Index(page, "<script>")
end := strings.LastIndex(page, "</script>")
if start < 0 || end < 0 || end < start {
t.Fatal("admin.html has no inline script")
}
src := page[start+len("<script>") : end]
f := filepath.Join(t.TempDir(), "admin.js")
if err := os.WriteFile(f, []byte(src), 0o644); err != nil {
t.Fatal(err)
}
if out, err := exec.Command(nodeBin, "--check", f).CombinedOutput(); err != nil {
t.Fatalf("node --check: %v\n%s", err, out)
}
}