feat(receiver): storage layer and webhook payload compat

Dual-dialect store (SQLite via modernc.org, MySQL via go-sql-driver, both pure Go) with order-tolerant start/complete upserts, filtered and paginated listing, and aggregate queries (per-day, top domains, query types, statuses, duration percentiles, top clients). Sender gains optional EXPLOREDNS_WEBHOOK_TOKEN bearer auth; a round-trip test pins receiver structs byte-compatible with the sender payloads. Note: go directive moves to 1.25.0, required by modernc.org/sqlite. CI reads the version from go.mod so GOTOOLCHAIN=local stays satisfied. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-08 02:43:50 +10:00
parent d96f25b1d6
commit e8f56a1ef4
11 changed files with 1468 additions and 23 deletions
@@ -276,7 +276,7 @@ func newHandler(ctx context.Context) *Handler {
 		maxRunning: envInt("EXPLOREDNS_MAX_JOBS", defaultMaxRunningJobs),
 		version:    "dev",
 		limiter:    newRateLimiter(limit, window),
-		webhook:    newWebhookReporter(os.Getenv("EXPLOREDNS_WEBHOOK_URL")),
+		webhook:    newWebhookReporter(os.Getenv("EXPLOREDNS_WEBHOOK_URL"), os.Getenv("EXPLOREDNS_WEBHOOK_TOKEN")),
 		fp:         fingerprint.New(),
 	}

@@ -52,19 +52,22 @@ type webhookCompleteEvent struct {
 // single retry, and failures are logged but never surface to callers.
 type webhookReporter struct {
 	url        string
+	token      string
 	client     *http.Client
 	timeout    time.Duration
 	retryDelay time.Duration
 }

 // newWebhookReporter returns a reporter for url, or nil when url is empty
-// (webhook reporting disabled). A nil reporter is safe to call.
-func newWebhookReporter(url string) *webhookReporter {
+// (webhook reporting disabled). A nil reporter is safe to call. A non-empty
+// token is sent as an Authorization bearer token on every delivery.
+func newWebhookReporter(url, token string) *webhookReporter {
 	if url == "" {
 		return nil
 	}
 	return &webhookReporter{
 		url:        url,
+		token:      token,
 		client:     &http.Client{},
 		timeout:    5 * time.Second,
 		retryDelay: 2 * time.Second,
@@ -106,6 +109,9 @@ func (wr *webhookReporter) post(event string, body []byte) error {
 	}
 	req.Header.Set("Content-Type", "application/json")
 	req.Header.Set("X-ExploreDNS-Event", event)
+	if wr.token != "" {
+		req.Header.Set("Authorization", "Bearer "+wr.token)
+	}

 	resp, err := wr.client.Do(req)
 	if err != nil {
@@ -0,0 +1,126 @@
+package api
+
+import (
+	"bytes"
+	"encoding/json"
+	"testing"
+	"time"
+
+	rstore "gitea.hansenits.com.au/hits/ExploreDNS/internal/receiver/store"
+)
+
+// These tests pin the webhook payload contract between this package (the
+// sender) and internal/receiver/store (the receiver): every field a sender
+// struct marshals must decode into the receiver struct and vice versa.
+// DisallowUnknownFields turns any renamed or missing field into a failure.
+
+// decodeStrict unmarshals data into v, failing on unknown fields.
+func decodeStrict(t *testing.T, data []byte, v any) {
+	t.Helper()
+	dec := json.NewDecoder(bytes.NewReader(data))
+	dec.DisallowUnknownFields()
+	if err := dec.Decode(v); err != nil {
+		t.Fatalf("decode %s into %T: %v", data, v, err)
+	}
+}
+
+func TestWebhookStartEventReceiverCompat(t *testing.T) {
+	sent := webhookStartEvent{
+		Event:     webhookEventStart,
+		ID:        "job-1",
+		Domain:    "example.com",
+		QueryType: "AAAA",
+		AllRoots:  true,
+		ClientIP:  "203.0.113.9",
+		StartedAt: time.Date(2026, 7, 1, 10, 0, 0, 123456789, time.UTC),
+	}
+	raw, err := json.Marshal(sent)
+	if err != nil {
+		t.Fatalf("marshal sender: %v", err)
+	}
+
+	var got rstore.StartEvent
+	decodeStrict(t, raw, &got)
+	want := rstore.StartEvent{
+		Event:     sent.Event,
+		ID:        sent.ID,
+		Domain:    sent.Domain,
+		QueryType: sent.QueryType,
+		AllRoots:  sent.AllRoots,
+		ClientIP:  sent.ClientIP,
+		StartedAt: sent.StartedAt,
+	}
+	if got != want {
+		t.Errorf("receiver decoded %+v, want %+v", got, want)
+	}
+
+	// Round-trip back into the sender struct so receiver-only fields
+	// would also fail.
+	back, err := json.Marshal(got)
+	if err != nil {
+		t.Fatalf("marshal receiver: %v", err)
+	}
+	var sent2 webhookStartEvent
+	decodeStrict(t, back, &sent2)
+	if sent2 != sent {
+		t.Errorf("sender round-trip %+v, want %+v", sent2, sent)
+	}
+}
+
+func TestWebhookCompleteEventReceiverCompat(t *testing.T) {
+	sent := webhookCompleteEvent{
+		Event:       webhookEventComplete,
+		ID:          "job-1",
+		Domain:      "example.com",
+		QueryType:   "A",
+		ClientIP:    "203.0.113.9",
+		StartedAt:   time.Date(2026, 7, 1, 10, 0, 0, 0, time.UTC),
+		DoneAt:      time.Date(2026, 7, 1, 10, 0, 3, 500000000, time.UTC),
+		DurationMS:  3500,
+		Status:      statusError,
+		Error:       "traversal timed out after 5m0s",
+		ResultCount: 7,
+		Summary: &Summary{
+			Answers:  []SummaryAnswer{{Probability: 0.75, Records: []string{"example.com 300 IN A 192.0.2.1"}}},
+			ByStatus: []SummaryStatus{{Status: "servfail", Probability: 0.25}},
+		},
+	}
+	raw, err := json.Marshal(sent)
+	if err != nil {
+		t.Fatalf("marshal sender: %v", err)
+	}
+
+	var got rstore.CompleteEvent
+	decodeStrict(t, raw, &got)
+	if got.Event != sent.Event || got.ID != sent.ID || got.Domain != sent.Domain ||
+		got.QueryType != sent.QueryType || got.ClientIP != sent.ClientIP ||
+		!got.StartedAt.Equal(sent.StartedAt) || !got.DoneAt.Equal(sent.DoneAt) ||
+		got.DurationMS != sent.DurationMS || got.Status != sent.Status ||
+		got.Error != sent.Error || got.ResultCount != sent.ResultCount {
+		t.Errorf("receiver decoded %+v, want %+v", got, sent)
+	}
+
+	// The receiver keeps Summary as raw JSON; it must match the sender's
+	// marshalled Summary byte for byte.
+	wantSummary, err := json.Marshal(sent.Summary)
+	if err != nil {
+		t.Fatalf("marshal summary: %v", err)
+	}
+	if !bytes.Equal(got.Summary, wantSummary) {
+		t.Errorf("receiver Summary = %s, want %s", got.Summary, wantSummary)
+	}
+
+	back, err := json.Marshal(got)
+	if err != nil {
+		t.Fatalf("marshal receiver: %v", err)
+	}
+	var sent2 webhookCompleteEvent
+	decodeStrict(t, back, &sent2)
+	back2, err := json.Marshal(sent2)
+	if err != nil {
+		t.Fatalf("re-marshal sender: %v", err)
+	}
+	if !bytes.Equal(back2, raw) {
+		t.Errorf("sender round-trip = %s, want %s", back2, raw)
+	}
+}
@@ -235,7 +235,37 @@ func TestWebhook_RetriesOnceOnFailure(t *testing.T) {
 func TestWebhook_NilReporterSafe(t *testing.T) {
 	var wr *webhookReporter
 	wr.send("start", map[string]string{"event": "start"}) // must not panic
-	if newWebhookReporter("") != nil {
+	if newWebhookReporter("", "token") != nil {
 		t.Fatal("empty URL should disable the webhook reporter")
 	}
 }
+
+// TestWebhook_BearerToken verifies the Authorization header is sent exactly
+// when a token is configured (EXPLOREDNS_WEBHOOK_TOKEN on the real path).
+func TestWebhook_BearerToken(t *testing.T) {
+	for _, tc := range []struct {
+		name, token, wantAuth string
+	}{
+		{"token set", "s3cret", "Bearer s3cret"},
+		{"token unset", "", ""},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			done := make(chan string, 1)
+			ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				done <- r.Header.Get("Authorization")
+			}))
+			defer ts.Close()
+
+			newWebhookReporter(ts.URL, tc.token).send("start", map[string]string{"event": "start"})
+
+			select {
+			case got := <-done:
+				if got != tc.wantAuth {
+					t.Fatalf("Authorization = %q, want %q", got, tc.wantAuth)
+				}
+			case <-time.After(5 * time.Second):
+				t.Fatal("webhook was not delivered")
+			}
+		})
+	}
+}