Merge pull request 'feat/telemetry-receiver' (#29) from feat/telemetry-receiver into main
Reviewed-on: http://gitea.hansenits.com.au/hits/ExploreDNS/pulls/29
This commit was merged in pull request #29.
This commit is contained in:
@@ -67,3 +67,13 @@ jobs:
|
||||
tags: |
|
||||
gitea.hansenits.com.au/hits/exploredns-web:latest
|
||||
gitea.hansenits.com.au/hits/exploredns-web:${{ github.sha }}
|
||||
|
||||
- name: Build and push receiver image
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: Dockerfile.receiver
|
||||
push: true
|
||||
tags: |
|
||||
gitea.hansenits.com.au/hits/exploredns-receiver:latest
|
||||
gitea.hansenits.com.au/hits/exploredns-receiver:${{ github.sha }}
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
name: Release
|
||||
|
||||
# Builds release artifacts when a version tag (v*) is pushed:
|
||||
# - cross-compiled CLI + server binaries attached to the Gitea release
|
||||
# - version-tagged docker images for the CLI and web server
|
||||
# - cross-compiled CLI + server + receiver binaries attached to the Gitea release
|
||||
# - version-tagged docker images for the CLI, web server, and receiver
|
||||
# The owner usually creates the Gitea release by hand with notes; this
|
||||
# workflow attaches assets to it (creating a bare release only when none
|
||||
# exists) and skips assets that are already attached, so re-runs are safe.
|
||||
@@ -42,10 +42,13 @@ jobs:
|
||||
CGO_ENABLED=0 GOOS="$GOOS" GOARCH="$GOARCH" \
|
||||
go build -trimpath -ldflags "-s -w -X main.version=${TAG}" \
|
||||
-o "${OUT}/exploredns-server${EXT}" ./cmd/server
|
||||
CGO_ENABLED=0 GOOS="$GOOS" GOARCH="$GOARCH" \
|
||||
go build -trimpath -ldflags "-s -w -X main.version=${TAG}" \
|
||||
-o "${OUT}/exploredns-receiver${EXT}" ./cmd/exploredns-receiver
|
||||
if [ "$GOOS" = "windows" ]; then
|
||||
(cd "$OUT" && zip -q "../exploredns_${TAG}_${GOOS}_${GOARCH}.zip" exploredns.exe exploredns-server.exe)
|
||||
(cd "$OUT" && zip -q "../exploredns_${TAG}_${GOOS}_${GOARCH}.zip" exploredns.exe exploredns-server.exe exploredns-receiver.exe)
|
||||
else
|
||||
tar -czf "dist/exploredns_${TAG}_${GOOS}_${GOARCH}.tar.gz" -C "$OUT" exploredns exploredns-server
|
||||
tar -czf "dist/exploredns_${TAG}_${GOOS}_${GOARCH}.tar.gz" -C "$OUT" exploredns exploredns-server exploredns-receiver
|
||||
fi
|
||||
rm -rf "$OUT"
|
||||
done
|
||||
@@ -130,3 +133,15 @@ jobs:
|
||||
tags: |
|
||||
gitea.hansenits.com.au/hits/exploredns-web:latest
|
||||
gitea.hansenits.com.au/hits/exploredns-web:${{ github.ref_name }}
|
||||
|
||||
- name: Build and push receiver image
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: Dockerfile.receiver
|
||||
push: true
|
||||
build-args: |
|
||||
VERSION=${{ github.ref_name }}
|
||||
tags: |
|
||||
gitea.hansenits.com.au/hits/exploredns-receiver:latest
|
||||
gitea.hansenits.com.au/hits/exploredns-receiver:${{ github.ref_name }}
|
||||
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
# Build stage
|
||||
FROM golang:1.24-alpine AS builder
|
||||
FROM golang:1.25-alpine AS builder
|
||||
|
||||
WORKDIR /src
|
||||
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
# Build stage
|
||||
FROM golang:1.25-alpine AS builder
|
||||
|
||||
WORKDIR /src
|
||||
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
|
||||
COPY . .
|
||||
|
||||
ARG VERSION=dev
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w -X main.version=${VERSION}" -o /out/exploredns-receiver ./cmd/exploredns-receiver
|
||||
|
||||
# Final stage
|
||||
FROM alpine:3.21
|
||||
|
||||
RUN apk --no-cache add ca-certificates
|
||||
|
||||
COPY --from=builder /out/exploredns-receiver /usr/local/bin/exploredns-receiver
|
||||
|
||||
ENV RECEIVER_SQLITE_PATH=/data/exploredns-receiver.db
|
||||
VOLUME /data
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
ENTRYPOINT ["exploredns-receiver"]
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
# Build stage
|
||||
FROM golang:1.24-alpine AS builder
|
||||
FROM golang:1.25-alpine AS builder
|
||||
|
||||
WORKDIR /src
|
||||
|
||||
|
||||
@@ -1,12 +1,13 @@
|
||||
BINARY_NAME=exploredns
|
||||
SERVER_BINARY_NAME=exploredns-server
|
||||
RECEIVER_BINARY_NAME=exploredns-receiver
|
||||
BUILD_DIR=bin
|
||||
GO=go
|
||||
GOFLAGS=-v
|
||||
VERSION?=$(shell git describe --tags --always 2>/dev/null || echo dev)
|
||||
LDFLAGS=-ldflags "-X main.version=$(VERSION)"
|
||||
|
||||
.PHONY: build build-server build-all test lint clean cover deploy deploy-status
|
||||
.PHONY: build build-server build-receiver build-all test lint clean cover deploy deploy-status
|
||||
|
||||
build:
|
||||
$(GO) build $(GOFLAGS) $(LDFLAGS) -o $(BUILD_DIR)/$(BINARY_NAME) ./cmd/exploredns
|
||||
@@ -14,7 +15,10 @@ build:
|
||||
build-server:
|
||||
$(GO) build $(GOFLAGS) $(LDFLAGS) -o $(BUILD_DIR)/$(SERVER_BINARY_NAME) ./cmd/server
|
||||
|
||||
build-all: build build-server
|
||||
build-receiver:
|
||||
$(GO) build $(GOFLAGS) $(LDFLAGS) -o $(BUILD_DIR)/$(RECEIVER_BINARY_NAME) ./cmd/exploredns-receiver
|
||||
|
||||
build-all: build build-server build-receiver
|
||||
|
||||
test:
|
||||
$(GO) test -v -race -coverprofile=coverage.out ./...
|
||||
|
||||
@@ -297,6 +297,7 @@ variables at startup:
|
||||
| `EXPLOREDNS_CORS_ORIGIN` | *(unset)* | Off by default (the SPA is same-origin). Set an origin — or `*` for development — to enable cross-origin API access. |
|
||||
| `EXPLOREDNS_RATE_LIMIT` | `30/1h` | Per-client-IP token-bucket limit on `POST /api/traverse` in `N/duration` form (e.g. `10/10m`); invalid values fall back to the default. Over-limit requests get `429`. Buckets refill continuously. Direct localhost connections are exempt (dev loop, tests), but proxied requests are always limited by the real client IP from `Fly-Client-IP` / `X-Forwarded-For`. |
|
||||
| `EXPLOREDNS_WEBHOOK_URL` | *(unset)* | Off by default. When set, the server POSTs a usage-reporting JSON event to this URL on every traversal start and completion (see below). |
|
||||
| `EXPLOREDNS_WEBHOOK_TOKEN` | *(unset)* | Optional bearer token for webhook deliveries. When set, every webhook POST carries `Authorization: Bearer <token>`; pair it with the receiver's `RECEIVER_INGEST_TOKEN`. |
|
||||
|
||||
### Usage reporting
|
||||
|
||||
@@ -312,12 +313,17 @@ traversal, each with header `X-ExploreDNS-Event` naming the event:
|
||||
`X-Forwarded-For` entry, else the connection address). Delivery is
|
||||
fire-and-forget: a 5-second timeout, one retry after 2 seconds, and failures
|
||||
are logged without ever affecting the traversal or the API response. On
|
||||
Fly.io, configure it as a secret rather than in `fly.toml`:
|
||||
Fly.io, configure it (and the optional bearer token) as secrets rather than
|
||||
in `fly.toml`:
|
||||
|
||||
```sh
|
||||
fly secrets set EXPLOREDNS_WEBHOOK_URL=https://example.com/hook
|
||||
fly secrets set EXPLOREDNS_WEBHOOK_URL=https://example.com/hook \
|
||||
EXPLOREDNS_WEBHOOK_TOKEN=some-long-random-string
|
||||
```
|
||||
|
||||
This repo ships a matching receiver for these events — see
|
||||
[Usage telemetry receiver](#usage-telemetry-receiver).
|
||||
|
||||
---
|
||||
|
||||
## Deploying to Fly.io
|
||||
@@ -366,9 +372,9 @@ flyctl tokens create deploy -x 999999h
|
||||
|
||||
Pushing a `v*` tag triggers the full release pipeline:
|
||||
|
||||
1. `.gitea/workflows/release.yml` (`binaries` job) cross-compiles the CLI
|
||||
and server for linux/amd64, linux/arm64, darwin/amd64, darwin/arm64 and
|
||||
windows/amd64, packages them as
|
||||
1. `.gitea/workflows/release.yml` (`binaries` job) cross-compiles the CLI,
|
||||
server, and receiver for linux/amd64, linux/arm64, darwin/amd64,
|
||||
darwin/arm64 and windows/amd64, packages them as
|
||||
`exploredns_<tag>_<os>_<arch>.tar.gz` (`.zip` on Windows) plus a
|
||||
`SHA256SUMS` file, and attaches everything to the Gitea release for the
|
||||
tag. Create the release with notes by hand before (or after) pushing
|
||||
@@ -376,8 +382,8 @@ Pushing a `v*` tag triggers the full release pipeline:
|
||||
a bare one only when none exists, and skips already-attached assets so
|
||||
re-runs are safe.
|
||||
2. `.gitea/workflows/release.yml` (`docker` job) pushes
|
||||
`gitea.hansenits.com.au/hits/exploredns-cli` and `…/exploredns-web`
|
||||
images tagged `<tag>` and `latest`.
|
||||
`gitea.hansenits.com.au/hits/exploredns-cli`, `…/exploredns-web`, and
|
||||
`…/exploredns-receiver` images tagged `<tag>` and `latest`.
|
||||
3. `.gitea/workflows/deploy.yml` deploys the web server to Fly.io.
|
||||
|
||||
All binaries are stamped with the tag via
|
||||
@@ -420,6 +426,78 @@ with `--show-servers`. Suitable for piping into `jq`.
|
||||
|
||||
---
|
||||
|
||||
## Usage telemetry receiver
|
||||
|
||||
`cmd/exploredns-receiver` is a small companion service that receives the
|
||||
usage webhooks described above (`start`/`complete` events from
|
||||
`EXPLOREDNS_WEBHOOK_URL`), stores them in MySQL or SQLite, and serves a
|
||||
basic-auth-protected admin dashboard (`/admin`) plus JSON API
|
||||
(`/admin/api/traversals`, `/admin/api/stats`) over the collected data. It
|
||||
is a separate binary intended to run wherever you keep long-lived storage
|
||||
(e.g. a home Kubernetes cluster) while the public web server stays
|
||||
stateless.
|
||||
|
||||
Endpoints: `POST /webhook` (ingest, bearer-token protected when configured),
|
||||
`GET /healthz` (liveness/readiness), `GET /admin` and `GET /admin/api/*`
|
||||
(basic auth, always required).
|
||||
|
||||
### Configuration
|
||||
|
||||
| Variable | Default | Meaning |
|
||||
|---|---|---|
|
||||
| `RECEIVER_ADDR` | `:8080` | Listen address. |
|
||||
| `RECEIVER_MYSQL_DSN` | *(unset)* | [go-sql-driver DSN](https://github.com/go-sql-driver/mysql#dsn-data-source-name) (`user:pass@tcp(host:3306)/dbname`). When set, events are stored in MySQL and the SQLite settings are ignored. |
|
||||
| `RECEIVER_SQLITE_PATH` | `data/exploredns-receiver.db` | SQLite database path, used when no MySQL DSN is set (the container image defaults it to `/data/exploredns-receiver.db`). Parent directories are created automatically. |
|
||||
| `RECEIVER_INGEST_TOKEN` | *(unset)* | When set, `POST /webhook` requires `Authorization: Bearer <token>`. Leave unset only on trusted networks. |
|
||||
| `RECEIVER_ADMIN_USER` | `admin` | Basic-auth username for `/admin`. |
|
||||
| `RECEIVER_ADMIN_PASSWORD` | *(required)* | Basic-auth password for `/admin`; the receiver refuses to start without it. |
|
||||
|
||||
Both storage backends share one portable schema; pure-Go drivers
|
||||
(`modernc.org/sqlite`, `github.com/go-sql-driver/mysql`) keep the binary
|
||||
CGO-free. SQLite is the zero-setup default; point `RECEIVER_MYSQL_DSN` at
|
||||
an external MySQL when you want the data outside the pod/VM.
|
||||
|
||||
### Running with Docker
|
||||
|
||||
```sh
|
||||
docker run -d --name exploredns-receiver \
|
||||
-p 8080:8080 \
|
||||
-v exploredns-receiver-data:/data \
|
||||
-e RECEIVER_ADMIN_PASSWORD=change-me \
|
||||
-e RECEIVER_INGEST_TOKEN=some-long-random-string \
|
||||
gitea.hansenits.com.au/hits/exploredns-receiver:latest
|
||||
```
|
||||
|
||||
The image stores SQLite data under the `/data` volume; add
|
||||
`-e RECEIVER_MYSQL_DSN=...` to use MySQL instead.
|
||||
|
||||
### Running on Kubernetes
|
||||
|
||||
[deploy/k8s/receiver/](deploy/k8s/receiver/) contains commented template
|
||||
manifests: a single-replica deployment (SQLite on a 1Gi PVC mounted at
|
||||
`/data`, probes on `/healthz`), ClusterIP service, ingress with TLS
|
||||
placeholders, and a secret template for the `RECEIVER_*` variables. Edit
|
||||
the placeholder host/credentials, then:
|
||||
|
||||
```sh
|
||||
kubectl apply -f deploy/k8s/receiver/
|
||||
```
|
||||
|
||||
Keep one replica while on SQLite; MySQL removes that constraint.
|
||||
|
||||
### Pairing with the web server
|
||||
|
||||
Set the same token on both ends so the receiver only accepts events from
|
||||
your server — e.g. on Fly.io:
|
||||
|
||||
```sh
|
||||
fly secrets set EXPLOREDNS_WEBHOOK_URL=https://receiver.example.com/webhook \
|
||||
EXPLOREDNS_WEBHOOK_TOKEN=some-long-random-string
|
||||
# receiver side: RECEIVER_INGEST_TOKEN=some-long-random-string
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Comparison with dnstraverse
|
||||
|
||||
| Feature | dnstraverse (Ruby) | ExploreDNS (Go) |
|
||||
@@ -440,13 +518,16 @@ with `--show-servers`. Suitable for piping into `jq`.
|
||||
```
|
||||
cmd/exploredns/ CLI entry point and flag parsing
|
||||
cmd/server/ HTTP API server entry point
|
||||
cmd/exploredns-receiver/ Usage telemetry receiver entry point
|
||||
internal/config/ Configuration types, validation, and usage text
|
||||
internal/dns/ DNS query layer, root discovery, transport
|
||||
internal/traverse/ Core traversal engine, referral resolution, caching
|
||||
internal/fingerprint/ DNS server version fingerprinting (version.bind CHAOS)
|
||||
internal/output/ Result formatting — text tree and JSON renderers
|
||||
internal/integration/ End-to-end integration tests
|
||||
internal/receiver/ Telemetry receiver: HTTP server, admin UI, event store
|
||||
web/api/ HTTP handler, job store, SSE streaming, static assets
|
||||
deploy/k8s/receiver/ Kubernetes manifest templates for the receiver
|
||||
```
|
||||
|
||||
---
|
||||
@@ -456,7 +537,8 @@ web/api/ HTTP handler, job store, SSE streaming, static assets
|
||||
```sh
|
||||
make build # compile CLI binary to bin/exploredns
|
||||
make build-server # compile server binary to bin/exploredns-server
|
||||
make build-all # compile both binaries
|
||||
make build-receiver # compile telemetry receiver to bin/exploredns-receiver
|
||||
make build-all # compile all three binaries
|
||||
make test # run all unit and integration tests
|
||||
make lint # run go vet
|
||||
make clean # remove build artefacts
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
// Command exploredns-receiver stores usage webhooks posted by the
|
||||
// ExploreDNS API server in MySQL or SQLite.
|
||||
//
|
||||
// Configuration (environment):
|
||||
//
|
||||
// RECEIVER_ADDR listen address (default ":8080")
|
||||
// RECEIVER_MYSQL_DSN go-sql-driver DSN; when set, events go to MySQL
|
||||
// RECEIVER_SQLITE_PATH SQLite fallback path (default "data/exploredns-receiver.db")
|
||||
// RECEIVER_INGEST_TOKEN bearer token required on POST /webhook (open when unset)
|
||||
// RECEIVER_ADMIN_USER basic-auth username for /admin (default "admin")
|
||||
// RECEIVER_ADMIN_PASSWORD basic-auth password for /admin (required)
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"gitea.hansenits.com.au/hits/ExploreDNS/internal/receiver/server"
|
||||
"gitea.hansenits.com.au/hits/ExploreDNS/internal/receiver/store"
|
||||
)
|
||||
|
||||
// version is stamped at build time via -ldflags "-X main.version=...".
|
||||
var version = "dev"
|
||||
|
||||
func main() {
|
||||
addr := envOr("RECEIVER_ADDR", ":8080")
|
||||
|
||||
adminPass := os.Getenv("RECEIVER_ADMIN_PASSWORD")
|
||||
if adminPass == "" {
|
||||
fmt.Fprintln(os.Stderr, "Error: RECEIVER_ADMIN_PASSWORD is not set; refusing to start with an unprotected admin interface")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
var (
|
||||
st *store.Store
|
||||
backend string
|
||||
err error
|
||||
)
|
||||
if dsn := os.Getenv("RECEIVER_MYSQL_DSN"); dsn != "" {
|
||||
st, err = store.OpenMySQL(dsn)
|
||||
backend = "mysql " + store.RedactMySQLDSN(dsn)
|
||||
} else {
|
||||
path := envOr("RECEIVER_SQLITE_PATH", "data/exploredns-receiver.db")
|
||||
st, err = store.OpenSQLite(path)
|
||||
backend = "sqlite " + path
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
srv := server.New(addr, st)
|
||||
srv.SetVersion(version)
|
||||
srv.SetIngestToken(os.Getenv("RECEIVER_INGEST_TOKEN"))
|
||||
srv.SetAdminAuth(envOr("RECEIVER_ADMIN_USER", "admin"), adminPass)
|
||||
if err := srv.Start(); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
log.Printf("ExploreDNS receiver %s listening on %s, storing to %s", version, srv.Addr(), backend)
|
||||
|
||||
quit := make(chan os.Signal, 1)
|
||||
signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM)
|
||||
<-quit
|
||||
|
||||
log.Println("Shutting down...")
|
||||
if err := srv.Shutdown(15 * time.Second); err != nil {
|
||||
log.Printf("Shutdown error: %v", err)
|
||||
}
|
||||
if err := st.Close(); err != nil {
|
||||
log.Printf("Close store error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// envOr returns the environment variable name, or def when unset or empty.
|
||||
func envOr(name, def string) string {
|
||||
if v := os.Getenv(name); v != "" {
|
||||
return v
|
||||
}
|
||||
return def
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
# TEMPLATE — single-replica receiver deployment. Keep replicas at 1 while
|
||||
# using the SQLite backend: the database file on the RWO volume supports only
|
||||
# one writer. With RECEIVER_MYSQL_DSN you may scale out and drop the volume.
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: exploredns-receiver
|
||||
namespace: exploredns-receiver
|
||||
labels:
|
||||
app: exploredns-receiver
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: Recreate # RWO volume: never run old and new pods concurrently
|
||||
selector:
|
||||
matchLabels:
|
||||
app: exploredns-receiver
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: exploredns-receiver
|
||||
spec:
|
||||
containers:
|
||||
- name: receiver
|
||||
image: gitea.hansenits.com.au/hits/exploredns-receiver:latest
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 8080
|
||||
envFrom:
|
||||
- secretRef:
|
||||
name: exploredns-receiver
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /data
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /healthz
|
||||
port: http
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 15
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /healthz
|
||||
port: http
|
||||
initialDelaySeconds: 2
|
||||
periodSeconds: 10
|
||||
resources:
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 64Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 256Mi
|
||||
volumes:
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: exploredns-receiver-data
|
||||
@@ -0,0 +1,33 @@
|
||||
# TEMPLATE — replace receiver.example.com with your real host and wire up
|
||||
# TLS for your cluster (the webhook bearer token and admin password travel
|
||||
# in headers, so plain HTTP is not acceptable across the internet).
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: exploredns-receiver
|
||||
namespace: exploredns-receiver
|
||||
annotations: {}
|
||||
# nginx ingress controller:
|
||||
# cert-manager.io/cluster-issuer: letsencrypt
|
||||
# nginx.ingress.kubernetes.io/proxy-body-size: 1m
|
||||
#
|
||||
# traefik:
|
||||
# traefik.ingress.kubernetes.io/router.entrypoints: websecure
|
||||
# traefik.ingress.kubernetes.io/router.tls: "true"
|
||||
spec:
|
||||
# ingressClassName: nginx
|
||||
tls:
|
||||
- hosts:
|
||||
- receiver.example.com
|
||||
secretName: exploredns-receiver-tls # created by cert-manager or by hand
|
||||
rules:
|
||||
- host: receiver.example.com
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: exploredns-receiver
|
||||
port:
|
||||
name: http
|
||||
@@ -0,0 +1,6 @@
|
||||
# TEMPLATE — optional. Skip this file (and drop the namespace fields from the
|
||||
# other manifests) to deploy into an existing namespace.
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: exploredns-receiver
|
||||
@@ -0,0 +1,16 @@
|
||||
# TEMPLATE — backing storage for the SQLite database (RECEIVER_SQLITE_PATH
|
||||
# defaults to /data/exploredns-receiver.db in the container image). Not needed
|
||||
# when RECEIVER_MYSQL_DSN is set, but harmless to keep. Set storageClassName
|
||||
# if your cluster has no default class.
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: exploredns-receiver-data
|
||||
namespace: exploredns-receiver
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 1Gi
|
||||
# storageClassName: standard
|
||||
@@ -0,0 +1,25 @@
|
||||
# TEMPLATE — fill in real values before applying, or create the secret
|
||||
# imperatively instead and never commit credentials:
|
||||
#
|
||||
# kubectl -n exploredns-receiver create secret generic exploredns-receiver \
|
||||
# --from-literal=RECEIVER_ADMIN_USER=admin \
|
||||
# --from-literal=RECEIVER_ADMIN_PASSWORD='change-me' \
|
||||
# --from-literal=RECEIVER_INGEST_TOKEN='change-me-too'
|
||||
#
|
||||
# The deployment loads every key here as an environment variable (envFrom).
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: exploredns-receiver
|
||||
namespace: exploredns-receiver
|
||||
type: Opaque
|
||||
stringData:
|
||||
RECEIVER_ADMIN_USER: admin
|
||||
RECEIVER_ADMIN_PASSWORD: change-me
|
||||
# Bearer token the main app must send on POST /webhook. Must match the
|
||||
# sender's EXPLOREDNS_WEBHOOK_TOKEN. Leave unset to accept unauthenticated
|
||||
# posts (not recommended for an internet-facing receiver).
|
||||
RECEIVER_INGEST_TOKEN: change-me-too
|
||||
# Uncomment to store events in an external MySQL instead of the SQLite
|
||||
# file on the PVC (go-sql-driver DSN).
|
||||
# RECEIVER_MYSQL_DSN: "user:pass@tcp(mysql.example.com:3306)/exploredns"
|
||||
@@ -0,0 +1,16 @@
|
||||
# TEMPLATE — cluster-internal service in front of the receiver pod.
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: exploredns-receiver
|
||||
namespace: exploredns-receiver
|
||||
labels:
|
||||
app: exploredns-receiver
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app: exploredns-receiver
|
||||
ports:
|
||||
- name: http
|
||||
port: 8080
|
||||
targetPort: http
|
||||
@@ -1,16 +1,27 @@
|
||||
module gitea.hansenits.com.au/hits/ExploreDNS
|
||||
|
||||
go 1.24.0
|
||||
go 1.25.0
|
||||
|
||||
require (
|
||||
github.com/go-sql-driver/mysql v1.10.0
|
||||
github.com/miekg/dns v1.1.72
|
||||
golang.org/x/net v0.48.0
|
||||
golang.org/x/net v0.54.0
|
||||
modernc.org/sqlite v1.53.0
|
||||
)
|
||||
|
||||
require (
|
||||
golang.org/x/mod v0.31.0 // indirect
|
||||
golang.org/x/sync v0.19.0 // indirect
|
||||
golang.org/x/sys v0.39.0 // indirect
|
||||
golang.org/x/text v0.32.0 // indirect
|
||||
golang.org/x/tools v0.40.0 // indirect
|
||||
filippo.io/edwards25519 v1.2.0 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/ncruces/go-strftime v1.0.0 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
golang.org/x/mod v0.36.0 // indirect
|
||||
golang.org/x/sync v0.20.0 // indirect
|
||||
golang.org/x/sys v0.44.0 // indirect
|
||||
golang.org/x/text v0.37.0 // indirect
|
||||
golang.org/x/tools v0.45.0 // indirect
|
||||
modernc.org/libc v1.73.4 // indirect
|
||||
modernc.org/mathutil v1.7.1 // indirect
|
||||
modernc.org/memory v1.11.0 // indirect
|
||||
)
|
||||
|
||||
@@ -1,16 +1,63 @@
|
||||
filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo=
|
||||
filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/go-sql-driver/mysql v1.10.0 h1:Q+1LV8DkHJvSYAdR83XzuhDaTykuDx0l6fkXxoWCWfw=
|
||||
github.com/go-sql-driver/mysql v1.10.0/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk=
|
||||
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
|
||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs=
|
||||
github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/miekg/dns v1.1.72 h1:vhmr+TF2A3tuoGNkLDFK9zi36F2LS+hKTRW0Uf8kbzI=
|
||||
github.com/miekg/dns v1.1.72/go.mod h1:+EuEPhdHOsfk6Wk5TT2CzssZdqkmFhf8r+aVyDEToIs=
|
||||
golang.org/x/mod v0.31.0 h1:HaW9xtz0+kOcWKwli0ZXy79Ix+UW/vOfmWI5QVd2tgI=
|
||||
golang.org/x/mod v0.31.0/go.mod h1:43JraMp9cGx1Rx3AqioxrbrhNsLl2l/iNAvuBkrezpg=
|
||||
golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU=
|
||||
golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY=
|
||||
golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
|
||||
golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||
golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk=
|
||||
golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU=
|
||||
golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY=
|
||||
golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA=
|
||||
golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc=
|
||||
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
|
||||
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4=
|
||||
golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ=
|
||||
golang.org/x/net v0.54.0 h1:2zJIZAxAHV/OHCDTCOHAYehQzLfSXuf/5SoL/Dv6w/w=
|
||||
golang.org/x/net v0.54.0/go.mod h1:Sj4oj8jK6XmHpBZU/zWHw3BV3abl4Kvi+Ut7cQcY+cQ=
|
||||
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
|
||||
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ=
|
||||
golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc=
|
||||
golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
|
||||
golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8=
|
||||
golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0=
|
||||
modernc.org/cc/v4 v4.28.4 h1:Hd/4Es+MBj+/7hSdZaisNyu6bv3V0Dp2MdllyfqaH+c=
|
||||
modernc.org/cc/v4 v4.28.4/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
|
||||
modernc.org/ccgo/v4 v4.34.4 h1:OVnSOWQjVKOYkFxoHYB+qQmSHK5gqMqARM+K9DpR/Ws=
|
||||
modernc.org/ccgo/v4 v4.34.4/go.mod h1:qdKqE8FNIYyysougB1RX9MxCzp5oJOcQXSobANJ4TuE=
|
||||
modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM=
|
||||
modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU=
|
||||
modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
|
||||
modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
|
||||
modernc.org/gc/v3 v3.1.3 h1:6QAplYyVO+KdPW3pGnqmJDUxtkec8ooEWvks/hhU3lc=
|
||||
modernc.org/gc/v3 v3.1.3/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
|
||||
modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks=
|
||||
modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI=
|
||||
modernc.org/libc v1.73.4 h1:+ra4Ui8ngyt8HDcO1FTDPWlkAh6yOdaO2yAoh8MddQA=
|
||||
modernc.org/libc v1.73.4/go.mod h1:DXZ3eO8qMCNn2SnmTNCiC71nJ9Rcq3PsnpU6Vc4rWK8=
|
||||
modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
|
||||
modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
|
||||
modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI=
|
||||
modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
|
||||
modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg=
|
||||
modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
|
||||
modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
|
||||
modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
|
||||
modernc.org/sqlite v1.53.0 h1:20WG8N9q4ji/dEqGk4uiI0c6OPjSeLTNYGFCc3+7c1M=
|
||||
modernc.org/sqlite v1.53.0/go.mod h1:xoEpOIpGrgT48H5iiyt/YXPCZPEzlfmfFwtk8Lklw8s=
|
||||
modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
|
||||
modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
|
||||
modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
|
||||
modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
|
||||
|
||||
@@ -0,0 +1,280 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"crypto/subtle"
|
||||
_ "embed"
|
||||
"encoding/json"
|
||||
"log"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"gitea.hansenits.com.au/hits/ExploreDNS/internal/receiver/store"
|
||||
)
|
||||
|
||||
//go:embed static/admin.html
|
||||
var adminHTML []byte
|
||||
|
||||
const (
|
||||
maxListLimit = 200
|
||||
defaultStatsDays = 30
|
||||
maxStatsDays = 365
|
||||
topLimit = 10
|
||||
)
|
||||
|
||||
// requireAdmin gates next behind HTTP basic auth against the configured
|
||||
// admin credentials. An empty configured password locks the subtree
|
||||
// entirely rather than opening it.
|
||||
func (h *handler) requireAdmin(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
user, pass, ok := r.BasicAuth()
|
||||
// Evaluate both comparisons unconditionally to keep timing uniform.
|
||||
userOK := secretEqual(user, h.adminUser)
|
||||
passOK := secretEqual(pass, h.adminPass)
|
||||
if !ok || !userOK || !passOK || h.adminPass == "" {
|
||||
w.Header().Set("WWW-Authenticate", `Basic realm="ExploreDNS receiver admin", charset="UTF-8"`)
|
||||
writeError(w, http.StatusUnauthorized, "authentication required")
|
||||
return
|
||||
}
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
// secretEqual compares two strings in constant time; both sides are hashed
|
||||
// first so length differences do not leak.
|
||||
func secretEqual(got, want string) bool {
|
||||
g := sha256.Sum256([]byte(got))
|
||||
w := sha256.Sum256([]byte(want))
|
||||
return subtle.ConstantTimeCompare(g[:], w[:]) == 1
|
||||
}
|
||||
|
||||
// adminPage handles GET /admin, serving the embedded admin UI.
|
||||
func (h *handler) adminPage(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
w.Write(adminHTML) //nolint:errcheck
|
||||
}
|
||||
|
||||
// traversalItem is the admin API JSON shape of one stored traversal.
|
||||
// Completion fields are omitted while a traversal is still running.
|
||||
type traversalItem struct {
|
||||
ID string `json:"id"`
|
||||
Domain string `json:"domain"`
|
||||
QueryType string `json:"query_type"`
|
||||
AllRoots bool `json:"all_roots"`
|
||||
ClientIP string `json:"client_ip"`
|
||||
StartedAt string `json:"started_at,omitempty"`
|
||||
DoneAt string `json:"done_at,omitempty"`
|
||||
DurationMS *int64 `json:"duration_ms,omitempty"`
|
||||
Status string `json:"status"`
|
||||
Error string `json:"error,omitempty"`
|
||||
ResultCount *int `json:"result_count,omitempty"`
|
||||
Summary json.RawMessage `json:"summary,omitempty"`
|
||||
ReceivedAt string `json:"received_at"`
|
||||
}
|
||||
|
||||
func toItem(tr store.Traversal) traversalItem {
|
||||
it := traversalItem{
|
||||
ID: tr.ID, Domain: tr.Domain, QueryType: tr.QueryType, AllRoots: tr.AllRoots,
|
||||
ClientIP: tr.ClientIP, StartedAt: rfc3339(tr.StartedAt), Status: tr.Status,
|
||||
Error: tr.Error, DurationMS: tr.DurationMS, ResultCount: tr.ResultCount,
|
||||
ReceivedAt: rfc3339(tr.FirstSeen),
|
||||
}
|
||||
if tr.DoneAt != nil {
|
||||
it.DoneAt = rfc3339(*tr.DoneAt)
|
||||
}
|
||||
if tr.Summary != "" {
|
||||
it.Summary = json.RawMessage(tr.Summary)
|
||||
}
|
||||
return it
|
||||
}
|
||||
|
||||
// rfc3339 renders t as UTC RFC 3339, or "" for the zero time.
|
||||
func rfc3339(t time.Time) string {
|
||||
if t.IsZero() {
|
||||
return ""
|
||||
}
|
||||
return t.UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
// adminTraversals handles GET /admin/api/traversals.
|
||||
func (h *handler) adminTraversals(w http.ResponseWriter, r *http.Request) {
|
||||
q := r.URL.Query()
|
||||
|
||||
limit := 50
|
||||
if v := q.Get("limit"); v != "" {
|
||||
n, err := strconv.Atoi(v)
|
||||
if err != nil || n < 1 {
|
||||
writeError(w, http.StatusBadRequest, "invalid limit "+strconv.Quote(v))
|
||||
return
|
||||
}
|
||||
limit = min(n, maxListLimit)
|
||||
}
|
||||
offset := 0
|
||||
if v := q.Get("offset"); v != "" {
|
||||
n, err := strconv.Atoi(v)
|
||||
if err != nil || n < 0 {
|
||||
writeError(w, http.StatusBadRequest, "invalid offset "+strconv.Quote(v))
|
||||
return
|
||||
}
|
||||
offset = n
|
||||
}
|
||||
f := store.ListFilter{Domain: q.Get("domain"), Status: q.Get("status")}
|
||||
if v := q.Get("from"); v != "" {
|
||||
t, err := time.Parse(time.RFC3339, v)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid from: "+err.Error())
|
||||
return
|
||||
}
|
||||
f.From = t
|
||||
}
|
||||
if v := q.Get("to"); v != "" {
|
||||
t, err := time.Parse(time.RFC3339, v)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid to: "+err.Error())
|
||||
return
|
||||
}
|
||||
f.To = t
|
||||
}
|
||||
|
||||
rows, total, err := h.st.ListTraversals(r.Context(), f, limit, offset)
|
||||
if err != nil {
|
||||
log.Printf("receiver admin: list traversals: %v", err)
|
||||
writeError(w, http.StatusInternalServerError, "list traversals failed")
|
||||
return
|
||||
}
|
||||
items := make([]traversalItem, 0, len(rows))
|
||||
for _, tr := range rows {
|
||||
items = append(items, toItem(tr))
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"total": total, "items": items})
|
||||
}
|
||||
|
||||
type statsResponse struct {
|
||||
Days int `json:"days"`
|
||||
Totals statsTotals `json:"totals"`
|
||||
PerDay []dayJSON `json:"per_day"`
|
||||
TopDomains []nameCountJSON `json:"top_domains"`
|
||||
QueryTypes []nameCountJSON `json:"query_types"`
|
||||
Statuses []nameCountJSON `json:"statuses"`
|
||||
Durations durationsJSON `json:"durations"`
|
||||
TopClients []nameCountJSON `json:"top_clients"`
|
||||
}
|
||||
|
||||
type statsTotals struct {
|
||||
AllTime int `json:"all_time"`
|
||||
Last24h int `json:"last_24h"`
|
||||
Last7d int `json:"last_7d"`
|
||||
DistinctDomains int `json:"distinct_domains"`
|
||||
DistinctClients int `json:"distinct_clients"`
|
||||
ErrorRate float64 `json:"error_rate"`
|
||||
}
|
||||
|
||||
type dayJSON struct {
|
||||
Day string `json:"day"`
|
||||
Total int `json:"total"`
|
||||
Errors int `json:"errors"`
|
||||
}
|
||||
|
||||
type nameCountJSON struct {
|
||||
Name string `json:"name"`
|
||||
Count int `json:"count"`
|
||||
}
|
||||
|
||||
type durationsJSON struct {
|
||||
Count int `json:"count"`
|
||||
AvgMS float64 `json:"avg_ms"`
|
||||
P50MS int64 `json:"p50_ms"`
|
||||
P95MS int64 `json:"p95_ms"`
|
||||
}
|
||||
|
||||
func nameCounts(rows []store.NameCount) []nameCountJSON {
|
||||
out := make([]nameCountJSON, 0, len(rows))
|
||||
for _, r := range rows {
|
||||
out = append(out, nameCountJSON{Name: r.Name, Count: r.Count})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// adminStats handles GET /admin/api/stats. The totals block uses fixed
|
||||
// windows; everything else covers the last ?days calendar days (UTC),
|
||||
// matching the StatsPerDay window.
|
||||
func (h *handler) adminStats(w http.ResponseWriter, r *http.Request) {
|
||||
days := defaultStatsDays
|
||||
if v := r.URL.Query().Get("days"); v != "" {
|
||||
n, err := strconv.Atoi(v)
|
||||
if err != nil || n < 1 {
|
||||
writeError(w, http.StatusBadRequest, "invalid days "+strconv.Quote(v))
|
||||
return
|
||||
}
|
||||
days = min(n, maxStatsDays)
|
||||
}
|
||||
|
||||
ctx := r.Context()
|
||||
now := time.Now().UTC()
|
||||
since := now.Truncate(24*time.Hour).AddDate(0, 0, -(days - 1))
|
||||
|
||||
fail := func(what string, err error) {
|
||||
log.Printf("receiver admin: %s: %v", what, err)
|
||||
writeError(w, http.StatusInternalServerError, what+" failed")
|
||||
}
|
||||
totals, err := h.st.Totals(ctx, now)
|
||||
if err != nil {
|
||||
fail("totals", err)
|
||||
return
|
||||
}
|
||||
perDay, err := h.st.StatsPerDay(ctx, days)
|
||||
if err != nil {
|
||||
fail("per-day stats", err)
|
||||
return
|
||||
}
|
||||
topDomains, err := h.st.TopDomains(ctx, since, topLimit)
|
||||
if err != nil {
|
||||
fail("top domains", err)
|
||||
return
|
||||
}
|
||||
queryTypes, err := h.st.QueryTypeCounts(ctx, since)
|
||||
if err != nil {
|
||||
fail("query types", err)
|
||||
return
|
||||
}
|
||||
statuses, err := h.st.StatusCounts(ctx, since)
|
||||
if err != nil {
|
||||
fail("statuses", err)
|
||||
return
|
||||
}
|
||||
dur, err := h.st.Durations(ctx, since)
|
||||
if err != nil {
|
||||
fail("durations", err)
|
||||
return
|
||||
}
|
||||
topClients, err := h.st.TopClientIPs(ctx, since, topLimit)
|
||||
if err != nil {
|
||||
fail("top clients", err)
|
||||
return
|
||||
}
|
||||
|
||||
resp := statsResponse{
|
||||
Days: days,
|
||||
Totals: statsTotals{
|
||||
AllTime: totals.AllTime,
|
||||
Last24h: totals.Last24h,
|
||||
Last7d: totals.Last7d,
|
||||
DistinctDomains: totals.DistinctDomains,
|
||||
DistinctClients: totals.DistinctClients,
|
||||
},
|
||||
PerDay: make([]dayJSON, 0, len(perDay)),
|
||||
TopDomains: nameCounts(topDomains),
|
||||
QueryTypes: nameCounts(queryTypes),
|
||||
Statuses: nameCounts(statuses),
|
||||
Durations: durationsJSON{Count: dur.Count, AvgMS: dur.AvgMS, P50MS: dur.P50MS, P95MS: dur.P95MS},
|
||||
TopClients: nameCounts(topClients),
|
||||
}
|
||||
if totals.AllTime > 0 {
|
||||
resp.Totals.ErrorRate = float64(totals.Errors) / float64(totals.AllTime)
|
||||
}
|
||||
for _, d := range perDay {
|
||||
resp.PerDay = append(resp.PerDay, dayJSON{Day: d.Day, Total: d.Total, Errors: d.Errors})
|
||||
}
|
||||
writeJSON(w, http.StatusOK, resp)
|
||||
}
|
||||
@@ -0,0 +1,482 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.hansenits.com.au/hits/ExploreDNS/internal/receiver/store"
|
||||
)
|
||||
|
||||
const (
|
||||
testAdminUser = "admin"
|
||||
testAdminPass = "swordfish"
|
||||
)
|
||||
|
||||
// get performs a GET with optional basic-auth credentials.
|
||||
func get(h http.Handler, path, user, pass string) *httptest.ResponseRecorder {
|
||||
req := httptest.NewRequest(http.MethodGet, path, nil)
|
||||
if user != "" || pass != "" {
|
||||
req.SetBasicAuth(user, pass)
|
||||
}
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, req)
|
||||
return w
|
||||
}
|
||||
|
||||
func TestStartRequiresAdminPassword(t *testing.T) {
|
||||
st, err := store.OpenSQLite(filepath.Join(t.TempDir(), "receiver.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("OpenSQLite: %v", err)
|
||||
}
|
||||
defer st.Close()
|
||||
|
||||
srv := New("127.0.0.1:0", st)
|
||||
err = srv.Start()
|
||||
if err == nil {
|
||||
srv.Shutdown(time.Second) //nolint:errcheck
|
||||
t.Fatal("Start succeeded without an admin password")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "RECEIVER_ADMIN_PASSWORD") {
|
||||
t.Errorf("error = %q, want mention of RECEIVER_ADMIN_PASSWORD", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdminAuthMatrix(t *testing.T) {
|
||||
paths := []string{"/admin", "/admin/api/traversals", "/admin/api/stats"}
|
||||
creds := []struct {
|
||||
name string
|
||||
user, pass string
|
||||
want int
|
||||
}{
|
||||
{"no credentials", "", "", http.StatusUnauthorized},
|
||||
{"wrong user", "root", testAdminPass, http.StatusUnauthorized},
|
||||
{"wrong password", testAdminUser, "nope", http.StatusUnauthorized},
|
||||
{"correct credentials", testAdminUser, testAdminPass, http.StatusOK},
|
||||
}
|
||||
h, _ := newTestHandler(t, "")
|
||||
for _, path := range paths {
|
||||
for _, tc := range creds {
|
||||
t.Run(path+" "+tc.name, func(t *testing.T) {
|
||||
w := get(h, path, tc.user, tc.pass)
|
||||
if w.Code != tc.want {
|
||||
t.Fatalf("status = %d, want %d (%s)", w.Code, tc.want, w.Body)
|
||||
}
|
||||
if tc.want == http.StatusUnauthorized {
|
||||
if got := w.Header().Get("WWW-Authenticate"); !strings.Contains(got, `Basic realm=`) {
|
||||
t.Errorf("WWW-Authenticate = %q, want Basic realm", got)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestAdminSubtreeAuthenticated pins that unknown and unauthenticated paths
|
||||
// under /admin still 401 rather than falling through to a handler.
|
||||
func TestAdminSubtreeAuthenticated(t *testing.T) {
|
||||
h, _ := newTestHandler(t, "")
|
||||
for _, path := range []string{"/admin/", "/admin/secret", "/admin/api/other"} {
|
||||
if got := get(h, path, "", "").Code; got != http.StatusUnauthorized {
|
||||
t.Errorf("GET %s without creds = %d, want 401", path, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestAdminLockedWithoutPassword pins that a handler built without an admin
|
||||
// password rejects everything, even blank credentials.
|
||||
func TestAdminLockedWithoutPassword(t *testing.T) {
|
||||
st, err := store.OpenSQLite(filepath.Join(t.TempDir(), "receiver.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("OpenSQLite: %v", err)
|
||||
}
|
||||
defer st.Close()
|
||||
h := newHandler(st, "test-version", "", "admin", "")
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/admin", nil)
|
||||
req.SetBasicAuth("admin", "")
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, req)
|
||||
if w.Code != http.StatusUnauthorized {
|
||||
t.Errorf("status = %d, want 401", w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// TestWebhookAndHealthzStayOpen asserts the sender and k8s probe endpoints
|
||||
// need no admin credentials.
|
||||
func TestWebhookAndHealthzStayOpen(t *testing.T) {
|
||||
h, _ := newTestHandler(t, "")
|
||||
if got := post(h, startJSON, nil).Code; got != http.StatusNoContent {
|
||||
t.Errorf("POST /webhook without creds = %d, want 204", got)
|
||||
}
|
||||
if got := get(h, "/healthz", "", "").Code; got != http.StatusOK {
|
||||
t.Errorf("GET /healthz without creds = %d, want 200", got)
|
||||
}
|
||||
}
|
||||
|
||||
// seedList inserts the deterministic fixed-date dataset used by the
|
||||
// traversals endpoint tests (same shape as the store package's seed).
|
||||
func seedList(t *testing.T, st *store.Store) {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
base := time.Date(2026, 7, 1, 0, 0, 0, 0, time.UTC)
|
||||
rows := []struct {
|
||||
id, domain, qtype, ip, status string
|
||||
day int
|
||||
durMS int64
|
||||
}{
|
||||
{"a1", "example.com", "A", "203.0.113.1", store.StatusComplete, 0, 100},
|
||||
{"a2", "example.com", "AAAA", "203.0.113.1", store.StatusComplete, 0, 200},
|
||||
{"a3", "sub.example.com", "A", "203.0.113.2", store.StatusError, 1, 300},
|
||||
{"a4", "other.net", "MX", "203.0.113.3", store.StatusComplete, 1, 400},
|
||||
{"a5", "other.net", "A", "203.0.113.1", store.StatusComplete, 2, 500},
|
||||
{"a6", "under_score.org", "A", "203.0.113.4", store.StatusComplete, 2, 600},
|
||||
}
|
||||
for i, r := range rows {
|
||||
started := base.AddDate(0, 0, r.day).Add(time.Duration(i) * time.Minute)
|
||||
ev := store.CompleteEvent{
|
||||
ID: r.id, Domain: r.domain, QueryType: r.qtype, ClientIP: r.ip,
|
||||
StartedAt: started, DoneAt: started.Add(time.Duration(r.durMS) * time.Millisecond),
|
||||
DurationMS: r.durMS, Status: r.status, ResultCount: 1,
|
||||
Summary: json.RawMessage(`{"answers":[{"probability":1}]}`),
|
||||
}
|
||||
if r.status == store.StatusError {
|
||||
ev.Error = "lookup failed"
|
||||
}
|
||||
if err := st.RecordComplete(ctx, ev); err != nil {
|
||||
t.Fatalf("seed %s: %v", r.id, err)
|
||||
}
|
||||
}
|
||||
if err := st.RecordStart(ctx, store.StartEvent{
|
||||
ID: "a7", Domain: "running.io", QueryType: "A", ClientIP: "203.0.113.5",
|
||||
StartedAt: base.AddDate(0, 0, 2).Add(time.Hour),
|
||||
}); err != nil {
|
||||
t.Fatalf("seed a7: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
type listResponse struct {
|
||||
Total int `json:"total"`
|
||||
Items []struct {
|
||||
ID string `json:"id"`
|
||||
Domain string `json:"domain"`
|
||||
QueryType string `json:"query_type"`
|
||||
ClientIP string `json:"client_ip"`
|
||||
StartedAt string `json:"started_at"`
|
||||
DoneAt string `json:"done_at"`
|
||||
DurationMS *int64 `json:"duration_ms"`
|
||||
Status string `json:"status"`
|
||||
Error string `json:"error"`
|
||||
ResultCount *int `json:"result_count"`
|
||||
Summary json.RawMessage `json:"summary"`
|
||||
ReceivedAt string `json:"received_at"`
|
||||
} `json:"items"`
|
||||
}
|
||||
|
||||
func listGET(t *testing.T, h http.Handler, query string) listResponse {
|
||||
t.Helper()
|
||||
w := get(h, "/admin/api/traversals"+query, testAdminUser, testAdminPass)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("GET %s = %d (%s)", query, w.Code, w.Body)
|
||||
}
|
||||
var resp listResponse
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("decode %s: %v", query, err)
|
||||
}
|
||||
return resp
|
||||
}
|
||||
|
||||
func TestAdminTraversalsEndpoint(t *testing.T) {
|
||||
h, st := newTestHandler(t, "")
|
||||
seedList(t, st)
|
||||
|
||||
// Default page: everything, newest first.
|
||||
resp := listGET(t, h, "")
|
||||
if resp.Total != 7 || len(resp.Items) != 7 {
|
||||
t.Fatalf("total = %d, len = %d, want 7 and 7", resp.Total, len(resp.Items))
|
||||
}
|
||||
first := resp.Items[0]
|
||||
if first.ID != "a7" || first.Status != store.StatusRunning {
|
||||
t.Errorf("first item = %s/%s, want a7/running", first.ID, first.Status)
|
||||
}
|
||||
if first.DurationMS != nil || first.DoneAt != "" || len(first.Summary) != 0 {
|
||||
t.Errorf("running item has completion fields: %+v", first)
|
||||
}
|
||||
if first.StartedAt != "2026-07-03T01:00:00Z" {
|
||||
t.Errorf("started_at = %q, want 2026-07-03T01:00:00Z", first.StartedAt)
|
||||
}
|
||||
if first.ReceivedAt == "" {
|
||||
t.Error("received_at empty")
|
||||
}
|
||||
|
||||
// Completed rows carry duration, result count, and raw summary JSON.
|
||||
last := resp.Items[6]
|
||||
if last.ID != "a1" || last.DurationMS == nil || *last.DurationMS != 100 ||
|
||||
last.ResultCount == nil || *last.ResultCount != 1 {
|
||||
t.Errorf("oldest item = %+v, want a1 with duration 100 and 1 result", last)
|
||||
}
|
||||
if string(last.Summary) != `{"answers":[{"probability":1}]}` {
|
||||
t.Errorf("summary = %s", last.Summary)
|
||||
}
|
||||
|
||||
// Pagination.
|
||||
resp = listGET(t, h, "?limit=3&offset=3")
|
||||
if resp.Total != 7 || len(resp.Items) != 3 {
|
||||
t.Fatalf("page 2 total = %d, len = %d, want 7 and 3", resp.Total, len(resp.Items))
|
||||
}
|
||||
if resp.Items[0].ID != "a4" || resp.Items[1].ID != "a3" || resp.Items[2].ID != "a2" {
|
||||
t.Errorf("page 2 order = %s,%s,%s, want a4,a3,a2",
|
||||
resp.Items[0].ID, resp.Items[1].ID, resp.Items[2].ID)
|
||||
}
|
||||
|
||||
// Filters.
|
||||
if resp := listGET(t, h, "?domain=example"); resp.Total != 3 {
|
||||
t.Errorf("domain filter total = %d, want 3", resp.Total)
|
||||
}
|
||||
resp = listGET(t, h, "?status=error")
|
||||
if resp.Total != 1 || resp.Items[0].ID != "a3" || resp.Items[0].Error != "lookup failed" {
|
||||
t.Errorf("status filter = %+v, want a3 with error", resp)
|
||||
}
|
||||
resp = listGET(t, h, "?from=2026-07-02T00:00:00Z&to=2026-07-02T23:59:59Z")
|
||||
if resp.Total != 2 {
|
||||
t.Errorf("time window total = %d, want 2", resp.Total)
|
||||
}
|
||||
|
||||
// Limit above the cap succeeds and is clamped rather than rejected.
|
||||
if resp := listGET(t, h, "?limit=1000"); resp.Total != 7 {
|
||||
t.Errorf("clamped limit total = %d, want 7", resp.Total)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdminTraversalsBadParams(t *testing.T) {
|
||||
h, _ := newTestHandler(t, "")
|
||||
for _, q := range []string{"?limit=abc", "?limit=0", "?offset=-1", "?from=notatime", "?to=2026-13-99"} {
|
||||
if got := get(h, "/admin/api/traversals"+q, testAdminUser, testAdminPass).Code; got != http.StatusBadRequest {
|
||||
t.Errorf("GET %s = %d, want 400", q, got)
|
||||
}
|
||||
}
|
||||
if got := get(h, "/admin/api/stats?days=x", testAdminUser, testAdminPass).Code; got != http.StatusBadRequest {
|
||||
t.Errorf("GET stats?days=x = %d, want 400", got)
|
||||
}
|
||||
}
|
||||
|
||||
type statsBody struct {
|
||||
Days int `json:"days"`
|
||||
Totals struct {
|
||||
AllTime int `json:"all_time"`
|
||||
Last24h int `json:"last_24h"`
|
||||
Last7d int `json:"last_7d"`
|
||||
DistinctDomains int `json:"distinct_domains"`
|
||||
DistinctClients int `json:"distinct_clients"`
|
||||
ErrorRate float64 `json:"error_rate"`
|
||||
} `json:"totals"`
|
||||
PerDay []struct {
|
||||
Day string `json:"day"`
|
||||
Total int `json:"total"`
|
||||
Errors int `json:"errors"`
|
||||
} `json:"per_day"`
|
||||
TopDomains []struct {
|
||||
Name string `json:"name"`
|
||||
Count int `json:"count"`
|
||||
} `json:"top_domains"`
|
||||
QueryTypes []struct {
|
||||
Name string `json:"name"`
|
||||
Count int `json:"count"`
|
||||
} `json:"query_types"`
|
||||
Statuses []struct {
|
||||
Name string `json:"name"`
|
||||
Count int `json:"count"`
|
||||
} `json:"statuses"`
|
||||
Durations struct {
|
||||
Count int `json:"count"`
|
||||
AvgMS float64 `json:"avg_ms"`
|
||||
P50MS int64 `json:"p50_ms"`
|
||||
P95MS int64 `json:"p95_ms"`
|
||||
} `json:"durations"`
|
||||
TopClients []struct {
|
||||
Name string `json:"name"`
|
||||
Count int `json:"count"`
|
||||
} `json:"top_clients"`
|
||||
}
|
||||
|
||||
func statsGET(t *testing.T, h http.Handler, query string) statsBody {
|
||||
t.Helper()
|
||||
w := get(h, "/admin/api/stats"+query, testAdminUser, testAdminPass)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("GET stats%s = %d (%s)", query, w.Code, w.Body)
|
||||
}
|
||||
var resp statsBody
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("decode stats: %v", err)
|
||||
}
|
||||
return resp
|
||||
}
|
||||
|
||||
func TestAdminStatsEndpoint(t *testing.T) {
|
||||
h, st := newTestHandler(t, "")
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC()
|
||||
|
||||
complete := func(id, domain, qtype, ip, status string, ago time.Duration, durMS int64) {
|
||||
t.Helper()
|
||||
ev := store.CompleteEvent{
|
||||
ID: id, Domain: domain, QueryType: qtype, ClientIP: ip,
|
||||
StartedAt: now.Add(-ago), DoneAt: now.Add(-ago).Add(time.Duration(durMS) * time.Millisecond),
|
||||
DurationMS: durMS, Status: status, ResultCount: 1,
|
||||
}
|
||||
if status == store.StatusError {
|
||||
ev.Error = "boom"
|
||||
}
|
||||
if err := st.RecordComplete(ctx, ev); err != nil {
|
||||
t.Fatalf("seed %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
complete("r1", "a.com", "A", "203.0.113.1", store.StatusComplete, time.Hour, 100)
|
||||
complete("r2", "a.com", "A", "203.0.113.2", store.StatusError, 2*time.Hour, 200)
|
||||
complete("r3", "b.net", "A", "203.0.113.1", store.StatusComplete, 30*time.Hour, 300)
|
||||
complete("r4", "c.org", "MX", "203.0.113.3", store.StatusComplete, 8*24*time.Hour, 400)
|
||||
if err := st.RecordStart(ctx, store.StartEvent{
|
||||
ID: "r5", Domain: "a.com", QueryType: "A", ClientIP: "203.0.113.1",
|
||||
StartedAt: now.Add(-time.Hour),
|
||||
}); err != nil {
|
||||
t.Fatalf("seed r5: %v", err)
|
||||
}
|
||||
|
||||
s := statsGET(t, h, "?days=30")
|
||||
if s.Days != 30 {
|
||||
t.Errorf("days = %d, want 30", s.Days)
|
||||
}
|
||||
tot := s.Totals
|
||||
if tot.AllTime != 5 || tot.Last24h != 3 || tot.Last7d != 4 ||
|
||||
tot.DistinctDomains != 3 || tot.DistinctClients != 3 {
|
||||
t.Errorf("totals = %+v, want all 5, 24h 3, 7d 4, domains 3, clients 3", tot)
|
||||
}
|
||||
if tot.ErrorRate != 0.2 {
|
||||
t.Errorf("error_rate = %v, want 0.2", tot.ErrorRate)
|
||||
}
|
||||
|
||||
var perDayTotal, perDayErrors int
|
||||
for _, d := range s.PerDay {
|
||||
perDayTotal += d.Total
|
||||
perDayErrors += d.Errors
|
||||
}
|
||||
if perDayTotal != 5 || perDayErrors != 1 {
|
||||
t.Errorf("per_day sums = %d/%d, want 5/1 (%+v)", perDayTotal, perDayErrors, s.PerDay)
|
||||
}
|
||||
|
||||
if len(s.TopDomains) != 3 || s.TopDomains[0].Name != "a.com" || s.TopDomains[0].Count != 3 {
|
||||
t.Errorf("top_domains = %+v, want a.com x3 leading 3", s.TopDomains)
|
||||
}
|
||||
if len(s.QueryTypes) != 2 || s.QueryTypes[0].Name != "A" || s.QueryTypes[0].Count != 4 ||
|
||||
s.QueryTypes[1].Name != "MX" || s.QueryTypes[1].Count != 1 {
|
||||
t.Errorf("query_types = %+v, want A x4, MX x1", s.QueryTypes)
|
||||
}
|
||||
if len(s.Statuses) != 3 || s.Statuses[0].Name != store.StatusComplete || s.Statuses[0].Count != 3 {
|
||||
t.Errorf("statuses = %+v, want complete x3 leading", s.Statuses)
|
||||
}
|
||||
d := s.Durations
|
||||
if d.Count != 4 || d.AvgMS != 250 || d.P50MS != 200 || d.P95MS != 400 {
|
||||
t.Errorf("durations = %+v, want count 4 avg 250 p50 200 p95 400", d)
|
||||
}
|
||||
if len(s.TopClients) != 3 || s.TopClients[0].Name != "203.0.113.1" || s.TopClients[0].Count != 3 {
|
||||
t.Errorf("top_clients = %+v, want 203.0.113.1 x3 leading 3", s.TopClients)
|
||||
}
|
||||
|
||||
// A 7-day window drops r4 from the windowed aggregates but not from the
|
||||
// fixed totals.
|
||||
s = statsGET(t, h, "?days=7")
|
||||
if s.Totals.AllTime != 5 {
|
||||
t.Errorf("7d all_time = %d, want 5", s.Totals.AllTime)
|
||||
}
|
||||
if s.Durations.Count != 3 || s.Durations.AvgMS != 200 {
|
||||
t.Errorf("7d durations = %+v, want count 3 avg 200", s.Durations)
|
||||
}
|
||||
if len(s.QueryTypes) != 1 || s.QueryTypes[0].Count != 4 {
|
||||
t.Errorf("7d query_types = %+v, want A x4 only", s.QueryTypes)
|
||||
}
|
||||
|
||||
// days above the cap clamps to 365.
|
||||
if s := statsGET(t, h, "?days=9999"); s.Days != 365 {
|
||||
t.Errorf("days = %d, want 365", s.Days)
|
||||
}
|
||||
// default is 30.
|
||||
if s := statsGET(t, h, ""); s.Days != 30 {
|
||||
t.Errorf("default days = %d, want 30", s.Days)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdminStatsEmptyStore(t *testing.T) {
|
||||
h, _ := newTestHandler(t, "")
|
||||
s := statsGET(t, h, "")
|
||||
if s.Totals.AllTime != 0 || s.Totals.ErrorRate != 0 {
|
||||
t.Errorf("empty totals = %+v, want zeros", s.Totals)
|
||||
}
|
||||
// Arrays must be present (possibly empty), never null.
|
||||
w := get(h, "/admin/api/stats", testAdminUser, testAdminPass)
|
||||
body := w.Body.String()
|
||||
for _, key := range []string{`"per_day":[]`, `"top_domains":[]`, `"query_types":[]`, `"statuses":[]`, `"top_clients":[]`} {
|
||||
if !strings.Contains(body, key) {
|
||||
t.Errorf("stats body missing %s: %s", key, body)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestStaticAdmin_Markup asserts the admin page ships the stat cards,
|
||||
// charts, log table, and controls the API is built for.
|
||||
func TestStaticAdmin_Markup(t *testing.T) {
|
||||
h, _ := newTestHandler(t, "")
|
||||
w := get(h, "/admin", testAdminUser, testAdminPass)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("GET /admin = %d", w.Code)
|
||||
}
|
||||
if ct := w.Header().Get("Content-Type"); !strings.HasPrefix(ct, "text/html") {
|
||||
t.Errorf("Content-Type = %q, want text/html", ct)
|
||||
}
|
||||
page := w.Body.String()
|
||||
for _, want := range []string{
|
||||
`id="statAllTime"`, `id="statLast24h"`, `id="statLast7d"`,
|
||||
`id="statDomains"`, `id="statClients"`, `id="statErrorRate"`,
|
||||
`id="perDayChart"`, `id="statusChart"`, `id="domainsChart"`, `id="typesChart"`,
|
||||
`cdn.jsdelivr.net/npm/chart.js@4`, // Chart.js 4, lazy-loaded
|
||||
`id="logTable"`, `id="logBody"`,
|
||||
`<th>Received</th><th>Domain</th><th>Type</th><th>Status</th><th>Duration</th><th>Client IP</th><th>Results</th>`,
|
||||
`id="filterDomain"`, `id="filterStatus"`, `id="filterFrom"`, `id="filterTo"`, `id="applyFilters"`,
|
||||
`id="prevPage"`, `id="nextPage"`, `id="pageInfo"`,
|
||||
`id="autoRefresh"`, `id="daysSelect"`,
|
||||
`/admin/api/stats`, `/admin/api/traversals`,
|
||||
} {
|
||||
if !strings.Contains(page, want) {
|
||||
t.Errorf("admin.html missing %q", want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestStaticAdmin_ScriptSyntax runs node --check over the inline admin
|
||||
// script so syntax errors fail in CI, not in the browser.
|
||||
func TestStaticAdmin_ScriptSyntax(t *testing.T) {
|
||||
nodeBin, err := exec.LookPath("node")
|
||||
if err != nil {
|
||||
t.Skip("node not installed")
|
||||
}
|
||||
page := string(adminHTML)
|
||||
start := strings.Index(page, "<script>")
|
||||
end := strings.LastIndex(page, "</script>")
|
||||
if start < 0 || end < 0 || end < start {
|
||||
t.Fatal("admin.html has no inline script")
|
||||
}
|
||||
src := page[start+len("<script>") : end]
|
||||
f := filepath.Join(t.TempDir(), "admin.js")
|
||||
if err := os.WriteFile(f, []byte(src), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if out, err := exec.Command(nodeBin, "--check", f).CombinedOutput(); err != nil {
|
||||
t.Fatalf("node --check: %v\n%s", err, out)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,222 @@
|
||||
// Package server exposes the webhook telemetry receiver over HTTP:
|
||||
// - POST /webhook — ingest start/complete events from the main app
|
||||
// - GET /healthz — health check
|
||||
// - GET /admin — basic-auth-protected admin UI and JSON API
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.hansenits.com.au/hits/ExploreDNS/internal/receiver/store"
|
||||
)
|
||||
|
||||
// maxBodyBytes caps webhook request bodies, matching the sender-side API
|
||||
// request cap.
|
||||
const maxBodyBytes = 1 << 20
|
||||
|
||||
// Server is the receiver HTTP server.
|
||||
type Server struct {
|
||||
addr string
|
||||
version string
|
||||
token string
|
||||
adminUser string
|
||||
adminPass string
|
||||
st *store.Store
|
||||
srv *http.Server
|
||||
}
|
||||
|
||||
// New creates a Server that listens on addr and writes events to st.
|
||||
func New(addr string, st *store.Store) *Server {
|
||||
return &Server{addr: addr, version: "dev", st: st}
|
||||
}
|
||||
|
||||
// SetVersion records the build version reported by GET /healthz.
|
||||
// Call before Start; empty values are ignored.
|
||||
func (s *Server) SetVersion(v string) {
|
||||
if v != "" {
|
||||
s.version = v
|
||||
}
|
||||
}
|
||||
|
||||
// SetIngestToken enables bearer authentication on POST /webhook. Empty
|
||||
// leaves the endpoint open. Call before Start.
|
||||
func (s *Server) SetIngestToken(t string) { s.token = t }
|
||||
|
||||
// SetAdminAuth sets the basic-auth credentials for /admin. Call before
|
||||
// Start; Start refuses to run without a password so the admin interface
|
||||
// can never be exposed unprotected.
|
||||
func (s *Server) SetAdminAuth(user, pass string) {
|
||||
s.adminUser = user
|
||||
s.adminPass = pass
|
||||
}
|
||||
|
||||
// Start begins listening. Call Shutdown to stop gracefully.
|
||||
func (s *Server) Start() error {
|
||||
if s.adminPass == "" {
|
||||
return errors.New("admin password not set (RECEIVER_ADMIN_PASSWORD): refusing to expose /admin unprotected")
|
||||
}
|
||||
s.srv = &http.Server{
|
||||
Addr: s.addr,
|
||||
Handler: newHandler(s.st, s.version, s.token, s.adminUser, s.adminPass),
|
||||
ReadHeaderTimeout: 10 * time.Second,
|
||||
ReadTimeout: 30 * time.Second,
|
||||
WriteTimeout: 30 * time.Second,
|
||||
IdleTimeout: 120 * time.Second,
|
||||
}
|
||||
|
||||
ln, err := net.Listen("tcp", s.addr)
|
||||
if err != nil {
|
||||
return fmt.Errorf("listen %s: %w", s.addr, err)
|
||||
}
|
||||
s.addr = ln.Addr().String()
|
||||
|
||||
go func() {
|
||||
if err := s.srv.Serve(ln); err != nil && err != http.ErrServerClosed {
|
||||
log.Printf("receiver server: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Addr returns the address the server is listening on. Valid after Start.
|
||||
func (s *Server) Addr() string { return s.addr }
|
||||
|
||||
// Shutdown gracefully stops the server, waiting up to timeout for in-flight
|
||||
// requests to complete.
|
||||
func (s *Server) Shutdown(timeout time.Duration) error {
|
||||
if s.srv == nil {
|
||||
return nil
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
defer cancel()
|
||||
return s.srv.Shutdown(ctx)
|
||||
}
|
||||
|
||||
// handler routes receiver HTTP requests.
|
||||
type handler struct {
|
||||
st *store.Store
|
||||
version string
|
||||
token string
|
||||
adminUser string
|
||||
adminPass string
|
||||
mux *http.ServeMux
|
||||
}
|
||||
|
||||
// newHandler builds the receiver's HTTP handler. token "" leaves /webhook
|
||||
// open; adminPass "" leaves /admin permanently locked (every request 401s).
|
||||
func newHandler(st *store.Store, version, token, adminUser, adminPass string) http.Handler {
|
||||
h := &handler{st: st, version: version, token: token,
|
||||
adminUser: adminUser, adminPass: adminPass, mux: http.NewServeMux()}
|
||||
h.mux.HandleFunc("POST /webhook", h.ingest)
|
||||
h.mux.HandleFunc("GET /healthz", h.healthz)
|
||||
|
||||
// The whole /admin subtree sits behind basic auth, including paths the
|
||||
// inner mux will 404.
|
||||
admin := http.NewServeMux()
|
||||
admin.HandleFunc("GET /admin", h.adminPage)
|
||||
admin.HandleFunc("GET /admin/{$}", h.adminPage)
|
||||
admin.HandleFunc("GET /admin/api/traversals", h.adminTraversals)
|
||||
admin.HandleFunc("GET /admin/api/stats", h.adminStats)
|
||||
protected := h.requireAdmin(admin)
|
||||
h.mux.Handle("/admin", protected)
|
||||
h.mux.Handle("/admin/", protected)
|
||||
return h.mux
|
||||
}
|
||||
|
||||
// healthz handles GET /healthz.
|
||||
func (h *handler) healthz(w http.ResponseWriter, _ *http.Request) {
|
||||
writeJSON(w, http.StatusOK, map[string]string{"status": "ok", "version": h.version})
|
||||
}
|
||||
|
||||
// ingest handles POST /webhook: it routes on the payload's "event" field
|
||||
// and upserts the event into the store. The payload structs live in the
|
||||
// store package and mirror web/api/webhook.go exactly.
|
||||
func (h *handler) ingest(w http.ResponseWriter, r *http.Request) {
|
||||
if !h.authorized(r) {
|
||||
w.Header().Set("WWW-Authenticate", "Bearer")
|
||||
writeError(w, http.StatusUnauthorized, "missing or invalid bearer token")
|
||||
return
|
||||
}
|
||||
|
||||
r.Body = http.MaxBytesReader(w, r.Body, maxBodyBytes)
|
||||
body, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
var tooLarge *http.MaxBytesError
|
||||
if errors.As(err, &tooLarge) {
|
||||
writeError(w, http.StatusRequestEntityTooLarge,
|
||||
fmt.Sprintf("body exceeds %d bytes", tooLarge.Limit))
|
||||
return
|
||||
}
|
||||
writeError(w, http.StatusBadRequest, "read body: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
var probe struct {
|
||||
Event string `json:"event"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &probe); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid JSON: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
switch probe.Event {
|
||||
case "start":
|
||||
var ev store.StartEvent
|
||||
if err := json.Unmarshal(body, &ev); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid start event: "+err.Error())
|
||||
return
|
||||
}
|
||||
err = h.st.RecordStart(r.Context(), ev)
|
||||
case "complete":
|
||||
var ev store.CompleteEvent
|
||||
if err := json.Unmarshal(body, &ev); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid complete event: "+err.Error())
|
||||
return
|
||||
}
|
||||
err = h.st.RecordComplete(r.Context(), ev)
|
||||
default:
|
||||
writeError(w, http.StatusBadRequest, fmt.Sprintf("unknown event %q", probe.Event))
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
log.Printf("receiver: store %s event: %v", probe.Event, err)
|
||||
writeError(w, http.StatusInternalServerError, "store event failed")
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
// authorized checks the bearer token when one is configured.
|
||||
func (h *handler) authorized(r *http.Request) bool {
|
||||
if h.token == "" {
|
||||
return true
|
||||
}
|
||||
const prefix = "Bearer "
|
||||
auth := r.Header.Get("Authorization")
|
||||
if !strings.HasPrefix(auth, prefix) {
|
||||
return false
|
||||
}
|
||||
return secretEqual(strings.TrimPrefix(auth, prefix), h.token)
|
||||
}
|
||||
|
||||
// writeJSON encodes v as JSON and writes it to w with the given status code.
|
||||
func writeJSON(w http.ResponseWriter, status int, v any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
_ = json.NewEncoder(w).Encode(v)
|
||||
}
|
||||
|
||||
// writeError writes a JSON error response.
|
||||
func writeError(w http.ResponseWriter, status int, msg string) {
|
||||
writeJSON(w, status, map[string]string{"error": msg})
|
||||
}
|
||||
@@ -0,0 +1,209 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.hansenits.com.au/hits/ExploreDNS/internal/receiver/store"
|
||||
)
|
||||
|
||||
func newTestHandler(t *testing.T, token string) (http.Handler, *store.Store) {
|
||||
t.Helper()
|
||||
st, err := store.OpenSQLite(filepath.Join(t.TempDir(), "receiver.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("OpenSQLite: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { st.Close() })
|
||||
return newHandler(st, "test-version", token, testAdminUser, testAdminPass), st
|
||||
}
|
||||
|
||||
func post(h http.Handler, body string, headers map[string]string) *httptest.ResponseRecorder {
|
||||
req := httptest.NewRequest(http.MethodPost, "/webhook", strings.NewReader(body))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
for k, v := range headers {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, req)
|
||||
return w
|
||||
}
|
||||
|
||||
const startJSON = `{"event":"start","id":"job-1","domain":"example.com","query_type":"A",` +
|
||||
`"all_roots":true,"client_ip":"203.0.113.9","started_at":"2026-07-01T10:00:00Z"}`
|
||||
|
||||
const completeJSON = `{"event":"complete","id":"job-1","domain":"example.com","query_type":"A",` +
|
||||
`"client_ip":"203.0.113.9","started_at":"2026-07-01T10:00:00Z","done_at":"2026-07-01T10:00:03Z",` +
|
||||
`"duration_ms":3000,"status":"complete","result_count":2,` +
|
||||
`"summary":{"answers":[{"probability":1,"records":["example.com 300 IN A 192.0.2.1"]}]}}`
|
||||
|
||||
func TestIngestAuthMatrix(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
token string
|
||||
authHeader string
|
||||
want int
|
||||
}{
|
||||
{"no token configured, no header", "", "", http.StatusNoContent},
|
||||
{"no token configured, stray header", "", "Bearer whatever", http.StatusNoContent},
|
||||
{"token configured, missing header", "s3cret", "", http.StatusUnauthorized},
|
||||
{"token configured, wrong scheme", "s3cret", "Basic s3cret", http.StatusUnauthorized},
|
||||
{"token configured, wrong token", "s3cret", "Bearer nope", http.StatusUnauthorized},
|
||||
{"token configured, correct token", "s3cret", "Bearer s3cret", http.StatusNoContent},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
h, _ := newTestHandler(t, tc.token)
|
||||
headers := map[string]string{}
|
||||
if tc.authHeader != "" {
|
||||
headers["Authorization"] = tc.authHeader
|
||||
}
|
||||
if got := post(h, startJSON, headers).Code; got != tc.want {
|
||||
t.Errorf("status = %d, want %d", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIngestStartEvent(t *testing.T) {
|
||||
h, st := newTestHandler(t, "")
|
||||
|
||||
w := post(h, startJSON, nil)
|
||||
if w.Code != http.StatusNoContent {
|
||||
t.Fatalf("status = %d, want 204 (%s)", w.Code, w.Body)
|
||||
}
|
||||
|
||||
rows, total, err := st.ListTraversals(context.Background(), store.ListFilter{}, 10, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("ListTraversals: %v", err)
|
||||
}
|
||||
if total != 1 {
|
||||
t.Fatalf("total = %d, want 1", total)
|
||||
}
|
||||
row := rows[0]
|
||||
if row.ID != "job-1" || row.Domain != "example.com" || row.QueryType != "A" ||
|
||||
!row.AllRoots || row.ClientIP != "203.0.113.9" || row.Status != store.StatusRunning {
|
||||
t.Errorf("row = %+v", row)
|
||||
}
|
||||
want := time.Date(2026, 7, 1, 10, 0, 0, 0, time.UTC)
|
||||
if !row.StartedAt.Equal(want) {
|
||||
t.Errorf("StartedAt = %v, want %v", row.StartedAt, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIngestCompleteEvent(t *testing.T) {
|
||||
h, st := newTestHandler(t, "")
|
||||
|
||||
if w := post(h, startJSON, nil); w.Code != http.StatusNoContent {
|
||||
t.Fatalf("start status = %d", w.Code)
|
||||
}
|
||||
if w := post(h, completeJSON, nil); w.Code != http.StatusNoContent {
|
||||
t.Fatalf("complete status = %d", w.Code)
|
||||
}
|
||||
|
||||
rows, total, err := st.ListTraversals(context.Background(), store.ListFilter{}, 10, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("ListTraversals: %v", err)
|
||||
}
|
||||
if total != 1 {
|
||||
t.Fatalf("total = %d, want 1", total)
|
||||
}
|
||||
row := rows[0]
|
||||
if row.Status != store.StatusComplete {
|
||||
t.Errorf("Status = %q, want complete", row.Status)
|
||||
}
|
||||
if row.DurationMS == nil || *row.DurationMS != 3000 {
|
||||
t.Errorf("DurationMS = %v, want 3000", row.DurationMS)
|
||||
}
|
||||
if row.ResultCount == nil || *row.ResultCount != 2 {
|
||||
t.Errorf("ResultCount = %v, want 2", row.ResultCount)
|
||||
}
|
||||
if !strings.Contains(row.Summary, "192.0.2.1") {
|
||||
t.Errorf("Summary = %q, want raw summary JSON", row.Summary)
|
||||
}
|
||||
if !row.AllRoots {
|
||||
t.Error("AllRoots lost after complete")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIngestRejectsBadInput(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
body string
|
||||
}{
|
||||
{"malformed JSON", `{"event":`},
|
||||
{"unknown event", `{"event":"pause","id":"x"}`},
|
||||
{"empty event", `{"id":"x"}`},
|
||||
{"wrong type for field", `{"event":"start","id":42}`},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
h, st := newTestHandler(t, "")
|
||||
if got := post(h, tc.body, nil).Code; got != http.StatusBadRequest {
|
||||
t.Errorf("status = %d, want 400", got)
|
||||
}
|
||||
_, total, err := st.ListTraversals(context.Background(), store.ListFilter{}, 10, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("ListTraversals: %v", err)
|
||||
}
|
||||
if total != 0 {
|
||||
t.Errorf("stored %d rows from rejected input", total)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIngestBodyCap(t *testing.T) {
|
||||
h, _ := newTestHandler(t, "")
|
||||
big := `{"event":"start","id":"x","domain":"` + strings.Repeat("a", maxBodyBytes) + `"}`
|
||||
if got := post(h, big, nil).Code; got != http.StatusRequestEntityTooLarge {
|
||||
t.Errorf("status = %d, want 413", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHealthz(t *testing.T) {
|
||||
h, _ := newTestHandler(t, "s3cret")
|
||||
req := httptest.NewRequest(http.MethodGet, "/healthz", nil)
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, req)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200", w.Code)
|
||||
}
|
||||
body := w.Body.String()
|
||||
// healthz stays open even when an ingest token is configured.
|
||||
if !strings.Contains(body, `"status":"ok"`) || !strings.Contains(body, `"version":"test-version"`) {
|
||||
t.Errorf("body = %s", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServerStartShutdown(t *testing.T) {
|
||||
st, err := store.OpenSQLite(filepath.Join(t.TempDir(), "receiver.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("OpenSQLite: %v", err)
|
||||
}
|
||||
defer st.Close()
|
||||
|
||||
srv := New("127.0.0.1:0", st)
|
||||
srv.SetVersion("v-test")
|
||||
srv.SetAdminAuth(testAdminUser, testAdminPass)
|
||||
if err := srv.Start(); err != nil {
|
||||
t.Fatalf("Start: %v", err)
|
||||
}
|
||||
defer srv.Shutdown(2 * time.Second)
|
||||
|
||||
resp, err := http.Get("http://" + srv.Addr() + "/healthz")
|
||||
if err != nil {
|
||||
t.Fatalf("GET /healthz: %v", err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Errorf("healthz status = %d, want 200", resp.StatusCode)
|
||||
}
|
||||
if err := srv.Shutdown(2 * time.Second); err != nil {
|
||||
t.Errorf("Shutdown: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,486 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en" data-theme="dark">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>ExploreDNS Receiver — Admin</title>
|
||||
<style>
|
||||
/* ── Design tokens (mirrors web/api/static/index.html) ─────────── */
|
||||
:root {
|
||||
--bg: #0f1117;
|
||||
--bg-surface: #1a1d27;
|
||||
--bg-card: #21242f;
|
||||
--bg-input: #2a2d3a;
|
||||
--border: #353847;
|
||||
--border-focus: #4a7cf6;
|
||||
--text: #e8eaf0;
|
||||
--text-muted: #8b8fa8;
|
||||
--text-dim: #5c6070;
|
||||
--primary: #4a7cf6;
|
||||
--primary-hover:#3a6ce6;
|
||||
--success: #3ecf8e;
|
||||
--warning: #f5a623;
|
||||
--danger: #f04438;
|
||||
--info: #38bdf8;
|
||||
--radius: 8px;
|
||||
--radius-sm: 5px;
|
||||
--shadow: 0 4px 20px rgba(0,0,0,0.4);
|
||||
--font-mono: 'JetBrains Mono', 'Fira Mono', 'Cascadia Code', 'Consolas', monospace;
|
||||
}
|
||||
|
||||
/* ── Reset ─────────────────────────────────────────────────────── */
|
||||
*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
|
||||
html { font-size: 15px; }
|
||||
body {
|
||||
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
|
||||
background: var(--bg);
|
||||
color: var(--text);
|
||||
min-height: 100vh;
|
||||
line-height: 1.5;
|
||||
}
|
||||
a { color: var(--primary); text-decoration: none; }
|
||||
button { cursor: pointer; font: inherit; }
|
||||
|
||||
/* ── Layout ─────────────────────────────────────────────────────── */
|
||||
.app { display: flex; flex-direction: column; min-height: 100vh; }
|
||||
header {
|
||||
background: var(--bg-surface);
|
||||
border-bottom: 1px solid var(--border);
|
||||
padding: 0 1.5rem;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
height: 56px;
|
||||
position: sticky;
|
||||
top: 0;
|
||||
z-index: 100;
|
||||
}
|
||||
.logo {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.5rem;
|
||||
font-weight: 700;
|
||||
font-size: 1.1rem;
|
||||
letter-spacing: -0.01em;
|
||||
}
|
||||
.logo-icon {
|
||||
width: 28px; height: 28px;
|
||||
background: var(--primary);
|
||||
border-radius: 7px;
|
||||
display: flex; align-items: center; justify-content: center;
|
||||
font-size: 14px;
|
||||
}
|
||||
.logo-sub { color: var(--text-muted); font-weight: 500; font-size: 0.85rem; }
|
||||
.header-controls { display: flex; align-items: center; gap: 1rem; font-size: 0.85rem; color: var(--text-muted); }
|
||||
.header-controls label { display: flex; align-items: center; gap: 0.35rem; cursor: pointer; }
|
||||
main { flex: 1; padding: 2rem 1.5rem; max-width: 1100px; margin: 0 auto; width: 100%; }
|
||||
footer {
|
||||
text-align: center;
|
||||
padding: 1rem;
|
||||
color: var(--text-dim);
|
||||
font-size: 0.8rem;
|
||||
border-top: 1px solid var(--border);
|
||||
}
|
||||
|
||||
/* ── Cards ─────────────────────────────────────────────────────── */
|
||||
.card {
|
||||
background: var(--bg-card);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius);
|
||||
padding: 1.25rem 1.5rem;
|
||||
box-shadow: var(--shadow);
|
||||
}
|
||||
.card-title {
|
||||
font-weight: 600;
|
||||
font-size: 0.85rem;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.06em;
|
||||
color: var(--text-muted);
|
||||
margin-bottom: 1rem;
|
||||
}
|
||||
|
||||
/* ── Stat cards ────────────────────────────────────────────────── */
|
||||
.stat-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fit, minmax(150px, 1fr));
|
||||
gap: 0.75rem;
|
||||
margin-bottom: 1.5rem;
|
||||
}
|
||||
.stat-card { padding: 0.9rem 1.1rem; }
|
||||
.stat-label {
|
||||
font-size: 0.72rem;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.06em;
|
||||
color: var(--text-muted);
|
||||
}
|
||||
.stat-value { font-size: 1.5rem; font-weight: 700; font-variant-numeric: tabular-nums; }
|
||||
|
||||
/* ── Charts ────────────────────────────────────────────────────── */
|
||||
.chart-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fit, minmax(300px, 1fr));
|
||||
gap: 0.75rem;
|
||||
margin-bottom: 1.5rem;
|
||||
}
|
||||
.chart-card.wide { grid-column: 1 / -1; }
|
||||
.chart-card canvas { max-height: 260px; }
|
||||
|
||||
/* ── Filters ───────────────────────────────────────────────────── */
|
||||
.filters { display: flex; gap: 0.5rem; flex-wrap: wrap; margin-bottom: 1rem; }
|
||||
.filters input, .filters select, .header-controls select {
|
||||
background: var(--bg-input);
|
||||
border: 1px solid var(--border);
|
||||
color: var(--text);
|
||||
border-radius: var(--radius-sm);
|
||||
padding: 6px 10px;
|
||||
font: inherit;
|
||||
font-size: 0.85rem;
|
||||
}
|
||||
.filters input:focus, .filters select:focus { outline: none; border-color: var(--border-focus); }
|
||||
.filters button, .pager button {
|
||||
background: var(--primary);
|
||||
border: none;
|
||||
color: #fff;
|
||||
border-radius: var(--radius-sm);
|
||||
padding: 6px 14px;
|
||||
font-size: 0.85rem;
|
||||
}
|
||||
.filters button:hover, .pager button:hover:not(:disabled) { background: var(--primary-hover); }
|
||||
.pager button:disabled { background: var(--bg-input); color: var(--text-dim); cursor: default; }
|
||||
|
||||
/* ── Log table ─────────────────────────────────────────────────── */
|
||||
table { width: 100%; border-collapse: collapse; font-size: 0.85rem; }
|
||||
th, td { text-align: left; padding: 7px 10px; border-bottom: 1px solid var(--border); }
|
||||
th {
|
||||
font-size: 0.72rem;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.06em;
|
||||
color: var(--text-muted);
|
||||
}
|
||||
tr.log-row { cursor: pointer; }
|
||||
tr.log-row:hover td { background: var(--bg-input); }
|
||||
td.mono { font-family: var(--font-mono); font-size: 0.8rem; }
|
||||
.chip {
|
||||
display: inline-block;
|
||||
padding: 1px 9px;
|
||||
border-radius: 999px;
|
||||
font-size: 0.72rem;
|
||||
font-weight: 600;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.04em;
|
||||
}
|
||||
.chip-complete { background: rgba(62,207,142,0.15); color: var(--success); }
|
||||
.chip-error { background: rgba(240,68,56,0.15); color: var(--danger); }
|
||||
.chip-running { background: rgba(56,189,248,0.15); color: var(--info); }
|
||||
tr.detail-row td { background: var(--bg-surface); padding: 0.75rem 1rem; }
|
||||
tr.detail-row pre {
|
||||
font-family: var(--font-mono);
|
||||
font-size: 0.75rem;
|
||||
white-space: pre-wrap;
|
||||
word-break: break-word;
|
||||
max-height: 320px;
|
||||
overflow: auto;
|
||||
color: var(--text-muted);
|
||||
}
|
||||
.pager {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: flex-end;
|
||||
gap: 0.75rem;
|
||||
margin-top: 1rem;
|
||||
font-size: 0.85rem;
|
||||
color: var(--text-muted);
|
||||
}
|
||||
.empty { color: var(--text-dim); text-align: center; padding: 1.5rem 0; }
|
||||
|
||||
#errorBar {
|
||||
display: none;
|
||||
background: rgba(240,68,56,0.12);
|
||||
border: 1px solid var(--danger);
|
||||
color: var(--danger);
|
||||
border-radius: var(--radius-sm);
|
||||
padding: 0.5rem 1rem;
|
||||
margin-bottom: 1rem;
|
||||
font-size: 0.85rem;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="app">
|
||||
<header>
|
||||
<div class="logo"><div class="logo-icon">📡</div>ExploreDNS <span class="logo-sub">receiver admin</span></div>
|
||||
<div class="header-controls">
|
||||
<label><input type="checkbox" id="autoRefresh"> auto-refresh 30s</label>
|
||||
<select id="daysSelect" title="chart window">
|
||||
<option value="7">7 days</option>
|
||||
<option value="30" selected>30 days</option>
|
||||
<option value="90">90 days</option>
|
||||
</select>
|
||||
</div>
|
||||
</header>
|
||||
<main>
|
||||
<div id="errorBar"></div>
|
||||
|
||||
<section class="stat-grid" id="statCards">
|
||||
<div class="card stat-card"><div class="stat-label">All time</div><div class="stat-value" id="statAllTime">–</div></div>
|
||||
<div class="card stat-card"><div class="stat-label">Last 24 h</div><div class="stat-value" id="statLast24h">–</div></div>
|
||||
<div class="card stat-card"><div class="stat-label">Last 7 d</div><div class="stat-value" id="statLast7d">–</div></div>
|
||||
<div class="card stat-card"><div class="stat-label">Domains</div><div class="stat-value" id="statDomains">–</div></div>
|
||||
<div class="card stat-card"><div class="stat-label">Clients</div><div class="stat-value" id="statClients">–</div></div>
|
||||
<div class="card stat-card"><div class="stat-label">Error rate</div><div class="stat-value" id="statErrorRate">–</div></div>
|
||||
</section>
|
||||
|
||||
<section class="chart-grid">
|
||||
<div class="card chart-card wide"><div class="card-title">Traversals per day</div><canvas id="perDayChart"></canvas></div>
|
||||
<div class="card chart-card"><div class="card-title">Status</div><canvas id="statusChart"></canvas></div>
|
||||
<div class="card chart-card"><div class="card-title">Top domains</div><canvas id="domainsChart"></canvas></div>
|
||||
<div class="card chart-card"><div class="card-title">Query types</div><canvas id="typesChart"></canvas></div>
|
||||
</section>
|
||||
|
||||
<section class="card log-card">
|
||||
<div class="card-title">Log</div>
|
||||
<div class="filters">
|
||||
<input id="filterDomain" placeholder="domain contains…">
|
||||
<select id="filterStatus">
|
||||
<option value="">all statuses</option>
|
||||
<option value="running">running</option>
|
||||
<option value="complete">complete</option>
|
||||
<option value="error">error</option>
|
||||
</select>
|
||||
<input type="datetime-local" id="filterFrom" title="from">
|
||||
<input type="datetime-local" id="filterTo" title="to">
|
||||
<button id="applyFilters">Apply</button>
|
||||
</div>
|
||||
<table id="logTable">
|
||||
<thead>
|
||||
<tr><th>Received</th><th>Domain</th><th>Type</th><th>Status</th><th>Duration</th><th>Client IP</th><th>Results</th></tr>
|
||||
</thead>
|
||||
<tbody id="logBody"></tbody>
|
||||
</table>
|
||||
<div class="pager">
|
||||
<button id="prevPage">‹ Prev</button>
|
||||
<span id="pageInfo"></span>
|
||||
<button id="nextPage">Next ›</button>
|
||||
</div>
|
||||
</section>
|
||||
</main>
|
||||
<footer>ExploreDNS webhook receiver</footer>
|
||||
</div>
|
||||
<script>
|
||||
(() => {
|
||||
'use strict';
|
||||
const CHART_SRC = 'https://cdn.jsdelivr.net/npm/chart.js@4';
|
||||
const PAGE_SIZE = 50;
|
||||
const $ = (id) => document.getElementById(id);
|
||||
let page = 0;
|
||||
let total = 0;
|
||||
const charts = {};
|
||||
let chartLoad = null;
|
||||
let refreshTimer = null;
|
||||
|
||||
const esc = (s) => String(s).replace(/[&<>"']/g,
|
||||
(c) => ({'&':'&','<':'<','>':'>','"':'"',"'":'''}[c]));
|
||||
|
||||
// Chart.js is lazy-loaded on first use so the page renders without the CDN.
|
||||
function loadChartJS() {
|
||||
if (window.Chart) return Promise.resolve();
|
||||
if (!chartLoad) {
|
||||
chartLoad = new Promise((resolve, reject) => {
|
||||
const s = document.createElement('script');
|
||||
s.src = CHART_SRC;
|
||||
s.onload = resolve;
|
||||
s.onerror = () => { chartLoad = null; reject(new Error('failed to load Chart.js')); };
|
||||
document.head.appendChild(s);
|
||||
});
|
||||
}
|
||||
return chartLoad;
|
||||
}
|
||||
|
||||
async function fetchJSON(url) {
|
||||
// Resolve against origin (never contains userinfo) so the dashboard
|
||||
// still works when opened via a http://user:pass@host bookmark —
|
||||
// fetch() rejects relative URLs on pages with credentialed base URLs.
|
||||
const abs = new URL(url, window.location.origin).toString();
|
||||
const resp = await fetch(abs, { headers: { 'Accept': 'application/json' } });
|
||||
if (!resp.ok) throw new Error(url + ': HTTP ' + resp.status);
|
||||
return resp.json();
|
||||
}
|
||||
|
||||
function showError(err) {
|
||||
const bar = $('errorBar');
|
||||
bar.textContent = String(err);
|
||||
bar.style.display = 'block';
|
||||
}
|
||||
function clearError() { $('errorBar').style.display = 'none'; }
|
||||
|
||||
const fmtDuration = (ms) => ms == null ? '—'
|
||||
: (ms >= 1000 ? (ms / 1000).toFixed(2) + ' s' : ms + ' ms');
|
||||
const fmtLocal = (iso) => iso ? new Date(iso).toLocaleString() : '—';
|
||||
|
||||
function cssVar(name) {
|
||||
return getComputedStyle(document.documentElement).getPropertyValue(name).trim();
|
||||
}
|
||||
|
||||
function upsertChart(id, cfg) {
|
||||
if (charts[id]) {
|
||||
charts[id].data = cfg.data;
|
||||
charts[id].update();
|
||||
} else {
|
||||
charts[id] = new Chart($(id), cfg);
|
||||
}
|
||||
}
|
||||
|
||||
function drawCharts(s) {
|
||||
Chart.defaults.color = cssVar('--text-muted');
|
||||
Chart.defaults.borderColor = cssVar('--border');
|
||||
const statusColor = { complete: cssVar('--success'), error: cssVar('--danger'), running: cssVar('--info') };
|
||||
upsertChart('perDayChart', {
|
||||
type: 'line',
|
||||
data: {
|
||||
labels: s.per_day.map((d) => d.day),
|
||||
datasets: [
|
||||
{ label: 'traversals', data: s.per_day.map((d) => d.total),
|
||||
borderColor: cssVar('--primary'), backgroundColor: 'rgba(74,124,246,0.15)', fill: true, tension: 0.25 },
|
||||
{ label: 'errors', data: s.per_day.map((d) => d.errors),
|
||||
borderColor: cssVar('--danger'), backgroundColor: 'transparent', tension: 0.25 },
|
||||
],
|
||||
},
|
||||
options: { scales: { y: { beginAtZero: true, ticks: { precision: 0 } } } },
|
||||
});
|
||||
upsertChart('statusChart', {
|
||||
type: 'doughnut',
|
||||
data: {
|
||||
labels: s.statuses.map((x) => x.name),
|
||||
datasets: [{
|
||||
data: s.statuses.map((x) => x.count),
|
||||
backgroundColor: s.statuses.map((x) => statusColor[x.name] || cssVar('--warning')),
|
||||
borderWidth: 0,
|
||||
}],
|
||||
},
|
||||
});
|
||||
upsertChart('domainsChart', {
|
||||
type: 'bar',
|
||||
data: {
|
||||
labels: s.top_domains.map((x) => x.name),
|
||||
datasets: [{ label: 'traversals', data: s.top_domains.map((x) => x.count), backgroundColor: cssVar('--primary') }],
|
||||
},
|
||||
options: { indexAxis: 'y', plugins: { legend: { display: false } },
|
||||
scales: { x: { beginAtZero: true, ticks: { precision: 0 } } } },
|
||||
});
|
||||
upsertChart('typesChart', {
|
||||
type: 'bar',
|
||||
data: {
|
||||
labels: s.query_types.map((x) => x.name),
|
||||
datasets: [{ label: 'traversals', data: s.query_types.map((x) => x.count), backgroundColor: cssVar('--info') }],
|
||||
},
|
||||
options: { plugins: { legend: { display: false } },
|
||||
scales: { y: { beginAtZero: true, ticks: { precision: 0 } } } },
|
||||
});
|
||||
}
|
||||
|
||||
async function refreshStats() {
|
||||
const s = await fetchJSON('/admin/api/stats?days=' + encodeURIComponent($('daysSelect').value));
|
||||
$('statAllTime').textContent = s.totals.all_time;
|
||||
$('statLast24h').textContent = s.totals.last_24h;
|
||||
$('statLast7d').textContent = s.totals.last_7d;
|
||||
$('statDomains').textContent = s.totals.distinct_domains;
|
||||
$('statClients').textContent = s.totals.distinct_clients;
|
||||
$('statErrorRate').textContent = (s.totals.error_rate * 100).toFixed(1) + '%';
|
||||
await loadChartJS();
|
||||
drawCharts(s);
|
||||
}
|
||||
|
||||
function filterQuery() {
|
||||
const p = new URLSearchParams();
|
||||
p.set('limit', PAGE_SIZE);
|
||||
p.set('offset', page * PAGE_SIZE);
|
||||
const domain = $('filterDomain').value.trim();
|
||||
if (domain) p.set('domain', domain);
|
||||
const status = $('filterStatus').value;
|
||||
if (status) p.set('status', status);
|
||||
const from = $('filterFrom').value;
|
||||
if (from) p.set('from', new Date(from).toISOString());
|
||||
const to = $('filterTo').value;
|
||||
if (to) p.set('to', new Date(to).toISOString());
|
||||
return p;
|
||||
}
|
||||
|
||||
function detailJSON(item) {
|
||||
if (item.summary != null) return JSON.stringify(item.summary, null, 2);
|
||||
return JSON.stringify(item, null, 2);
|
||||
}
|
||||
|
||||
async function refreshLog() {
|
||||
const data = await fetchJSON('/admin/api/traversals?' + filterQuery());
|
||||
total = data.total;
|
||||
const body = $('logBody');
|
||||
body.textContent = '';
|
||||
if (!data.items.length) {
|
||||
const tr = document.createElement('tr');
|
||||
tr.innerHTML = '<td colspan="7" class="empty">no traversals</td>';
|
||||
body.appendChild(tr);
|
||||
}
|
||||
for (const item of data.items) {
|
||||
const row = document.createElement('tr');
|
||||
row.className = 'log-row';
|
||||
row.innerHTML =
|
||||
'<td>' + esc(fmtLocal(item.received_at)) + '</td>' +
|
||||
'<td class="mono">' + esc(item.domain) + '</td>' +
|
||||
'<td class="mono">' + esc(item.query_type) + '</td>' +
|
||||
'<td><span class="chip chip-' + esc(item.status) + '">' + esc(item.status) + '</span></td>' +
|
||||
'<td>' + esc(fmtDuration(item.duration_ms)) + '</td>' +
|
||||
'<td class="mono">' + esc(item.client_ip) + '</td>' +
|
||||
'<td>' + esc(item.result_count == null ? '—' : item.result_count) + '</td>';
|
||||
const detail = document.createElement('tr');
|
||||
detail.className = 'detail-row';
|
||||
detail.style.display = 'none';
|
||||
const cell = document.createElement('td');
|
||||
cell.colSpan = 7;
|
||||
const pre = document.createElement('pre');
|
||||
pre.textContent = detailJSON(item);
|
||||
cell.appendChild(pre);
|
||||
detail.appendChild(cell);
|
||||
row.addEventListener('click', () => {
|
||||
detail.style.display = detail.style.display === 'none' ? '' : 'none';
|
||||
});
|
||||
body.appendChild(row);
|
||||
body.appendChild(detail);
|
||||
}
|
||||
const first = total ? page * PAGE_SIZE + 1 : 0;
|
||||
const last = Math.min((page + 1) * PAGE_SIZE, total);
|
||||
$('pageInfo').textContent = first + '–' + last + ' of ' + total;
|
||||
$('prevPage').disabled = page === 0;
|
||||
$('nextPage').disabled = last >= total;
|
||||
}
|
||||
|
||||
function refreshAll() {
|
||||
clearError();
|
||||
refreshStats().catch(showError);
|
||||
refreshLog().catch(showError);
|
||||
}
|
||||
|
||||
$('daysSelect').addEventListener('change', () => { refreshStats().catch(showError); });
|
||||
$('applyFilters').addEventListener('click', () => { page = 0; refreshLog().catch(showError); });
|
||||
$('filterDomain').addEventListener('keydown', (e) => {
|
||||
if (e.key === 'Enter') { page = 0; refreshLog().catch(showError); }
|
||||
});
|
||||
$('prevPage').addEventListener('click', () => {
|
||||
if (page > 0) { page--; refreshLog().catch(showError); }
|
||||
});
|
||||
$('nextPage').addEventListener('click', () => {
|
||||
if ((page + 1) * PAGE_SIZE < total) { page++; refreshLog().catch(showError); }
|
||||
});
|
||||
$('autoRefresh').addEventListener('change', (e) => {
|
||||
if (e.target.checked) {
|
||||
refreshTimer = setInterval(refreshAll, 30000);
|
||||
} else {
|
||||
clearInterval(refreshTimer);
|
||||
refreshTimer = null;
|
||||
}
|
||||
});
|
||||
|
||||
refreshAll();
|
||||
})();
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,103 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// dialect abstracts the SQL syntax differences between the sqlite and
|
||||
// mysql backends. Both drivers use '?' placeholders, and all datetimes are
|
||||
// bound and scanned as "2006-01-02 15:04:05.000000" UTC strings so value
|
||||
// handling stays identical; only the constructs below diverge.
|
||||
type dialect interface {
|
||||
name() string
|
||||
|
||||
// datetimeType is the column type for datetime values. sqlite has no
|
||||
// datetime type (TEXT affinity stores our formatted strings verbatim,
|
||||
// which sort and compare lexically); mysql needs DATETIME(6) because
|
||||
// plain DATETIME truncates the sub-second precision we write.
|
||||
datetimeType() string
|
||||
|
||||
// upsert appends the insert-or-update clause to an INSERT statement,
|
||||
// overwriting cols with the values from the attempted insert. sqlite
|
||||
// spells this ON CONFLICT(id) DO UPDATE SET col=excluded.col; mysql
|
||||
// spells it ON DUPLICATE KEY UPDATE col=VALUES(col) (VALUES() is
|
||||
// deprecated in MySQL 8.0.20+ but is the only form MariaDB and older
|
||||
// MySQL also accept).
|
||||
upsert(insert string, cols []string) string
|
||||
|
||||
// createIndex creates an index if it does not exist. sqlite supports
|
||||
// CREATE INDEX IF NOT EXISTS; mysql has no IF NOT EXISTS for indexes,
|
||||
// so existence is checked via information_schema first.
|
||||
createIndex(ctx context.Context, db *sql.DB, index, table, column string) error
|
||||
|
||||
// likeContains returns a "col contains substr" predicate plus its
|
||||
// bind argument, with LIKE wildcards in substr escaped. The ESCAPE
|
||||
// literal differs: mysql string literals treat backslash as an escape
|
||||
// character (so the SQL needs '\\'), sqlite ones do not (so it needs
|
||||
// '\', and has no default escape character at all).
|
||||
likeContains(col, substr string) (predicate string, arg string)
|
||||
}
|
||||
|
||||
// escapeLike backslash-escapes LIKE pattern metacharacters in s.
|
||||
func escapeLike(s string) string {
|
||||
r := strings.NewReplacer(`\`, `\\`, `%`, `\%`, `_`, `\_`)
|
||||
return r.Replace(s)
|
||||
}
|
||||
|
||||
type sqliteDialect struct{}
|
||||
|
||||
func (sqliteDialect) name() string { return "sqlite" }
|
||||
func (sqliteDialect) datetimeType() string { return "TEXT" }
|
||||
|
||||
func (sqliteDialect) upsert(insert string, cols []string) string {
|
||||
sets := make([]string, len(cols))
|
||||
for i, c := range cols {
|
||||
sets[i] = c + "=excluded." + c
|
||||
}
|
||||
return insert + " ON CONFLICT(id) DO UPDATE SET " + strings.Join(sets, ", ")
|
||||
}
|
||||
|
||||
func (sqliteDialect) createIndex(ctx context.Context, db *sql.DB, index, table, column string) error {
|
||||
_, err := db.ExecContext(ctx,
|
||||
"CREATE INDEX IF NOT EXISTS "+index+" ON "+table+" ("+column+")")
|
||||
return err
|
||||
}
|
||||
|
||||
func (sqliteDialect) likeContains(col, substr string) (string, string) {
|
||||
return col + ` LIKE ? ESCAPE '\'`, "%" + escapeLike(substr) + "%"
|
||||
}
|
||||
|
||||
type mysqlDialect struct{}
|
||||
|
||||
func (mysqlDialect) name() string { return "mysql" }
|
||||
func (mysqlDialect) datetimeType() string { return "DATETIME(6)" }
|
||||
|
||||
func (mysqlDialect) upsert(insert string, cols []string) string {
|
||||
sets := make([]string, len(cols))
|
||||
for i, c := range cols {
|
||||
sets[i] = c + "=VALUES(" + c + ")"
|
||||
}
|
||||
return insert + " ON DUPLICATE KEY UPDATE " + strings.Join(sets, ", ")
|
||||
}
|
||||
|
||||
func (mysqlDialect) createIndex(ctx context.Context, db *sql.DB, index, table, column string) error {
|
||||
var n int
|
||||
err := db.QueryRowContext(ctx,
|
||||
`SELECT COUNT(*) FROM information_schema.statistics
|
||||
WHERE table_schema = DATABASE() AND table_name = ? AND index_name = ?`,
|
||||
table, index).Scan(&n)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n > 0 {
|
||||
return nil
|
||||
}
|
||||
_, err = db.ExecContext(ctx, "CREATE INDEX "+index+" ON "+table+" ("+column+")")
|
||||
return err
|
||||
}
|
||||
|
||||
func (mysqlDialect) likeContains(col, substr string) (string, string) {
|
||||
return col + ` LIKE ? ESCAPE '\\'`, "%" + escapeLike(substr) + "%"
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"time"
|
||||
)
|
||||
|
||||
// StartEvent and CompleteEvent are the receiver-side decodings of the
|
||||
// webhook payloads posted by the main app. Field names and JSON tags must
|
||||
// stay in sync with webhookStartEvent/webhookCompleteEvent in
|
||||
// web/api/webhook.go; a compatibility test in web/api enforces this.
|
||||
|
||||
// StartEvent mirrors web/api webhookStartEvent.
|
||||
type StartEvent struct {
|
||||
Event string `json:"event"`
|
||||
ID string `json:"id"`
|
||||
Domain string `json:"domain"`
|
||||
QueryType string `json:"query_type"`
|
||||
AllRoots bool `json:"all_roots"`
|
||||
ClientIP string `json:"client_ip"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
}
|
||||
|
||||
// CompleteEvent mirrors web/api webhookCompleteEvent. Summary is kept as
|
||||
// raw JSON: the receiver stores it verbatim and never interprets it.
|
||||
type CompleteEvent struct {
|
||||
Event string `json:"event"`
|
||||
ID string `json:"id"`
|
||||
Domain string `json:"domain"`
|
||||
QueryType string `json:"query_type"`
|
||||
ClientIP string `json:"client_ip"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
DoneAt time.Time `json:"done_at"`
|
||||
DurationMS int64 `json:"duration_ms"`
|
||||
Status string `json:"status"`
|
||||
Error string `json:"error,omitempty"`
|
||||
ResultCount int `json:"result_count"`
|
||||
Summary json.RawMessage `json:"summary"`
|
||||
}
|
||||
@@ -0,0 +1,322 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"math"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Traversal is one stored traversal row. Pointer fields are NULL until the
|
||||
// complete event arrives.
|
||||
type Traversal struct {
|
||||
ID string
|
||||
Domain string
|
||||
QueryType string
|
||||
AllRoots bool
|
||||
ClientIP string
|
||||
StartedAt time.Time
|
||||
DoneAt *time.Time
|
||||
DurationMS *int64
|
||||
Status string
|
||||
Error string
|
||||
ResultCount *int
|
||||
Summary string
|
||||
FirstSeen time.Time
|
||||
LastSeen time.Time
|
||||
}
|
||||
|
||||
// ListFilter narrows ListTraversals. Zero values mean "no constraint";
|
||||
// From/To bound started_at inclusively.
|
||||
type ListFilter struct {
|
||||
Domain string // substring match
|
||||
Status string
|
||||
From time.Time
|
||||
To time.Time
|
||||
}
|
||||
|
||||
const defaultListLimit = 50
|
||||
|
||||
// where renders the filter as a WHERE clause (or "") plus bind args.
|
||||
func (f ListFilter) where(d dialect) (string, []any) {
|
||||
var preds []string
|
||||
var args []any
|
||||
if f.Domain != "" {
|
||||
p, arg := d.likeContains("domain", f.Domain)
|
||||
preds = append(preds, p)
|
||||
args = append(args, arg)
|
||||
}
|
||||
if f.Status != "" {
|
||||
preds = append(preds, "status = ?")
|
||||
args = append(args, f.Status)
|
||||
}
|
||||
if !f.From.IsZero() {
|
||||
preds = append(preds, "started_at >= ?")
|
||||
args = append(args, fmtTime(f.From))
|
||||
}
|
||||
if !f.To.IsZero() {
|
||||
preds = append(preds, "started_at <= ?")
|
||||
args = append(args, fmtTime(f.To))
|
||||
}
|
||||
if len(preds) == 0 {
|
||||
return "", nil
|
||||
}
|
||||
return " WHERE " + strings.Join(preds, " AND "), args
|
||||
}
|
||||
|
||||
// ListTraversals returns one page of matching traversals ordered newest
|
||||
// first, plus the total match count. limit <= 0 selects a default page size.
|
||||
func (s *Store) ListTraversals(ctx context.Context, f ListFilter, limit, offset int) ([]Traversal, int, error) {
|
||||
where, args := f.where(s.d)
|
||||
|
||||
var total int
|
||||
if err := s.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM traversals"+where, args...).Scan(&total); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
if limit <= 0 {
|
||||
limit = defaultListLimit
|
||||
}
|
||||
if offset < 0 {
|
||||
offset = 0
|
||||
}
|
||||
q := `SELECT id, domain, query_type, all_roots, client_ip, started_at, done_at,
|
||||
duration_ms, status, error, result_count, summary, first_seen, last_seen
|
||||
FROM traversals` + where + ` ORDER BY started_at DESC, id LIMIT ? OFFSET ?`
|
||||
rows, err := s.db.QueryContext(ctx, q, append(args, limit, offset)...)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []Traversal
|
||||
for rows.Next() {
|
||||
var (
|
||||
tr Traversal
|
||||
startedAt, doneAt sql.NullString
|
||||
errMsg, summary sql.NullString
|
||||
durationMS sql.NullInt64
|
||||
resultCount sql.NullInt64
|
||||
firstSeen, lastSeen string
|
||||
)
|
||||
if err := rows.Scan(&tr.ID, &tr.Domain, &tr.QueryType, &tr.AllRoots, &tr.ClientIP,
|
||||
&startedAt, &doneAt, &durationMS, &tr.Status, &errMsg, &resultCount,
|
||||
&summary, &firstSeen, &lastSeen); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
tr.Error = errMsg.String
|
||||
tr.Summary = summary.String
|
||||
if durationMS.Valid {
|
||||
v := durationMS.Int64
|
||||
tr.DurationMS = &v
|
||||
}
|
||||
if resultCount.Valid {
|
||||
v := int(resultCount.Int64)
|
||||
tr.ResultCount = &v
|
||||
}
|
||||
if startedAt.Valid {
|
||||
if tr.StartedAt, err = parseDBTime(startedAt.String); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
}
|
||||
if doneAt.Valid {
|
||||
t, err := parseDBTime(doneAt.String)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
tr.DoneAt = &t
|
||||
}
|
||||
if tr.FirstSeen, err = parseDBTime(firstSeen); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
if tr.LastSeen, err = parseDBTime(lastSeen); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
out = append(out, tr)
|
||||
}
|
||||
return out, total, rows.Err()
|
||||
}
|
||||
|
||||
// Totals summarises whole-table traversal volume. Errors counts all-time
|
||||
// error-status rows so callers can derive an error rate.
|
||||
type Totals struct {
|
||||
AllTime int
|
||||
Last24h int
|
||||
Last7d int
|
||||
DistinctDomains int
|
||||
DistinctClients int
|
||||
Errors int
|
||||
}
|
||||
|
||||
// Totals returns whole-table totals with the recent windows measured back
|
||||
// from now. COALESCE covers the empty table, where SUM yields NULL.
|
||||
func (s *Store) Totals(ctx context.Context, now time.Time) (Totals, error) {
|
||||
var t Totals
|
||||
err := s.db.QueryRowContext(ctx, `SELECT COUNT(*),
|
||||
COALESCE(SUM(CASE WHEN started_at >= ? THEN 1 ELSE 0 END), 0),
|
||||
COALESCE(SUM(CASE WHEN started_at >= ? THEN 1 ELSE 0 END), 0),
|
||||
COUNT(DISTINCT domain),
|
||||
COUNT(DISTINCT client_ip),
|
||||
COALESCE(SUM(CASE WHEN status = ? THEN 1 ELSE 0 END), 0)
|
||||
FROM traversals`,
|
||||
fmtTime(now.Add(-24*time.Hour)), fmtTime(now.AddDate(0, 0, -7)), StatusError).
|
||||
Scan(&t.AllTime, &t.Last24h, &t.Last7d, &t.DistinctDomains, &t.DistinctClients, &t.Errors)
|
||||
return t, err
|
||||
}
|
||||
|
||||
// DayStats is one calendar day's traversal volume.
|
||||
type DayStats struct {
|
||||
Day string // "2006-01-02" (UTC)
|
||||
Total int
|
||||
Errors int
|
||||
}
|
||||
|
||||
// StatsPerDay returns per-day totals and error counts for the last days
|
||||
// calendar days (UTC), oldest first. Days with no traffic are omitted.
|
||||
func (s *Store) StatsPerDay(ctx context.Context, days int) ([]DayStats, error) {
|
||||
if days <= 0 {
|
||||
days = 1
|
||||
}
|
||||
midnight := time.Now().UTC().Truncate(24 * time.Hour)
|
||||
cutoff := midnight.AddDate(0, 0, -(days - 1))
|
||||
// substr on the stored value yields "YYYY-MM-DD" in both backends
|
||||
// (MySQL casts DATETIME to its string form implicitly).
|
||||
rows, err := s.db.QueryContext(ctx, `SELECT substr(started_at, 1, 10) AS day,
|
||||
COUNT(*), SUM(CASE WHEN status = ? THEN 1 ELSE 0 END)
|
||||
FROM traversals WHERE started_at >= ? GROUP BY day ORDER BY day`,
|
||||
StatusError, fmtTime(cutoff))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []DayStats
|
||||
for rows.Next() {
|
||||
var d DayStats
|
||||
if err := rows.Scan(&d.Day, &d.Total, &d.Errors); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, d)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// NameCount is a generic (value, count) aggregate row.
|
||||
type NameCount struct {
|
||||
Name string
|
||||
Count int
|
||||
}
|
||||
|
||||
// TopDomains returns the most-queried domains since the given time (zero =
|
||||
// all time), most frequent first, at most limit rows.
|
||||
func (s *Store) TopDomains(ctx context.Context, since time.Time, limit int) ([]NameCount, error) {
|
||||
return s.countBy(ctx, "domain", since, limit)
|
||||
}
|
||||
|
||||
// TopClientIPs returns the most active client IPs since the given time.
|
||||
func (s *Store) TopClientIPs(ctx context.Context, since time.Time, limit int) ([]NameCount, error) {
|
||||
return s.countBy(ctx, "client_ip", since, limit)
|
||||
}
|
||||
|
||||
// QueryTypeCounts returns traversal counts per query type since the given
|
||||
// time, most frequent first.
|
||||
func (s *Store) QueryTypeCounts(ctx context.Context, since time.Time) ([]NameCount, error) {
|
||||
return s.countBy(ctx, "query_type", since, 0)
|
||||
}
|
||||
|
||||
// StatusCounts returns traversal counts per status since the given time,
|
||||
// most frequent first.
|
||||
func (s *Store) StatusCounts(ctx context.Context, since time.Time) ([]NameCount, error) {
|
||||
return s.countBy(ctx, "status", since, 0)
|
||||
}
|
||||
|
||||
// countBy groups rows by col and counts them. col is always one of the
|
||||
// fixed column names above, never user input.
|
||||
func (s *Store) countBy(ctx context.Context, col string, since time.Time, limit int) ([]NameCount, error) {
|
||||
q := "SELECT " + col + ", COUNT(*) AS n FROM traversals"
|
||||
var args []any
|
||||
if !since.IsZero() {
|
||||
q += " WHERE started_at >= ?"
|
||||
args = append(args, fmtTime(since))
|
||||
}
|
||||
q += " GROUP BY " + col + " ORDER BY n DESC, " + col
|
||||
if limit > 0 {
|
||||
q += " LIMIT ?"
|
||||
args = append(args, limit)
|
||||
}
|
||||
rows, err := s.db.QueryContext(ctx, q, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []NameCount
|
||||
for rows.Next() {
|
||||
var nc NameCount
|
||||
if err := rows.Scan(&nc.Name, &nc.Count); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, nc)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// DurationStats summarises completed-traversal durations.
|
||||
type DurationStats struct {
|
||||
Count int
|
||||
AvgMS float64
|
||||
P50MS int64
|
||||
P95MS int64
|
||||
}
|
||||
|
||||
// Durations returns duration statistics for traversals started since the
|
||||
// given time (zero = all time). Percentiles are computed in Go (nearest
|
||||
// rank) because SQL percentile support differs between the backends.
|
||||
func (s *Store) Durations(ctx context.Context, since time.Time) (DurationStats, error) {
|
||||
q := "SELECT duration_ms FROM traversals WHERE duration_ms IS NOT NULL"
|
||||
var args []any
|
||||
if !since.IsZero() {
|
||||
q += " AND started_at >= ?"
|
||||
args = append(args, fmtTime(since))
|
||||
}
|
||||
rows, err := s.db.QueryContext(ctx, q, args...)
|
||||
if err != nil {
|
||||
return DurationStats{}, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var values []int64
|
||||
var sum int64
|
||||
for rows.Next() {
|
||||
var v int64
|
||||
if err := rows.Scan(&v); err != nil {
|
||||
return DurationStats{}, err
|
||||
}
|
||||
values = append(values, v)
|
||||
sum += v
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return DurationStats{}, err
|
||||
}
|
||||
if len(values) == 0 {
|
||||
return DurationStats{}, nil
|
||||
}
|
||||
sort.Slice(values, func(i, j int) bool { return values[i] < values[j] })
|
||||
return DurationStats{
|
||||
Count: len(values),
|
||||
AvgMS: float64(sum) / float64(len(values)),
|
||||
P50MS: percentile(values, 0.50),
|
||||
P95MS: percentile(values, 0.95),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// percentile returns the nearest-rank percentile of sorted values.
|
||||
func percentile(sorted []int64, q float64) int64 {
|
||||
rank := int(math.Ceil(q * float64(len(sorted))))
|
||||
if rank < 1 {
|
||||
rank = 1
|
||||
}
|
||||
return sorted[rank-1]
|
||||
}
|
||||
@@ -0,0 +1,217 @@
|
||||
// Package store persists webhook telemetry events from the ExploreDNS API
|
||||
// server into MySQL or SQLite through a shared database/sql layer.
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/go-sql-driver/mysql"
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
// Traversal status values, mirroring the job statuses in web/api.
|
||||
const (
|
||||
StatusRunning = "running"
|
||||
StatusComplete = "complete"
|
||||
StatusError = "error"
|
||||
)
|
||||
|
||||
// schemaVersion is recorded in the schema_version table on first open so
|
||||
// future releases can detect and migrate older layouts.
|
||||
const schemaVersion = 1
|
||||
|
||||
// dbTimeLayout is the canonical datetime encoding: fixed-width so sqlite
|
||||
// TEXT comparisons order chronologically, and a valid MySQL DATETIME(6)
|
||||
// literal.
|
||||
const dbTimeLayout = "2006-01-02 15:04:05.000000"
|
||||
|
||||
// Store persists traversal telemetry events.
|
||||
type Store struct {
|
||||
db *sql.DB
|
||||
d dialect
|
||||
}
|
||||
|
||||
// OpenSQLite opens (creating if needed) a SQLite-backed store at path,
|
||||
// creating parent directories first.
|
||||
func OpenSQLite(path string) (*Store, error) {
|
||||
if dir := filepath.Dir(path); dir != "." && dir != "" {
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return nil, fmt.Errorf("create sqlite directory: %w", err)
|
||||
}
|
||||
}
|
||||
db, err := sql.Open("sqlite", path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open sqlite %s: %w", path, err)
|
||||
}
|
||||
// A single connection serialises writers so concurrent ingests never
|
||||
// see SQLITE_BUSY.
|
||||
db.SetMaxOpenConns(1)
|
||||
return open(db, sqliteDialect{})
|
||||
}
|
||||
|
||||
// OpenMySQL opens a MySQL-backed store using a go-sql-driver DSN
|
||||
// (user:pass@tcp(host:port)/dbname).
|
||||
func OpenMySQL(dsn string) (*Store, error) {
|
||||
if _, err := mysql.ParseDSN(dsn); err != nil {
|
||||
return nil, fmt.Errorf("parse mysql dsn: %w", err)
|
||||
}
|
||||
db, err := sql.Open("mysql", dsn)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open mysql: %w", err)
|
||||
}
|
||||
return open(db, mysqlDialect{})
|
||||
}
|
||||
|
||||
// RedactMySQLDSN returns dsn with any password replaced, safe for logging.
|
||||
func RedactMySQLDSN(dsn string) string {
|
||||
cfg, err := mysql.ParseDSN(dsn)
|
||||
if err != nil {
|
||||
return "(unparsable DSN)"
|
||||
}
|
||||
if cfg.Passwd != "" {
|
||||
cfg.Passwd = "xxxxx"
|
||||
}
|
||||
return cfg.FormatDSN()
|
||||
}
|
||||
|
||||
func open(db *sql.DB, d dialect) (*Store, error) {
|
||||
s := &Store{db: db, d: d}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
if err := s.init(ctx); err != nil {
|
||||
db.Close()
|
||||
return nil, fmt.Errorf("init %s schema: %w", d.name(), err)
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// Backend names the database backend ("sqlite" or "mysql").
|
||||
func (s *Store) Backend() string { return s.d.name() }
|
||||
|
||||
// Close releases the underlying database handle.
|
||||
func (s *Store) Close() error { return s.db.Close() }
|
||||
|
||||
// init creates the schema if missing and stamps schema_version.
|
||||
func (s *Store) init(ctx context.Context) error {
|
||||
dt := s.d.datetimeType()
|
||||
stmts := []string{
|
||||
`CREATE TABLE IF NOT EXISTS schema_version (version INTEGER NOT NULL)`,
|
||||
fmt.Sprintf(`CREATE TABLE IF NOT EXISTS traversals (
|
||||
id VARCHAR(64) NOT NULL PRIMARY KEY,
|
||||
domain VARCHAR(255) NOT NULL DEFAULT '',
|
||||
query_type VARCHAR(16) NOT NULL DEFAULT '',
|
||||
all_roots BOOLEAN NOT NULL DEFAULT 0,
|
||||
client_ip VARCHAR(64) NOT NULL DEFAULT '',
|
||||
started_at %s NULL,
|
||||
done_at %s NULL,
|
||||
duration_ms BIGINT NULL,
|
||||
status VARCHAR(16) NOT NULL DEFAULT '',
|
||||
error TEXT NULL,
|
||||
result_count INT NULL,
|
||||
summary TEXT NULL,
|
||||
first_seen %s NOT NULL,
|
||||
last_seen %s NOT NULL
|
||||
)`, dt, dt, dt, dt),
|
||||
}
|
||||
for _, q := range stmts {
|
||||
if _, err := s.db.ExecContext(ctx, q); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, idx := range [][2]string{
|
||||
{"idx_traversals_started_at", "started_at"},
|
||||
{"idx_traversals_domain", "domain"},
|
||||
{"idx_traversals_client_ip", "client_ip"},
|
||||
{"idx_traversals_status", "status"},
|
||||
} {
|
||||
if err := s.d.createIndex(ctx, s.db, idx[0], "traversals", idx[1]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// Stamp the version on first creation only. Checked in Go because
|
||||
// MySQL and sqlite disagree on FROM-less SELECT ... WHERE syntax.
|
||||
var n int
|
||||
if err := s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM schema_version`).Scan(&n); err != nil {
|
||||
return err
|
||||
}
|
||||
if n == 0 {
|
||||
_, err := s.db.ExecContext(ctx, `INSERT INTO schema_version (version) VALUES (?)`, schemaVersion)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RecordStart upserts a "start" event. Duplicate deliveries refresh the
|
||||
// request attributes but never touch status or completion fields, so a
|
||||
// retried start arriving after the complete event cannot clobber the
|
||||
// terminal state.
|
||||
func (s *Store) RecordStart(ctx context.Context, ev StartEvent) error {
|
||||
now := fmtTime(time.Now())
|
||||
q := s.d.upsert(`INSERT INTO traversals
|
||||
(id, domain, query_type, all_roots, client_ip, started_at, status, first_seen, last_seen)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
[]string{"domain", "query_type", "all_roots", "client_ip", "started_at", "last_seen"})
|
||||
_, err := s.db.ExecContext(ctx, q,
|
||||
ev.ID, ev.Domain, ev.QueryType, ev.AllRoots, ev.ClientIP,
|
||||
dbTime(ev.StartedAt), StatusRunning, now, now)
|
||||
return err
|
||||
}
|
||||
|
||||
// RecordComplete upserts a "complete" event. It creates the row when the
|
||||
// start event has not arrived (out-of-order delivery) and overwrites the
|
||||
// completion fields on duplicates; all_roots and first_seen are start-only
|
||||
// and left untouched.
|
||||
func (s *Store) RecordComplete(ctx context.Context, ev CompleteEvent) error {
|
||||
now := fmtTime(time.Now())
|
||||
q := s.d.upsert(`INSERT INTO traversals
|
||||
(id, domain, query_type, client_ip, started_at, done_at, duration_ms,
|
||||
status, error, result_count, summary, first_seen, last_seen)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
[]string{"domain", "query_type", "client_ip", "started_at", "done_at",
|
||||
"duration_ms", "status", "error", "result_count", "summary", "last_seen"})
|
||||
var summary any
|
||||
if len(ev.Summary) > 0 {
|
||||
summary = string(ev.Summary)
|
||||
}
|
||||
_, err := s.db.ExecContext(ctx, q,
|
||||
ev.ID, ev.Domain, ev.QueryType, ev.ClientIP,
|
||||
dbTime(ev.StartedAt), dbTime(ev.DoneAt), ev.DurationMS,
|
||||
ev.Status, nullIfEmpty(ev.Error), ev.ResultCount, summary, now, now)
|
||||
return err
|
||||
}
|
||||
|
||||
// fmtTime encodes t for storage.
|
||||
func fmtTime(t time.Time) string { return t.UTC().Format(dbTimeLayout) }
|
||||
|
||||
// dbTime encodes t for a nullable datetime column; zero times become NULL
|
||||
// (MySQL DATETIME cannot hold year 1).
|
||||
func dbTime(t time.Time) any {
|
||||
if t.IsZero() {
|
||||
return nil
|
||||
}
|
||||
return fmtTime(t)
|
||||
}
|
||||
|
||||
func nullIfEmpty(s string) any {
|
||||
if s == "" {
|
||||
return nil
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// parseDBTime decodes a stored datetime. Besides the canonical layout it
|
||||
// accepts second precision and RFC 3339 in case the MySQL DSN enables
|
||||
// parseTime (database/sql then hands strings back in RFC 3339).
|
||||
func parseDBTime(s string) (time.Time, error) {
|
||||
for _, layout := range []string{dbTimeLayout, "2006-01-02 15:04:05", time.RFC3339Nano} {
|
||||
if t, err := time.ParseInLocation(layout, s, time.UTC); err == nil {
|
||||
return t, nil
|
||||
}
|
||||
}
|
||||
return time.Time{}, fmt.Errorf("unrecognised datetime %q", s)
|
||||
}
|
||||
@@ -0,0 +1,544 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func newTestStore(t *testing.T) *Store {
|
||||
t.Helper()
|
||||
s, err := OpenSQLite(filepath.Join(t.TempDir(), "receiver.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("OpenSQLite: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { s.Close() })
|
||||
return s
|
||||
}
|
||||
|
||||
func startEvent(id string) StartEvent {
|
||||
return StartEvent{
|
||||
Event: "start",
|
||||
ID: id,
|
||||
Domain: "example.com",
|
||||
QueryType: "A",
|
||||
AllRoots: true,
|
||||
ClientIP: "203.0.113.9",
|
||||
StartedAt: time.Date(2026, 7, 1, 10, 0, 0, 0, time.UTC),
|
||||
}
|
||||
}
|
||||
|
||||
func completeEvent(id string) CompleteEvent {
|
||||
return CompleteEvent{
|
||||
Event: "complete",
|
||||
ID: id,
|
||||
Domain: "example.com",
|
||||
QueryType: "A",
|
||||
ClientIP: "203.0.113.9",
|
||||
StartedAt: time.Date(2026, 7, 1, 10, 0, 0, 0, time.UTC),
|
||||
DoneAt: time.Date(2026, 7, 1, 10, 0, 3, 0, time.UTC),
|
||||
DurationMS: 3000,
|
||||
Status: StatusComplete,
|
||||
ResultCount: 4,
|
||||
Summary: json.RawMessage(`{"answers":[{"probability":1,"records":["example.com 300 IN A 192.0.2.1"]}]}`),
|
||||
}
|
||||
}
|
||||
|
||||
// getRow fetches a single traversal by exact id via ListTraversals.
|
||||
func getRow(t *testing.T, s *Store, id string) Traversal {
|
||||
t.Helper()
|
||||
rows, _, err := s.ListTraversals(context.Background(), ListFilter{}, 1000, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("ListTraversals: %v", err)
|
||||
}
|
||||
for _, r := range rows {
|
||||
if r.ID == id {
|
||||
return r
|
||||
}
|
||||
}
|
||||
t.Fatalf("row %q not found", id)
|
||||
return Traversal{}
|
||||
}
|
||||
|
||||
func TestRecordStartThenComplete(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
ctx := context.Background()
|
||||
|
||||
if err := s.RecordStart(ctx, startEvent("t1")); err != nil {
|
||||
t.Fatalf("RecordStart: %v", err)
|
||||
}
|
||||
row := getRow(t, s, "t1")
|
||||
if row.Status != StatusRunning {
|
||||
t.Errorf("Status = %q, want %q", row.Status, StatusRunning)
|
||||
}
|
||||
if !row.AllRoots {
|
||||
t.Error("AllRoots = false, want true")
|
||||
}
|
||||
if row.DoneAt != nil || row.DurationMS != nil || row.ResultCount != nil {
|
||||
t.Errorf("completion fields set before complete: %+v", row)
|
||||
}
|
||||
if !row.StartedAt.Equal(startEvent("t1").StartedAt) {
|
||||
t.Errorf("StartedAt = %v, want %v", row.StartedAt, startEvent("t1").StartedAt)
|
||||
}
|
||||
|
||||
ev := completeEvent("t1")
|
||||
if err := s.RecordComplete(ctx, ev); err != nil {
|
||||
t.Fatalf("RecordComplete: %v", err)
|
||||
}
|
||||
row = getRow(t, s, "t1")
|
||||
if row.Status != StatusComplete {
|
||||
t.Errorf("Status = %q, want %q", row.Status, StatusComplete)
|
||||
}
|
||||
if row.DoneAt == nil || !row.DoneAt.Equal(ev.DoneAt) {
|
||||
t.Errorf("DoneAt = %v, want %v", row.DoneAt, ev.DoneAt)
|
||||
}
|
||||
if row.DurationMS == nil || *row.DurationMS != 3000 {
|
||||
t.Errorf("DurationMS = %v, want 3000", row.DurationMS)
|
||||
}
|
||||
if row.ResultCount == nil || *row.ResultCount != 4 {
|
||||
t.Errorf("ResultCount = %v, want 4", row.ResultCount)
|
||||
}
|
||||
if row.Summary != string(ev.Summary) {
|
||||
t.Errorf("Summary = %q, want %q", row.Summary, ev.Summary)
|
||||
}
|
||||
if !row.AllRoots {
|
||||
t.Error("AllRoots clobbered by complete event")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompleteBeforeStart(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
ctx := context.Background()
|
||||
|
||||
if err := s.RecordComplete(ctx, completeEvent("t1")); err != nil {
|
||||
t.Fatalf("RecordComplete: %v", err)
|
||||
}
|
||||
row := getRow(t, s, "t1")
|
||||
if row.Status != StatusComplete {
|
||||
t.Fatalf("Status = %q, want %q", row.Status, StatusComplete)
|
||||
}
|
||||
|
||||
// The delayed start must fill in start-only fields without touching
|
||||
// the terminal state.
|
||||
if err := s.RecordStart(ctx, startEvent("t1")); err != nil {
|
||||
t.Fatalf("RecordStart: %v", err)
|
||||
}
|
||||
row = getRow(t, s, "t1")
|
||||
if row.Status != StatusComplete {
|
||||
t.Errorf("Status = %q after late start, want %q", row.Status, StatusComplete)
|
||||
}
|
||||
if row.DoneAt == nil || row.DurationMS == nil {
|
||||
t.Errorf("completion fields lost after late start: %+v", row)
|
||||
}
|
||||
if !row.AllRoots {
|
||||
t.Error("AllRoots not filled in by late start")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDuplicateStartAfterComplete(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
ctx := context.Background()
|
||||
|
||||
if err := s.RecordStart(ctx, startEvent("t1")); err != nil {
|
||||
t.Fatalf("RecordStart: %v", err)
|
||||
}
|
||||
if err := s.RecordComplete(ctx, completeEvent("t1")); err != nil {
|
||||
t.Fatalf("RecordComplete: %v", err)
|
||||
}
|
||||
if err := s.RecordStart(ctx, startEvent("t1")); err != nil {
|
||||
t.Fatalf("duplicate RecordStart: %v", err)
|
||||
}
|
||||
|
||||
row := getRow(t, s, "t1")
|
||||
if row.Status != StatusComplete {
|
||||
t.Errorf("Status = %q after duplicate start, want %q", row.Status, StatusComplete)
|
||||
}
|
||||
if row.DoneAt == nil {
|
||||
t.Error("DoneAt lost after duplicate start")
|
||||
}
|
||||
if row.Summary == "" {
|
||||
t.Error("Summary lost after duplicate start")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDuplicateCompleteIdempotent(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
ctx := context.Background()
|
||||
|
||||
ev := completeEvent("t1")
|
||||
ev.Status = StatusError
|
||||
ev.Error = "traversal timed out after 5m0s"
|
||||
for i := 0; i < 3; i++ {
|
||||
if err := s.RecordComplete(ctx, ev); err != nil {
|
||||
t.Fatalf("RecordComplete #%d: %v", i, err)
|
||||
}
|
||||
}
|
||||
|
||||
rows, total, err := s.ListTraversals(ctx, ListFilter{}, 10, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("ListTraversals: %v", err)
|
||||
}
|
||||
if total != 1 || len(rows) != 1 {
|
||||
t.Fatalf("total = %d, len = %d, want 1 row", total, len(rows))
|
||||
}
|
||||
if rows[0].Status != StatusError || rows[0].Error != ev.Error {
|
||||
t.Errorf("row = %q/%q, want %q/%q", rows[0].Status, rows[0].Error, StatusError, ev.Error)
|
||||
}
|
||||
}
|
||||
|
||||
// seedRows inserts a deterministic mixed dataset for list/aggregate tests.
|
||||
func seedRows(t *testing.T, s *Store) {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
base := time.Date(2026, 7, 1, 0, 0, 0, 0, time.UTC)
|
||||
rows := []struct {
|
||||
id, domain, qtype, ip, status string
|
||||
day int
|
||||
durMS int64
|
||||
}{
|
||||
{"a1", "example.com", "A", "203.0.113.1", StatusComplete, 0, 100},
|
||||
{"a2", "example.com", "AAAA", "203.0.113.1", StatusComplete, 0, 200},
|
||||
{"a3", "sub.example.com", "A", "203.0.113.2", StatusError, 1, 300},
|
||||
{"a4", "other.net", "MX", "203.0.113.3", StatusComplete, 1, 400},
|
||||
{"a5", "other.net", "A", "203.0.113.1", StatusComplete, 2, 500},
|
||||
{"a6", "under_score.org", "A", "203.0.113.4", StatusComplete, 2, 600},
|
||||
}
|
||||
for i, r := range rows {
|
||||
started := base.AddDate(0, 0, r.day).Add(time.Duration(i) * time.Minute)
|
||||
ev := CompleteEvent{
|
||||
ID: r.id, Domain: r.domain, QueryType: r.qtype, ClientIP: r.ip,
|
||||
StartedAt: started, DoneAt: started.Add(time.Duration(r.durMS) * time.Millisecond),
|
||||
DurationMS: r.durMS, Status: r.status, ResultCount: 1,
|
||||
}
|
||||
if r.status == StatusError {
|
||||
ev.Error = "lookup failed"
|
||||
}
|
||||
if err := s.RecordComplete(ctx, ev); err != nil {
|
||||
t.Fatalf("seed %s: %v", r.id, err)
|
||||
}
|
||||
}
|
||||
// One still-running traversal on day 2.
|
||||
if err := s.RecordStart(ctx, StartEvent{
|
||||
ID: "a7", Domain: "running.io", QueryType: "A", ClientIP: "203.0.113.5",
|
||||
StartedAt: base.AddDate(0, 0, 2).Add(time.Hour),
|
||||
}); err != nil {
|
||||
t.Fatalf("seed a7: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListTraversalsFiltersAndPagination(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
seedRows(t, s)
|
||||
ctx := context.Background()
|
||||
|
||||
rows, total, err := s.ListTraversals(ctx, ListFilter{}, 3, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("ListTraversals: %v", err)
|
||||
}
|
||||
if total != 7 || len(rows) != 3 {
|
||||
t.Fatalf("total = %d, len = %d, want 7 and 3", total, len(rows))
|
||||
}
|
||||
// Newest first: a7 (day2+1h) then a6, a5.
|
||||
if rows[0].ID != "a7" || rows[1].ID != "a6" || rows[2].ID != "a5" {
|
||||
t.Errorf("page 1 order = %s,%s,%s, want a7,a6,a5", rows[0].ID, rows[1].ID, rows[2].ID)
|
||||
}
|
||||
rows, _, err = s.ListTraversals(ctx, ListFilter{}, 3, 3)
|
||||
if err != nil {
|
||||
t.Fatalf("ListTraversals page 2: %v", err)
|
||||
}
|
||||
if rows[0].ID != "a4" || rows[1].ID != "a3" || rows[2].ID != "a2" {
|
||||
t.Errorf("page 2 order = %s,%s,%s, want a4,a3,a2", rows[0].ID, rows[1].ID, rows[2].ID)
|
||||
}
|
||||
|
||||
// Domain substring matches example.com and sub.example.com.
|
||||
rows, total, err = s.ListTraversals(ctx, ListFilter{Domain: "example"}, 0, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("domain filter: %v", err)
|
||||
}
|
||||
if total != 3 {
|
||||
t.Errorf("domain filter total = %d, want 3", total)
|
||||
}
|
||||
for _, r := range rows {
|
||||
if r.Domain != "example.com" && r.Domain != "sub.example.com" {
|
||||
t.Errorf("domain filter matched %q", r.Domain)
|
||||
}
|
||||
}
|
||||
|
||||
// LIKE metacharacters in the filter must be literal, not wildcards.
|
||||
_, total, err = s.ListTraversals(ctx, ListFilter{Domain: "%"}, 0, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("percent filter: %v", err)
|
||||
}
|
||||
if total != 0 {
|
||||
t.Errorf("%% filter total = %d, want 0", total)
|
||||
}
|
||||
_, total, err = s.ListTraversals(ctx, ListFilter{Domain: "under_score"}, 0, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("underscore filter: %v", err)
|
||||
}
|
||||
if total != 1 {
|
||||
t.Errorf("under_score filter total = %d, want 1", total)
|
||||
}
|
||||
|
||||
_, total, err = s.ListTraversals(ctx, ListFilter{Status: StatusError}, 0, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("status filter: %v", err)
|
||||
}
|
||||
if total != 1 {
|
||||
t.Errorf("status filter total = %d, want 1", total)
|
||||
}
|
||||
|
||||
base := time.Date(2026, 7, 1, 0, 0, 0, 0, time.UTC)
|
||||
rows, total, err = s.ListTraversals(ctx, ListFilter{
|
||||
From: base.AddDate(0, 0, 1),
|
||||
To: base.AddDate(0, 0, 2).Add(-time.Second),
|
||||
}, 0, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("time filter: %v", err)
|
||||
}
|
||||
if total != 2 {
|
||||
t.Fatalf("time filter total = %d, want 2 (got %+v)", total, rows)
|
||||
}
|
||||
if rows[0].ID != "a4" || rows[1].ID != "a3" {
|
||||
t.Errorf("time filter = %s,%s, want a4,a3", rows[0].ID, rows[1].ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatsPerDay(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC()
|
||||
for i, spec := range []struct {
|
||||
daysAgo int
|
||||
status string
|
||||
}{
|
||||
{0, StatusComplete}, {0, StatusError}, {1, StatusComplete}, {5, StatusComplete},
|
||||
} {
|
||||
ev := completeEvent(string(rune('a' + i)))
|
||||
ev.StartedAt = now.AddDate(0, 0, -spec.daysAgo)
|
||||
ev.DoneAt = ev.StartedAt.Add(time.Second)
|
||||
ev.Status = spec.status
|
||||
if err := s.RecordComplete(ctx, ev); err != nil {
|
||||
t.Fatalf("seed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
days, err := s.StatsPerDay(ctx, 3)
|
||||
if err != nil {
|
||||
t.Fatalf("StatsPerDay: %v", err)
|
||||
}
|
||||
if len(days) != 2 {
|
||||
t.Fatalf("len = %d, want 2 (%+v)", len(days), days)
|
||||
}
|
||||
yesterday := now.AddDate(0, 0, -1).Format("2006-01-02")
|
||||
today := now.Format("2006-01-02")
|
||||
if days[0].Day != yesterday || days[0].Total != 1 || days[0].Errors != 0 {
|
||||
t.Errorf("day[0] = %+v, want %s total 1 errors 0", days[0], yesterday)
|
||||
}
|
||||
if days[1].Day != today || days[1].Total != 2 || days[1].Errors != 1 {
|
||||
t.Errorf("day[1] = %+v, want %s total 2 errors 1", days[1], today)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTopDomains(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
seedRows(t, s)
|
||||
|
||||
top, err := s.TopDomains(context.Background(), time.Time{}, 2)
|
||||
if err != nil {
|
||||
t.Fatalf("TopDomains: %v", err)
|
||||
}
|
||||
want := []NameCount{{"example.com", 2}, {"other.net", 2}}
|
||||
if len(top) != 2 || top[0] != want[0] || top[1] != want[1] {
|
||||
t.Errorf("TopDomains = %+v, want %+v", top, want)
|
||||
}
|
||||
|
||||
// Window excludes day 0 rows (a1, a2), so example.com drops out and
|
||||
// other.net (a4, a5) leads.
|
||||
since := time.Date(2026, 7, 2, 0, 0, 0, 0, time.UTC)
|
||||
top, err = s.TopDomains(context.Background(), since, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("TopDomains since: %v", err)
|
||||
}
|
||||
if len(top) != 4 || top[0] != (NameCount{"other.net", 2}) {
|
||||
t.Errorf("TopDomains since = %+v, want other.net x2 leading 4 domains", top)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryTypeCounts(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
seedRows(t, s)
|
||||
|
||||
counts, err := s.QueryTypeCounts(context.Background(), time.Time{})
|
||||
if err != nil {
|
||||
t.Fatalf("QueryTypeCounts: %v", err)
|
||||
}
|
||||
want := []NameCount{{"A", 5}, {"AAAA", 1}, {"MX", 1}}
|
||||
if len(counts) != 3 || counts[0] != want[0] || counts[1] != want[1] || counts[2] != want[2] {
|
||||
t.Errorf("QueryTypeCounts = %+v, want %+v", counts, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatusCounts(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
seedRows(t, s)
|
||||
|
||||
counts, err := s.StatusCounts(context.Background(), time.Time{})
|
||||
if err != nil {
|
||||
t.Fatalf("StatusCounts: %v", err)
|
||||
}
|
||||
want := []NameCount{{StatusComplete, 5}, {StatusError, 1}, {StatusRunning, 1}}
|
||||
if len(counts) != 3 || counts[0] != want[0] || counts[1] != want[1] || counts[2] != want[2] {
|
||||
t.Errorf("StatusCounts = %+v, want %+v", counts, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTopClientIPs(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
seedRows(t, s)
|
||||
|
||||
top, err := s.TopClientIPs(context.Background(), time.Time{}, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("TopClientIPs: %v", err)
|
||||
}
|
||||
if len(top) != 1 || top[0].Name != "203.0.113.1" || top[0].Count != 3 {
|
||||
t.Errorf("TopClientIPs = %+v, want 203.0.113.1 x3", top)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDurations(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
seedRows(t, s)
|
||||
|
||||
// Durations 100..600; the running row has none and is excluded.
|
||||
stats, err := s.Durations(context.Background(), time.Time{})
|
||||
if err != nil {
|
||||
t.Fatalf("Durations: %v", err)
|
||||
}
|
||||
if stats.Count != 6 {
|
||||
t.Fatalf("Count = %d, want 6", stats.Count)
|
||||
}
|
||||
if stats.AvgMS != 350 {
|
||||
t.Errorf("AvgMS = %v, want 350", stats.AvgMS)
|
||||
}
|
||||
if stats.P50MS != 300 {
|
||||
t.Errorf("P50MS = %d, want 300", stats.P50MS)
|
||||
}
|
||||
if stats.P95MS != 600 {
|
||||
t.Errorf("P95MS = %d, want 600", stats.P95MS)
|
||||
}
|
||||
|
||||
// Empty window.
|
||||
stats, err = s.Durations(context.Background(), time.Date(2030, 1, 1, 0, 0, 0, 0, time.UTC))
|
||||
if err != nil {
|
||||
t.Fatalf("Durations empty: %v", err)
|
||||
}
|
||||
if stats.Count != 0 || stats.AvgMS != 0 || stats.P50MS != 0 || stats.P95MS != 0 {
|
||||
t.Errorf("empty Durations = %+v, want zeros", stats)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTotals(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC()
|
||||
|
||||
tot, err := s.Totals(ctx, now)
|
||||
if err != nil {
|
||||
t.Fatalf("Totals empty: %v", err)
|
||||
}
|
||||
if tot != (Totals{}) {
|
||||
t.Errorf("empty Totals = %+v, want zeros", tot)
|
||||
}
|
||||
|
||||
for _, spec := range []struct {
|
||||
id, domain, ip, status string
|
||||
ago time.Duration
|
||||
}{
|
||||
{"t1", "a.com", "203.0.113.1", StatusComplete, time.Hour},
|
||||
{"t2", "a.com", "203.0.113.2", StatusError, 2 * time.Hour},
|
||||
{"t3", "b.net", "203.0.113.1", StatusComplete, 48 * time.Hour},
|
||||
{"t4", "c.org", "203.0.113.3", StatusComplete, 10 * 24 * time.Hour},
|
||||
} {
|
||||
ev := completeEvent(spec.id)
|
||||
ev.Domain = spec.domain
|
||||
ev.ClientIP = spec.ip
|
||||
ev.Status = spec.status
|
||||
ev.StartedAt = now.Add(-spec.ago)
|
||||
ev.DoneAt = ev.StartedAt.Add(time.Second)
|
||||
if err := s.RecordComplete(ctx, ev); err != nil {
|
||||
t.Fatalf("seed %s: %v", spec.id, err)
|
||||
}
|
||||
}
|
||||
|
||||
tot, err = s.Totals(ctx, now)
|
||||
if err != nil {
|
||||
t.Fatalf("Totals: %v", err)
|
||||
}
|
||||
want := Totals{AllTime: 4, Last24h: 2, Last7d: 3, DistinctDomains: 3, DistinctClients: 3, Errors: 1}
|
||||
if tot != want {
|
||||
t.Errorf("Totals = %+v, want %+v", tot, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenSQLiteCreatesParentDirs(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "nested", "dir", "receiver.db")
|
||||
s, err := OpenSQLite(path)
|
||||
if err != nil {
|
||||
t.Fatalf("OpenSQLite: %v", err)
|
||||
}
|
||||
defer s.Close()
|
||||
if s.Backend() != "sqlite" {
|
||||
t.Errorf("Backend = %q, want sqlite", s.Backend())
|
||||
}
|
||||
|
||||
// Reopening must not fail on the existing schema and must keep the
|
||||
// stamped version.
|
||||
s.Close()
|
||||
s2, err := OpenSQLite(path)
|
||||
if err != nil {
|
||||
t.Fatalf("reopen: %v", err)
|
||||
}
|
||||
defer s2.Close()
|
||||
var v int
|
||||
if err := s2.db.QueryRow(`SELECT version FROM schema_version`).Scan(&v); err != nil {
|
||||
t.Fatalf("schema_version: %v", err)
|
||||
}
|
||||
if v != schemaVersion {
|
||||
t.Errorf("schema version = %d, want %d", v, schemaVersion)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedactMySQLDSN(t *testing.T) {
|
||||
got := RedactMySQLDSN("user:s3cret@tcp(db.example.com:3306)/exploredns")
|
||||
if got != "user:xxxxx@tcp(db.example.com:3306)/exploredns" {
|
||||
t.Errorf("RedactMySQLDSN = %q", got)
|
||||
}
|
||||
if got := RedactMySQLDSN("::::"); got != "(unparsable DSN)" {
|
||||
t.Errorf("RedactMySQLDSN(bad) = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMySQLDialectSQL pins the MySQL-side SQL text, which unit tests cannot
|
||||
// execute without a server.
|
||||
func TestMySQLDialectSQL(t *testing.T) {
|
||||
d := mysqlDialect{}
|
||||
got := d.upsert("INSERT INTO traversals (id, domain) VALUES (?, ?)", []string{"domain", "last_seen"})
|
||||
want := "INSERT INTO traversals (id, domain) VALUES (?, ?)" +
|
||||
" ON DUPLICATE KEY UPDATE domain=VALUES(domain), last_seen=VALUES(last_seen)"
|
||||
if got != want {
|
||||
t.Errorf("upsert = %q, want %q", got, want)
|
||||
}
|
||||
pred, arg := d.likeContains("domain", `50%_o\ff`)
|
||||
if pred != `domain LIKE ? ESCAPE '\\'` {
|
||||
t.Errorf("likeContains predicate = %q", pred)
|
||||
}
|
||||
if arg != `%50\%\_o\\ff%` {
|
||||
t.Errorf("likeContains arg = %q", arg)
|
||||
}
|
||||
if d.datetimeType() != "DATETIME(6)" {
|
||||
t.Errorf("datetimeType = %q", d.datetimeType())
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -276,7 +276,7 @@ func newHandler(ctx context.Context) *Handler {
|
||||
maxRunning: envInt("EXPLOREDNS_MAX_JOBS", defaultMaxRunningJobs),
|
||||
version: "dev",
|
||||
limiter: newRateLimiter(limit, window),
|
||||
webhook: newWebhookReporter(os.Getenv("EXPLOREDNS_WEBHOOK_URL")),
|
||||
webhook: newWebhookReporter(os.Getenv("EXPLOREDNS_WEBHOOK_URL"), os.Getenv("EXPLOREDNS_WEBHOOK_TOKEN")),
|
||||
fp: fingerprint.New(),
|
||||
}
|
||||
|
||||
|
||||
+8
-2
@@ -52,19 +52,22 @@ type webhookCompleteEvent struct {
|
||||
// single retry, and failures are logged but never surface to callers.
|
||||
type webhookReporter struct {
|
||||
url string
|
||||
token string
|
||||
client *http.Client
|
||||
timeout time.Duration
|
||||
retryDelay time.Duration
|
||||
}
|
||||
|
||||
// newWebhookReporter returns a reporter for url, or nil when url is empty
|
||||
// (webhook reporting disabled). A nil reporter is safe to call.
|
||||
func newWebhookReporter(url string) *webhookReporter {
|
||||
// (webhook reporting disabled). A nil reporter is safe to call. A non-empty
|
||||
// token is sent as an Authorization bearer token on every delivery.
|
||||
func newWebhookReporter(url, token string) *webhookReporter {
|
||||
if url == "" {
|
||||
return nil
|
||||
}
|
||||
return &webhookReporter{
|
||||
url: url,
|
||||
token: token,
|
||||
client: &http.Client{},
|
||||
timeout: 5 * time.Second,
|
||||
retryDelay: 2 * time.Second,
|
||||
@@ -106,6 +109,9 @@ func (wr *webhookReporter) post(event string, body []byte) error {
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("X-ExploreDNS-Event", event)
|
||||
if wr.token != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+wr.token)
|
||||
}
|
||||
|
||||
resp, err := wr.client.Do(req)
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
rstore "gitea.hansenits.com.au/hits/ExploreDNS/internal/receiver/store"
|
||||
)
|
||||
|
||||
// These tests pin the webhook payload contract between this package (the
|
||||
// sender) and internal/receiver/store (the receiver): every field a sender
|
||||
// struct marshals must decode into the receiver struct and vice versa.
|
||||
// DisallowUnknownFields turns any renamed or missing field into a failure.
|
||||
|
||||
// decodeStrict unmarshals data into v, failing on unknown fields.
|
||||
func decodeStrict(t *testing.T, data []byte, v any) {
|
||||
t.Helper()
|
||||
dec := json.NewDecoder(bytes.NewReader(data))
|
||||
dec.DisallowUnknownFields()
|
||||
if err := dec.Decode(v); err != nil {
|
||||
t.Fatalf("decode %s into %T: %v", data, v, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWebhookStartEventReceiverCompat(t *testing.T) {
|
||||
sent := webhookStartEvent{
|
||||
Event: webhookEventStart,
|
||||
ID: "job-1",
|
||||
Domain: "example.com",
|
||||
QueryType: "AAAA",
|
||||
AllRoots: true,
|
||||
ClientIP: "203.0.113.9",
|
||||
StartedAt: time.Date(2026, 7, 1, 10, 0, 0, 123456789, time.UTC),
|
||||
}
|
||||
raw, err := json.Marshal(sent)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal sender: %v", err)
|
||||
}
|
||||
|
||||
var got rstore.StartEvent
|
||||
decodeStrict(t, raw, &got)
|
||||
want := rstore.StartEvent{
|
||||
Event: sent.Event,
|
||||
ID: sent.ID,
|
||||
Domain: sent.Domain,
|
||||
QueryType: sent.QueryType,
|
||||
AllRoots: sent.AllRoots,
|
||||
ClientIP: sent.ClientIP,
|
||||
StartedAt: sent.StartedAt,
|
||||
}
|
||||
if got != want {
|
||||
t.Errorf("receiver decoded %+v, want %+v", got, want)
|
||||
}
|
||||
|
||||
// Round-trip back into the sender struct so receiver-only fields
|
||||
// would also fail.
|
||||
back, err := json.Marshal(got)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal receiver: %v", err)
|
||||
}
|
||||
var sent2 webhookStartEvent
|
||||
decodeStrict(t, back, &sent2)
|
||||
if sent2 != sent {
|
||||
t.Errorf("sender round-trip %+v, want %+v", sent2, sent)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWebhookCompleteEventReceiverCompat(t *testing.T) {
|
||||
sent := webhookCompleteEvent{
|
||||
Event: webhookEventComplete,
|
||||
ID: "job-1",
|
||||
Domain: "example.com",
|
||||
QueryType: "A",
|
||||
ClientIP: "203.0.113.9",
|
||||
StartedAt: time.Date(2026, 7, 1, 10, 0, 0, 0, time.UTC),
|
||||
DoneAt: time.Date(2026, 7, 1, 10, 0, 3, 500000000, time.UTC),
|
||||
DurationMS: 3500,
|
||||
Status: statusError,
|
||||
Error: "traversal timed out after 5m0s",
|
||||
ResultCount: 7,
|
||||
Summary: &Summary{
|
||||
Answers: []SummaryAnswer{{Probability: 0.75, Records: []string{"example.com 300 IN A 192.0.2.1"}}},
|
||||
ByStatus: []SummaryStatus{{Status: "servfail", Probability: 0.25}},
|
||||
},
|
||||
}
|
||||
raw, err := json.Marshal(sent)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal sender: %v", err)
|
||||
}
|
||||
|
||||
var got rstore.CompleteEvent
|
||||
decodeStrict(t, raw, &got)
|
||||
if got.Event != sent.Event || got.ID != sent.ID || got.Domain != sent.Domain ||
|
||||
got.QueryType != sent.QueryType || got.ClientIP != sent.ClientIP ||
|
||||
!got.StartedAt.Equal(sent.StartedAt) || !got.DoneAt.Equal(sent.DoneAt) ||
|
||||
got.DurationMS != sent.DurationMS || got.Status != sent.Status ||
|
||||
got.Error != sent.Error || got.ResultCount != sent.ResultCount {
|
||||
t.Errorf("receiver decoded %+v, want %+v", got, sent)
|
||||
}
|
||||
|
||||
// The receiver keeps Summary as raw JSON; it must match the sender's
|
||||
// marshalled Summary byte for byte.
|
||||
wantSummary, err := json.Marshal(sent.Summary)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal summary: %v", err)
|
||||
}
|
||||
if !bytes.Equal(got.Summary, wantSummary) {
|
||||
t.Errorf("receiver Summary = %s, want %s", got.Summary, wantSummary)
|
||||
}
|
||||
|
||||
back, err := json.Marshal(got)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal receiver: %v", err)
|
||||
}
|
||||
var sent2 webhookCompleteEvent
|
||||
decodeStrict(t, back, &sent2)
|
||||
back2, err := json.Marshal(sent2)
|
||||
if err != nil {
|
||||
t.Fatalf("re-marshal sender: %v", err)
|
||||
}
|
||||
if !bytes.Equal(back2, raw) {
|
||||
t.Errorf("sender round-trip = %s, want %s", back2, raw)
|
||||
}
|
||||
}
|
||||
+31
-1
@@ -235,7 +235,37 @@ func TestWebhook_RetriesOnceOnFailure(t *testing.T) {
|
||||
func TestWebhook_NilReporterSafe(t *testing.T) {
|
||||
var wr *webhookReporter
|
||||
wr.send("start", map[string]string{"event": "start"}) // must not panic
|
||||
if newWebhookReporter("") != nil {
|
||||
if newWebhookReporter("", "token") != nil {
|
||||
t.Fatal("empty URL should disable the webhook reporter")
|
||||
}
|
||||
}
|
||||
|
||||
// TestWebhook_BearerToken verifies the Authorization header is sent exactly
|
||||
// when a token is configured (EXPLOREDNS_WEBHOOK_TOKEN on the real path).
|
||||
func TestWebhook_BearerToken(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name, token, wantAuth string
|
||||
}{
|
||||
{"token set", "s3cret", "Bearer s3cret"},
|
||||
{"token unset", "", ""},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
done := make(chan string, 1)
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
done <- r.Header.Get("Authorization")
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
newWebhookReporter(ts.URL, tc.token).send("start", map[string]string{"event": "start"})
|
||||
|
||||
select {
|
||||
case got := <-done:
|
||||
if got != tc.wantAuth {
|
||||
t.Fatalf("Authorization = %q, want %q", got, tc.wantAuth)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("webhook was not delivered")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user