diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 07edd46..676f979 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -67,3 +67,13 @@ jobs: tags: | gitea.hansenits.com.au/hits/exploredns-web:latest gitea.hansenits.com.au/hits/exploredns-web:${{ github.sha }} + + - name: Build and push receiver image + uses: docker/build-push-action@v6 + with: + context: . + file: Dockerfile.receiver + push: true + tags: | + gitea.hansenits.com.au/hits/exploredns-receiver:latest + gitea.hansenits.com.au/hits/exploredns-receiver:${{ github.sha }} diff --git a/.gitea/workflows/release.yml b/.gitea/workflows/release.yml index c322e0a..406da0b 100644 --- a/.gitea/workflows/release.yml +++ b/.gitea/workflows/release.yml @@ -1,8 +1,8 @@ name: Release # Builds release artifacts when a version tag (v*) is pushed: -# - cross-compiled CLI + server binaries attached to the Gitea release -# - version-tagged docker images for the CLI and web server +# - cross-compiled CLI + server + receiver binaries attached to the Gitea release +# - version-tagged docker images for the CLI, web server, and receiver # The owner usually creates the Gitea release by hand with notes; this # workflow attaches assets to it (creating a bare release only when none # exists) and skips assets that are already attached, so re-runs are safe. @@ -42,10 +42,13 @@ jobs: CGO_ENABLED=0 GOOS="$GOOS" GOARCH="$GOARCH" \ go build -trimpath -ldflags "-s -w -X main.version=${TAG}" \ -o "${OUT}/exploredns-server${EXT}" ./cmd/server + CGO_ENABLED=0 GOOS="$GOOS" GOARCH="$GOARCH" \ + go build -trimpath -ldflags "-s -w -X main.version=${TAG}" \ + -o "${OUT}/exploredns-receiver${EXT}" ./cmd/exploredns-receiver if [ "$GOOS" = "windows" ]; then - (cd "$OUT" && zip -q "../exploredns_${TAG}_${GOOS}_${GOARCH}.zip" exploredns.exe exploredns-server.exe) + (cd "$OUT" && zip -q "../exploredns_${TAG}_${GOOS}_${GOARCH}.zip" exploredns.exe exploredns-server.exe exploredns-receiver.exe) else - tar -czf "dist/exploredns_${TAG}_${GOOS}_${GOARCH}.tar.gz" -C "$OUT" exploredns exploredns-server + tar -czf "dist/exploredns_${TAG}_${GOOS}_${GOARCH}.tar.gz" -C "$OUT" exploredns exploredns-server exploredns-receiver fi rm -rf "$OUT" done @@ -130,3 +133,15 @@ jobs: tags: | gitea.hansenits.com.au/hits/exploredns-web:latest gitea.hansenits.com.au/hits/exploredns-web:${{ github.ref_name }} + + - name: Build and push receiver image + uses: docker/build-push-action@v6 + with: + context: . + file: Dockerfile.receiver + push: true + build-args: | + VERSION=${{ github.ref_name }} + tags: | + gitea.hansenits.com.au/hits/exploredns-receiver:latest + gitea.hansenits.com.au/hits/exploredns-receiver:${{ github.ref_name }} diff --git a/Dockerfile.cli b/Dockerfile.cli index fd40a4c..109bc94 100644 --- a/Dockerfile.cli +++ b/Dockerfile.cli @@ -1,5 +1,5 @@ # Build stage -FROM golang:1.24-alpine AS builder +FROM golang:1.25-alpine AS builder WORKDIR /src diff --git a/Dockerfile.receiver b/Dockerfile.receiver new file mode 100644 index 0000000..1404efe --- /dev/null +++ b/Dockerfile.receiver @@ -0,0 +1,26 @@ +# Build stage +FROM golang:1.25-alpine AS builder + +WORKDIR /src + +COPY go.mod go.sum ./ +RUN go mod download + +COPY . . + +ARG VERSION=dev +RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w -X main.version=${VERSION}" -o /out/exploredns-receiver ./cmd/exploredns-receiver + +# Final stage +FROM alpine:3.21 + +RUN apk --no-cache add ca-certificates + +COPY --from=builder /out/exploredns-receiver /usr/local/bin/exploredns-receiver + +ENV RECEIVER_SQLITE_PATH=/data/exploredns-receiver.db +VOLUME /data + +EXPOSE 8080 + +ENTRYPOINT ["exploredns-receiver"] diff --git a/Dockerfile.web b/Dockerfile.web index 7636194..aac819b 100644 --- a/Dockerfile.web +++ b/Dockerfile.web @@ -1,5 +1,5 @@ # Build stage -FROM golang:1.24-alpine AS builder +FROM golang:1.25-alpine AS builder WORKDIR /src diff --git a/Makefile b/Makefile index d73dcb0..25e092d 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,13 @@ BINARY_NAME=exploredns SERVER_BINARY_NAME=exploredns-server +RECEIVER_BINARY_NAME=exploredns-receiver BUILD_DIR=bin GO=go GOFLAGS=-v VERSION?=$(shell git describe --tags --always 2>/dev/null || echo dev) LDFLAGS=-ldflags "-X main.version=$(VERSION)" -.PHONY: build build-server build-all test lint clean cover deploy deploy-status +.PHONY: build build-server build-receiver build-all test lint clean cover deploy deploy-status build: $(GO) build $(GOFLAGS) $(LDFLAGS) -o $(BUILD_DIR)/$(BINARY_NAME) ./cmd/exploredns @@ -14,7 +15,10 @@ build: build-server: $(GO) build $(GOFLAGS) $(LDFLAGS) -o $(BUILD_DIR)/$(SERVER_BINARY_NAME) ./cmd/server -build-all: build build-server +build-receiver: + $(GO) build $(GOFLAGS) $(LDFLAGS) -o $(BUILD_DIR)/$(RECEIVER_BINARY_NAME) ./cmd/exploredns-receiver + +build-all: build build-server build-receiver test: $(GO) test -v -race -coverprofile=coverage.out ./... diff --git a/README.md b/README.md index 692dcad..209e60a 100644 --- a/README.md +++ b/README.md @@ -297,6 +297,7 @@ variables at startup: | `EXPLOREDNS_CORS_ORIGIN` | *(unset)* | Off by default (the SPA is same-origin). Set an origin — or `*` for development — to enable cross-origin API access. | | `EXPLOREDNS_RATE_LIMIT` | `30/1h` | Per-client-IP token-bucket limit on `POST /api/traverse` in `N/duration` form (e.g. `10/10m`); invalid values fall back to the default. Over-limit requests get `429`. Buckets refill continuously. Direct localhost connections are exempt (dev loop, tests), but proxied requests are always limited by the real client IP from `Fly-Client-IP` / `X-Forwarded-For`. | | `EXPLOREDNS_WEBHOOK_URL` | *(unset)* | Off by default. When set, the server POSTs a usage-reporting JSON event to this URL on every traversal start and completion (see below). | +| `EXPLOREDNS_WEBHOOK_TOKEN` | *(unset)* | Optional bearer token for webhook deliveries. When set, every webhook POST carries `Authorization: Bearer `; pair it with the receiver's `RECEIVER_INGEST_TOKEN`. | ### Usage reporting @@ -312,12 +313,17 @@ traversal, each with header `X-ExploreDNS-Event` naming the event: `X-Forwarded-For` entry, else the connection address). Delivery is fire-and-forget: a 5-second timeout, one retry after 2 seconds, and failures are logged without ever affecting the traversal or the API response. On -Fly.io, configure it as a secret rather than in `fly.toml`: +Fly.io, configure it (and the optional bearer token) as secrets rather than +in `fly.toml`: ```sh -fly secrets set EXPLOREDNS_WEBHOOK_URL=https://example.com/hook +fly secrets set EXPLOREDNS_WEBHOOK_URL=https://example.com/hook \ + EXPLOREDNS_WEBHOOK_TOKEN=some-long-random-string ``` +This repo ships a matching receiver for these events — see +[Usage telemetry receiver](#usage-telemetry-receiver). + --- ## Deploying to Fly.io @@ -366,9 +372,9 @@ flyctl tokens create deploy -x 999999h Pushing a `v*` tag triggers the full release pipeline: -1. `.gitea/workflows/release.yml` (`binaries` job) cross-compiles the CLI - and server for linux/amd64, linux/arm64, darwin/amd64, darwin/arm64 and - windows/amd64, packages them as +1. `.gitea/workflows/release.yml` (`binaries` job) cross-compiles the CLI, + server, and receiver for linux/amd64, linux/arm64, darwin/amd64, + darwin/arm64 and windows/amd64, packages them as `exploredns___.tar.gz` (`.zip` on Windows) plus a `SHA256SUMS` file, and attaches everything to the Gitea release for the tag. Create the release with notes by hand before (or after) pushing @@ -376,8 +382,8 @@ Pushing a `v*` tag triggers the full release pipeline: a bare one only when none exists, and skips already-attached assets so re-runs are safe. 2. `.gitea/workflows/release.yml` (`docker` job) pushes - `gitea.hansenits.com.au/hits/exploredns-cli` and `…/exploredns-web` - images tagged `` and `latest`. + `gitea.hansenits.com.au/hits/exploredns-cli`, `…/exploredns-web`, and + `…/exploredns-receiver` images tagged `` and `latest`. 3. `.gitea/workflows/deploy.yml` deploys the web server to Fly.io. All binaries are stamped with the tag via @@ -420,6 +426,78 @@ with `--show-servers`. Suitable for piping into `jq`. --- +## Usage telemetry receiver + +`cmd/exploredns-receiver` is a small companion service that receives the +usage webhooks described above (`start`/`complete` events from +`EXPLOREDNS_WEBHOOK_URL`), stores them in MySQL or SQLite, and serves a +basic-auth-protected admin dashboard (`/admin`) plus JSON API +(`/admin/api/traversals`, `/admin/api/stats`) over the collected data. It +is a separate binary intended to run wherever you keep long-lived storage +(e.g. a home Kubernetes cluster) while the public web server stays +stateless. + +Endpoints: `POST /webhook` (ingest, bearer-token protected when configured), +`GET /healthz` (liveness/readiness), `GET /admin` and `GET /admin/api/*` +(basic auth, always required). + +### Configuration + +| Variable | Default | Meaning | +|---|---|---| +| `RECEIVER_ADDR` | `:8080` | Listen address. | +| `RECEIVER_MYSQL_DSN` | *(unset)* | [go-sql-driver DSN](https://github.com/go-sql-driver/mysql#dsn-data-source-name) (`user:pass@tcp(host:3306)/dbname`). When set, events are stored in MySQL and the SQLite settings are ignored. | +| `RECEIVER_SQLITE_PATH` | `data/exploredns-receiver.db` | SQLite database path, used when no MySQL DSN is set (the container image defaults it to `/data/exploredns-receiver.db`). Parent directories are created automatically. | +| `RECEIVER_INGEST_TOKEN` | *(unset)* | When set, `POST /webhook` requires `Authorization: Bearer `. Leave unset only on trusted networks. | +| `RECEIVER_ADMIN_USER` | `admin` | Basic-auth username for `/admin`. | +| `RECEIVER_ADMIN_PASSWORD` | *(required)* | Basic-auth password for `/admin`; the receiver refuses to start without it. | + +Both storage backends share one portable schema; pure-Go drivers +(`modernc.org/sqlite`, `github.com/go-sql-driver/mysql`) keep the binary +CGO-free. SQLite is the zero-setup default; point `RECEIVER_MYSQL_DSN` at +an external MySQL when you want the data outside the pod/VM. + +### Running with Docker + +```sh +docker run -d --name exploredns-receiver \ + -p 8080:8080 \ + -v exploredns-receiver-data:/data \ + -e RECEIVER_ADMIN_PASSWORD=change-me \ + -e RECEIVER_INGEST_TOKEN=some-long-random-string \ + gitea.hansenits.com.au/hits/exploredns-receiver:latest +``` + +The image stores SQLite data under the `/data` volume; add +`-e RECEIVER_MYSQL_DSN=...` to use MySQL instead. + +### Running on Kubernetes + +[deploy/k8s/receiver/](deploy/k8s/receiver/) contains commented template +manifests: a single-replica deployment (SQLite on a 1Gi PVC mounted at +`/data`, probes on `/healthz`), ClusterIP service, ingress with TLS +placeholders, and a secret template for the `RECEIVER_*` variables. Edit +the placeholder host/credentials, then: + +```sh +kubectl apply -f deploy/k8s/receiver/ +``` + +Keep one replica while on SQLite; MySQL removes that constraint. + +### Pairing with the web server + +Set the same token on both ends so the receiver only accepts events from +your server — e.g. on Fly.io: + +```sh +fly secrets set EXPLOREDNS_WEBHOOK_URL=https://receiver.example.com/webhook \ + EXPLOREDNS_WEBHOOK_TOKEN=some-long-random-string +# receiver side: RECEIVER_INGEST_TOKEN=some-long-random-string +``` + +--- + ## Comparison with dnstraverse | Feature | dnstraverse (Ruby) | ExploreDNS (Go) | @@ -440,13 +518,16 @@ with `--show-servers`. Suitable for piping into `jq`. ``` cmd/exploredns/ CLI entry point and flag parsing cmd/server/ HTTP API server entry point +cmd/exploredns-receiver/ Usage telemetry receiver entry point internal/config/ Configuration types, validation, and usage text internal/dns/ DNS query layer, root discovery, transport internal/traverse/ Core traversal engine, referral resolution, caching internal/fingerprint/ DNS server version fingerprinting (version.bind CHAOS) internal/output/ Result formatting — text tree and JSON renderers internal/integration/ End-to-end integration tests +internal/receiver/ Telemetry receiver: HTTP server, admin UI, event store web/api/ HTTP handler, job store, SSE streaming, static assets +deploy/k8s/receiver/ Kubernetes manifest templates for the receiver ``` --- @@ -456,7 +537,8 @@ web/api/ HTTP handler, job store, SSE streaming, static assets ```sh make build # compile CLI binary to bin/exploredns make build-server # compile server binary to bin/exploredns-server -make build-all # compile both binaries +make build-receiver # compile telemetry receiver to bin/exploredns-receiver +make build-all # compile all three binaries make test # run all unit and integration tests make lint # run go vet make clean # remove build artefacts diff --git a/deploy/k8s/receiver/deployment.yaml b/deploy/k8s/receiver/deployment.yaml new file mode 100644 index 0000000..321d375 --- /dev/null +++ b/deploy/k8s/receiver/deployment.yaml @@ -0,0 +1,57 @@ +# TEMPLATE — single-replica receiver deployment. Keep replicas at 1 while +# using the SQLite backend: the database file on the RWO volume supports only +# one writer. With RECEIVER_MYSQL_DSN you may scale out and drop the volume. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: exploredns-receiver + namespace: exploredns-receiver + labels: + app: exploredns-receiver +spec: + replicas: 1 + strategy: + type: Recreate # RWO volume: never run old and new pods concurrently + selector: + matchLabels: + app: exploredns-receiver + template: + metadata: + labels: + app: exploredns-receiver + spec: + containers: + - name: receiver + image: gitea.hansenits.com.au/hits/exploredns-receiver:latest + ports: + - name: http + containerPort: 8080 + envFrom: + - secretRef: + name: exploredns-receiver + volumeMounts: + - name: data + mountPath: /data + livenessProbe: + httpGet: + path: /healthz + port: http + initialDelaySeconds: 5 + periodSeconds: 15 + readinessProbe: + httpGet: + path: /healthz + port: http + initialDelaySeconds: 2 + periodSeconds: 10 + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 500m + memory: 256Mi + volumes: + - name: data + persistentVolumeClaim: + claimName: exploredns-receiver-data diff --git a/deploy/k8s/receiver/ingress.yaml b/deploy/k8s/receiver/ingress.yaml new file mode 100644 index 0000000..c7d9c61 --- /dev/null +++ b/deploy/k8s/receiver/ingress.yaml @@ -0,0 +1,33 @@ +# TEMPLATE — replace receiver.example.com with your real host and wire up +# TLS for your cluster (the webhook bearer token and admin password travel +# in headers, so plain HTTP is not acceptable across the internet). +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: exploredns-receiver + namespace: exploredns-receiver + annotations: {} + # nginx ingress controller: + # cert-manager.io/cluster-issuer: letsencrypt + # nginx.ingress.kubernetes.io/proxy-body-size: 1m + # + # traefik: + # traefik.ingress.kubernetes.io/router.entrypoints: websecure + # traefik.ingress.kubernetes.io/router.tls: "true" +spec: + # ingressClassName: nginx + tls: + - hosts: + - receiver.example.com + secretName: exploredns-receiver-tls # created by cert-manager or by hand + rules: + - host: receiver.example.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: exploredns-receiver + port: + name: http diff --git a/deploy/k8s/receiver/namespace.yaml b/deploy/k8s/receiver/namespace.yaml new file mode 100644 index 0000000..8c8f219 --- /dev/null +++ b/deploy/k8s/receiver/namespace.yaml @@ -0,0 +1,6 @@ +# TEMPLATE — optional. Skip this file (and drop the namespace fields from the +# other manifests) to deploy into an existing namespace. +apiVersion: v1 +kind: Namespace +metadata: + name: exploredns-receiver diff --git a/deploy/k8s/receiver/pvc.yaml b/deploy/k8s/receiver/pvc.yaml new file mode 100644 index 0000000..bab2037 --- /dev/null +++ b/deploy/k8s/receiver/pvc.yaml @@ -0,0 +1,16 @@ +# TEMPLATE — backing storage for the SQLite database (RECEIVER_SQLITE_PATH +# defaults to /data/exploredns-receiver.db in the container image). Not needed +# when RECEIVER_MYSQL_DSN is set, but harmless to keep. Set storageClassName +# if your cluster has no default class. +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: exploredns-receiver-data + namespace: exploredns-receiver +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + # storageClassName: standard diff --git a/deploy/k8s/receiver/secret.yaml b/deploy/k8s/receiver/secret.yaml new file mode 100644 index 0000000..5d6abce --- /dev/null +++ b/deploy/k8s/receiver/secret.yaml @@ -0,0 +1,25 @@ +# TEMPLATE — fill in real values before applying, or create the secret +# imperatively instead and never commit credentials: +# +# kubectl -n exploredns-receiver create secret generic exploredns-receiver \ +# --from-literal=RECEIVER_ADMIN_USER=admin \ +# --from-literal=RECEIVER_ADMIN_PASSWORD='change-me' \ +# --from-literal=RECEIVER_INGEST_TOKEN='change-me-too' +# +# The deployment loads every key here as an environment variable (envFrom). +apiVersion: v1 +kind: Secret +metadata: + name: exploredns-receiver + namespace: exploredns-receiver +type: Opaque +stringData: + RECEIVER_ADMIN_USER: admin + RECEIVER_ADMIN_PASSWORD: change-me + # Bearer token the main app must send on POST /webhook. Must match the + # sender's EXPLOREDNS_WEBHOOK_TOKEN. Leave unset to accept unauthenticated + # posts (not recommended for an internet-facing receiver). + RECEIVER_INGEST_TOKEN: change-me-too + # Uncomment to store events in an external MySQL instead of the SQLite + # file on the PVC (go-sql-driver DSN). + # RECEIVER_MYSQL_DSN: "user:pass@tcp(mysql.example.com:3306)/exploredns" diff --git a/deploy/k8s/receiver/service.yaml b/deploy/k8s/receiver/service.yaml new file mode 100644 index 0000000..34be9c1 --- /dev/null +++ b/deploy/k8s/receiver/service.yaml @@ -0,0 +1,16 @@ +# TEMPLATE — cluster-internal service in front of the receiver pod. +apiVersion: v1 +kind: Service +metadata: + name: exploredns-receiver + namespace: exploredns-receiver + labels: + app: exploredns-receiver +spec: + type: ClusterIP + selector: + app: exploredns-receiver + ports: + - name: http + port: 8080 + targetPort: http