HansenITSolutions/ExploreDNS
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

feat: rework engine and CLI for dnstraverse parity

Browse Source 
Port the traversal engine to the Ruby dnstraverse model so behaviour and
output match dns.squish.net:

- dns: single RD=0 query path (RD=1 only for upstream root discovery),
  per-run packet cache, EDNS0 512-fallback with warnings, UDP->TCP on
  truncation; fix --retries 0 and --root-server IP-literal handling;
  drop all hardcoded 127.0.0.1:53 resolvers
- traverse: hierarchical per-branch InfoCache, 7-step response
  classification with the full 10-status vocabulary, bailiwick
  partitioning, strictly-deeper lame-referral rule, refid grammar with
  .0 resolve subtrees and childset digits, per-IP branching at 1/n
  weight, cache-based glue resolution with noglue/loop dead ends, CNAME
  restarts from the deepest cached zone, fast-mode memoization,
  probability aggregation with Ruby-identical stats keys (sums to 1.0)
- output: byte-for-byte reference text format pinned by a golden test,
  reference CLI defaults, working --quiet/--show-X=false, TTY-aware
  colour, deduplicated deterministic JSON
- web: adapt API/SPA to the new engine, SSE events carry refid/status,
  fix subscribe/snapshot duplicate-event race and a statusCls TDZ bug,
  align SPA type list with the backend
- delete the old engine and dead code (net -4,350 lines)

Verified against live runs of the reference Ruby engine across five
domains (answers, NXDOMAIN, null MX, CNAME restart, glueless resolve)
with no divergences beyond the documented typo fixes.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Gary Hansen
2026-07-07 21:42:06 +10:00
co-authored by Claude Fable 5
parent af15c9c2d4
commit d71c7fbef2
53 changed files with 6685 additions and 9366 deletions
Download Patch File Download Diff File Expand all files Collapse all files
web/api/handler.go
+154 -67
View File
@@ -7,6 +7,8 @@ import (
"fmt"
"io/fs"
"net/http"
"sort"
"strings"
"sync"
"time"
@@ -40,23 +42,57 @@ type TraverseStartResponse struct {
// ProgressEvent carries a single traversal hook event.
type ProgressEvent struct {
Stage string `json:"stage"`
Depth int `json:"depth"`
Name string `json:"name"`
QType string `json:"qtype"`
Server string `json:"server,omitempty"`
Bailiwick string `json:"bailiwick,omitempty"`
IsResolve bool `json:"is_resolve,omitempty"`
Stage string `json:"stage"`
RefID string `json:"refid"`
Depth int `json:"depth"`
Name string `json:"name"`
QType string `json:"qtype"`
Server string `json:"server,omitempty"`
IPs string `json:"ips,omitempty"`
Bailiwick string `json:"bailiwick,omitempty"`
Status string `json:"status,omitempty"`
IsResolve bool `json:"is_resolve,omitempty"`
CompletedEarlier string `json:"completed_earlier,omitempty"`
}
// ResultItem is a single traversal step result for API consumers.
// ResultItem is one aggregated leaf outcome for API consumers. Parent and
// ParentIP identify the referring server so clients can render the noglue and
// lame-referral wordings; Qname/Qclass/Qtype are the failing query so clients
// can render the "While querying" line when it differs from the original.
type ResultItem struct {
Depth int `json:"depth"`
Probability float64 `json:"probability"`
ResponseType string `json:"response_type"`
Server string `json:"server,omitempty"`
Answers []string `json:"answers,omitempty"`
CNAMEChain []string `json:"cname_chain,omitempty"`
RefID string `json:"refid,omitempty"`
Depth int `json:"depth"`
Probability float64 `json:"probability"`
Status string `json:"status"`
Server string `json:"server,omitempty"`
IP string `json:"ip,omitempty"`
Parent string `json:"parent,omitempty"`
ParentIP string `json:"parent_ip,omitempty"`
Qname string `json:"qname,omitempty"`
Qclass string `json:"qclass,omitempty"`
Qtype string `json:"qtype,omitempty"`
Answers []string `json:"answers,omitempty"`
Message string `json:"message,omitempty"`
}
// SummaryAnswer is one distinct answered RRset with its accumulated
// probability (traverse.SummaryStats).
type SummaryAnswer struct {
Probability float64 `json:"probability"`
Records []string `json:"records"`
}
// SummaryStatus is the accumulated probability of one non-answered status.
type SummaryStatus struct {
Status string `json:"status"`
Probability float64 `json:"probability"`
}
// Summary is the grouped view of the aggregated leaves; probabilities across
// Answers plus ByStatus sum to 1.0.
type Summary struct {
Answers []SummaryAnswer `json:"answers,omitempty"`
ByStatus []SummaryStatus `json:"by_status,omitempty"`
}
// TraversalJob holds all state for a single asynchronous traversal.
@@ -66,6 +102,7 @@ type TraversalJob struct {
Domain string `json:"domain"`
QueryType string `json:"query_type"`
Results []ResultItem `json:"results,omitempty"`
Summary *Summary `json:"summary,omitempty"`
Progress []ProgressEvent `json:"progress,omitempty"`
Error string `json:"error,omitempty"`
StartedAt time.Time `json:"started_at"`
@@ -76,14 +113,18 @@ type TraversalJob struct {
cancel context.CancelFunc
}
// subscribe returns a channel that receives future progress events.
// The channel is closed when the job finishes.
func (j *TraversalJob) subscribe() <-chan ProgressEvent {
// subscribeSnapshot atomically registers a subscriber and snapshots the
// progress recorded so far. Publishing appends to Progress and sends to
// subscribers under the same lock, so every event lands either in the
// returned snapshot or on the channel — never both, never neither.
func (j *TraversalJob) subscribeSnapshot() (sub <-chan ProgressEvent, past []ProgressEvent, done bool) {
ch := make(chan ProgressEvent, 32)
j.mu.Lock()
defer j.mu.Unlock()
j.subs = append(j.subs, ch)
j.mu.Unlock()
return ch
past = make([]ProgressEvent, len(j.Progress))
copy(past, j.Progress)
return ch, past, j.Status != statusRunning
}
// publishLocked sends ev to all current subscribers. Caller must hold j.mu.
@@ -259,6 +300,7 @@ func (h *Handler) getTraversal(w http.ResponseWriter, r *http.Request) {
Domain string `json:"domain"`
QueryType string `json:"query_type"`
Results []ResultItem `json:"results,omitempty"`
Summary *Summary `json:"summary,omitempty"`
Progress []ProgressEvent `json:"progress,omitempty"`
Error string `json:"error,omitempty"`
StartedAt time.Time `json:"started_at"`
@@ -269,6 +311,7 @@ func (h *Handler) getTraversal(w http.ResponseWriter, r *http.Request) {
Domain: job.Domain,
QueryType: job.QueryType,
Results: job.Results,
Summary: job.Summary,
Progress: job.Progress,
Error: job.Error,
StartedAt: job.StartedAt,
@@ -300,19 +343,12 @@ func (h *Handler) streamTraversal(w http.ResponseWriter, r *http.Request) {
return
}
// Subscribe before snapshotting progress so we don't miss events between
// the two operations. Unsubscribe when the client disconnects so stale
// channels don't accumulate.
sub := job.subscribe()
// Subscribe and snapshot atomically so events published in between are
// neither missed nor delivered twice. Unsubscribe when the client
// disconnects so stale channels don't accumulate.
sub, past, alreadyDone := job.subscribeSnapshot()
defer job.unsubscribe(sub)
// Replay events already recorded.
job.mu.RLock()
past := make([]ProgressEvent, len(job.Progress))
copy(past, job.Progress)
alreadyDone := job.Status != statusRunning
job.mu.RUnlock()
sendSSE := func(ev ProgressEvent) bool {
b, err := json.Marshal(ev)
if err != nil {
@@ -375,26 +411,22 @@ func (h *Handler) runTraversal(ctx context.Context, job *TraversalJob, domain st
}
cfg.Hooks = &traverse.TraverserHooks{
OnEvent: func(event traverse.TraversalEvent) {
ref := event.Result.Referral
if ref == nil {
ref := event.Referral
if ref == nil || ref.IsRootRoot() {
return
}
stage := "start"
if event.Stage == traverse.EventComplete {
stage = "complete"
}
server := ""
if event.Result.Response != nil && event.Result.Response.Server != nil {
server = event.Result.Response.Server.String()
}
ev := ProgressEvent{
Stage: stage,
Depth: ref.Depth,
Name: trimFQDN(ref.Name),
QType: idns.QNameType(ref.Qtype),
Server: server,
Bailiwick: trimFQDN(ref.Bailiwick),
IsResolve: event.IsResolve,
Stage: event.Stage.String(),
RefID: event.RefID,
Depth: ref.Depth(),
Name: ref.Qname,
QType: traverse.TypeToString(ref.Qtype),
Server: ref.Server,
IPs: ref.TxtIPs(),
Bailiwick: ref.Bailiwick,
Status: string(event.Status),
IsResolve: event.IsResolve,
CompletedEarlier: event.CompletedEarlier,
}
job.mu.Lock()
@@ -405,7 +437,7 @@ func (h *Handler) runTraversal(ctx context.Context, job *TraversalJob, domain st
}
tr := traverse.NewTraverser(cfg)
rawResults, err := tr.Traverse(ctx, domain)
root, err := tr.Run(ctx, domain)
now := time.Now()
job.mu.Lock()
@@ -419,31 +451,87 @@ func (h *Handler) runTraversal(ctx context.Context, job *TraversalJob, domain st
return
}
items := make([]ResultItem, 0, len(rawResults))
for _, r := range rawResults {
items = append(items, toResultItem(r))
var items []ResultItem
if root != nil {
leaves := root.StatsList()
items = make([]ResultItem, 0, len(leaves))
for _, leaf := range leaves {
items = append(items, toResultItem(leaf))
}
job.Summary = toSummary(root.SummaryStats())
}
job.Results = items
job.Status = statusComplete
}
// toResultItem converts a TraversalResult to its API representation.
func toResultItem(r traverse.TraversalResult) ResultItem {
item := ResultItem{}
if r.Referral != nil {
item.Depth = r.Referral.Depth
item.Probability = r.Referral.Prob
// toSummary converts the engine's grouped stats to the API representation:
// answers sorted by RRset key (as SummaryStats returns them), remaining
// statuses sorted lexically like the CLI Summary Results section.
func toSummary(stats *traverse.SummaryStats) *Summary {
if stats == nil {
return nil
}
if r.Response != nil {
item.ResponseType = r.Response.Type.String()
if r.Response.Server != nil {
item.Server = r.Response.Server.String()
summary := &Summary{}
for _, answer := range stats.Answers {
item := SummaryAnswer{Probability: answer.Prob}
for _, rr := range answer.RRs {
item.Records = append(item.Records, collapseWhitespace(rr.String()))
}
if r.Response.Decoded != nil {
for _, rr := range r.Response.Decoded.Answers {
item.Answers = append(item.Answers, idns.FormatRecord(rr))
}
item.CNAMEChain = append(item.CNAMEChain, r.Response.Decoded.CNAMEChain...)
summary.Answers = append(summary.Answers, item)
}
statuses := make([]traverse.Status, 0, len(stats.ByStatus))
for status := range stats.ByStatus {
if status != traverse.StatusAnswered {
statuses = append(statuses, status)
}
}
sort.Slice(statuses, func(i, j int) bool { return statuses[i] < statuses[j] })
for _, status := range statuses {
summary.ByStatus = append(summary.ByStatus, SummaryStatus{
Status: string(status),
Probability: stats.ByStatus[status],
})
}
return summary
}
// collapseWhitespace renders an RR on one line with runs of whitespace
// collapsed to single spaces, matching the CLI summary records.
func collapseWhitespace(s string) string {
return strings.Join(strings.Fields(s), " ")
}
// toResultItem converts one aggregated leaf to its API representation.
func toResultItem(leaf *traverse.StatsEntry) ResultItem {
resp := leaf.Response
item := ResultItem{
Probability: leaf.Prob,
Status: string(resp.Status),
IP: resp.IP,
Server: resp.Server,
ParentIP: resp.ParentIP,
Qname: trimFQDN(resp.Qname),
Qclass: traverse.ClassToString(resp.Qclass),
Qtype: traverse.TypeToString(resp.Qtype),
}
if leaf.Referral != nil {
item.RefID = leaf.Referral.RefID
item.Depth = leaf.Referral.Depth()
item.Server = leaf.Referral.Server
item.ParentIP = leaf.Referral.ParentIP
if leaf.Referral.Parent != nil {
item.Parent = leaf.Referral.Parent.Server
}
}
if resp.DQ != nil {
for _, rr := range resp.DQ.Answers {
item.Answers = append(item.Answers, idns.FormatRecord(rr))
}
switch resp.Status {
case traverse.StatusError:
item.Message = resp.DQ.ErrorMessage
case traverse.StatusException:
item.Message = resp.DQ.ExceptionMessage
}
}
return item
@@ -495,4 +583,3 @@ func newUUID() string {
b[8] = (b[8] & 0x3f) | 0x80 // variant bits
return fmt.Sprintf("%08x-%04x-%04x-%04x-%012x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:])
}