commit 23816d00f7ecb1be115b9bfa68b9f0478722ba30 Author: Daniel Noulet Date: Wed Oct 7 15:03:50 2026 +0200 initial commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3f4422f --- /dev/null +++ b/.gitignore @@ -0,0 +1,26 @@ +# Build artefacts +/bin/ +/frontend/dist/ +/frontend/node_modules/ +/backend/webui/dist/* +!/backend/webui/dist/index.html + +# Flat-file store: the seed document is tracked, generated fixtures are not +/data/docs/* +!/data/docs/go-doc.md +/data/comments/ +!data/comments/.gitkeep + +# Editor / IDE +.idea/ +.vscode/ +*.swp + +# OS noise +.DS_Store +Thumbs.db + +# Logs and scratch +*.log +*.out +/tmp/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..1ff3bd3 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,37 @@ +# Production build for go-doc. +# Build context must be the workspace root (/home/dnoulet/go). +# Documents under go-doc/data/docs are copied into the image at /data/docs. +# Persistence must mount only /data/comments so this tree stays visible. +FROM node:22-alpine AS frontend + +WORKDIR /src/frontend +COPY go-doc/frontend/package.json go-doc/frontend/package-lock.json ./ +RUN npm ci --no-audit --no-fund +COPY go-doc/frontend/ ./ +RUN npm run build + +FROM golang:1.27-alpine AS builder + +RUN apk add --no-cache git ca-certificates + +WORKDIR /src +COPY go-doc/backend/go.mod go-doc/backend/go.sum ./ +RUN go mod download + +COPY go-doc/backend/ backend/ +COPY --from=frontend /src/frontend/dist/ backend/webui/dist/ +COPY go-doc/data/docs /staged/docs +RUN chmod -R a+rX /staged/docs && mkdir -p /empty/comments +RUN cd backend && CGO_ENABLED=0 go build -trimpath -ldflags '-s -w' -o /out/godoc . + +FROM gcr.io/distroless/static-debian12:nonroot + +COPY --from=builder /out/godoc /godoc +COPY --from=builder /staged/docs /data/docs +COPY --from=builder /empty/comments /data/comments + +EXPOSE 8080 + +ENV GO_DOC_DATA_DIR=/data + +ENTRYPOINT ["/godoc", "-addr", "0.0.0.0:8080"] diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..05e37ef --- /dev/null +++ b/Makefile @@ -0,0 +1,288 @@ +# ============================================================================= +# go-doc — documentation & audience feedback platform +# +# make help list every target +# make dev run infrastructure + backend + frontend together +# make test run unit, integration and BDD suites +# make build embed the React bundle into the portable Go binary +# make docker build the all-in-one image +# make docker-push build and push the image to the private registry +# make docker-run run the image on :8080 +# make helm-deploy-development deploy to development +# make helm-deploy-production deploy to production +# ============================================================================= + +SHELL := /bin/bash +.SHELLFLAGS := -eu -o pipefail -c + +GO ?= go +NPM ?= npm +COMPOSE ?= docker compose +HELM ?= helm + +ROOT := $(shell pwd) +WORKSPACE ?= $(dir $(ROOT)) +BACKEND_DIR := $(ROOT)/backend +FRONTEND_DIR := $(ROOT)/frontend +DATA_DIR := $(ROOT)/data +WEBUI_DIST := $(BACKEND_DIR)/webui/dist + +BINARY := $(ROOT)/bin/godoc +BACKEND_ADDR ?= 0.0.0.0:8080 +HOST_ADDR ?= 127.0.0.1:8080 +VITE_PORT ?= 5173 + +# Docker/Registry +IMAGE ?= go-doc +TAG ?= latest +REGISTRY_IMAGE ?= nexus.urpi.be/docker-hosted/go-doc + +# Helm +HELM_CHART := $(ROOT)/helm/go-doc +HELM_RELEASE ?= go-doc +HELM_NAMESPACE ?= go-doc +KUBECONFIG ?= $(HOME)/.kube/config +GO_LOOSE_CA_SECRET ?= local-ca + +# Go Loose credentials passed via environment or .env +GO_LOOSE_SET = $(if $(GO_LOOSE_CLIENT_ID),--set-string goLoose.clientID=$(GO_LOOSE_CLIENT_ID)) \ + $(if $(GO_LOOSE_CLIENT_SECRET),--set-string goLoose.clientSecret=$(GO_LOOSE_CLIENT_SECRET)) \ + $(if $(GO_LOOSE_CA_SECRET),--set-string goLoose.ca.existingSecret=$(GO_LOOSE_CA_SECRET)) + +# Parent directory that holds the go-* projects to document. Override with +# WORKSPACE=/path/to/projects when the projects live elsewhere. +DOCS_WORKSPACE ?= $(patsubst %/,%,$(WORKSPACE)) +DOCGEN_INTERVAL ?= 5s + +DOCS_COUNT ?= 120 +COMMENTS_PER_DOC ?= 25 +SEED_WORKERS ?= 8 + +GO_SOURCES := $(shell find $(BACKEND_DIR) -name '*.go' -type f 2>/dev/null) +FRONTEND_SOURCES := $(shell find $(FRONTEND_DIR)/src -type f 2>/dev/null) + +.DEFAULT_GOAL := help +.PHONY: help infra-up infra-down infra-logs backend-run frontend-run dev test \ + test-unit test-integration test-bdd test-race build build-frontend \ + build-backend fmt vet tidy seed seed-perf clean distclean run docker-build \ + docker docker-push docker-run helm-lint helm-deploy-development \ + helm-deploy-production helm-down-development helm-down-production \ + docs-sync docs-watch docs-dry-run docs-list + +## --------------------------------------------------------------------------- +## Help +## --------------------------------------------------------------------------- +help: ## Show this help + @printf '\n\033[1mgo-doc\033[0m — available targets\n\n' + @grep -hE '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) \ + | sort \ + | awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-18s\033[0m %s\n", $$1, $$2}' + @printf '\n' + +## --------------------------------------------------------------------------- +## Infrastructure +## --------------------------------------------------------------------------- +infra-up: ## Start local development containers detached + @if $(COMPOSE) version >/dev/null 2>&1; then \ + $(COMPOSE) up -d --remove-orphans; \ + else \ + echo "error: '$(COMPOSE)' is unavailable. Install Docker Compose v2." >&2; \ + exit 1; \ + fi + +infra-down: ## Tear down infrastructure and remove volumes + @if $(COMPOSE) version >/dev/null 2>&1; then \ + $(COMPOSE) down --volumes --remove-orphans; \ + else \ + echo "docker compose unavailable; nothing to tear down"; \ + fi + +infra-logs: ## Tail infrastructure logs + $(COMPOSE) logs -f + +docker-build: ## Build the container images + $(COMPOSE) build + +## --------------------------------------------------------------------------- +## Run +## --------------------------------------------------------------------------- +backend-run: ## Run the Go API natively on 0.0.0.0:8080 + @mkdir -p $(DATA_DIR) + @echo "→ backend http://$(HOST_ADDR)" + @cd $(BACKEND_DIR) && GO_DOC_DATA_DIR=$(DATA_DIR) $(GO) run main.go -addr $(BACKEND_ADDR) + +frontend-run: ## Run the Vite dev server on port 5173 + @echo "→ frontend http://$(VITE_PORT)" + @cd $(FRONTEND_DIR) && $(NPM) run dev -- --port $(VITE_PORT) + +dev: ## Run backend, frontend and infrastructure concurrently + @$(MAKE) --no-print-directory infra-up + @echo "→ backend http://$(HOST_ADDR)" + @echo "→ frontend http://$(VITE_PORT) (Ctrl-C stops everything)" + @trap 'echo; echo "→ stopping dev stack"; kill 0' EXIT INT TERM; \ + ( cd $(BACKEND_DIR) && GO_DOC_DATA_DIR=$(DATA_DIR) $(GO) run main.go -addr $(BACKEND_ADDR) ) & \ + ( cd $(FRONTEND_DIR) && $(NPM) run dev -- --port $(VITE_PORT) ) & \ + wait + +run: build ## Build the single-file binary and run it + @echo "→ serving SPA + API on http://$(HOST_ADDR)" + @$(BINARY) -addr $(BACKEND_ADDR) -data $(DATA_DIR) + +## --------------------------------------------------------------------------- +## Test +## --------------------------------------------------------------------------- +test: fmt vet ## Run every suite: unit, integration, BDD, race + @echo "→ go test ./... -count=1" + @cd $(BACKEND_DIR) && $(GO) test ./... -count=1 + @echo "→ race detector on the storage engine and API" + @cd $(BACKEND_DIR) && $(GO) test ./storage/... ./tests/... -race -count=1 + @echo "✓ all suites passed" + +test-unit: ## Run storage engine unit tests only + @cd $(BACKEND_DIR) && $(GO) test ./storage/... -count=1 -v + +test-integration: ## Run end-to-end API tests against the real filesystem + @cd $(BACKEND_DIR) && $(GO) test ./tests/ -run Integration -count=1 -v + +test-bdd: ## Run the BDD feature suites + @cd $(BACKEND_DIR) && $(GO) test ./tests/bdd/... -count=1 -v + +test-race: ## Run all tests under the race detector + @cd $(BACKEND_DIR) && $(GO) test ./... -race -count=1 + +## --------------------------------------------------------------------------- +## Build +## --------------------------------------------------------------------------- +build: build-frontend build-backend ## Build the SPA and embed it in the Go binary + @echo "✓ $(BINARY) serves both the SPA and /api" + +build-frontend: $(FRONTEND_SOURCES) ## Compile the React bundle into frontend/dist + @cd $(FRONTEND_DIR) && $(NPM) install --no-audit --no-fund + @cd $(FRONTEND_DIR) && $(NPM) run build + +build-backend: $(GO_SOURCES) ## Stage assets into webui/dist and compile the binary + @test -f $(FRONTEND_DIR)/dist/index.html || { \ + echo "error: frontend/dist/index.html missing — run 'make build-frontend' first" >&2; \ + exit 1; \ + } + @mkdir -p $(WEBUI_DIST) $(dir $(BINARY)) + @rm -rf $(WEBUI_DIST) + @mkdir -p $(WEBUI_DIST) + @cp -r $(FRONTEND_DIR)/dist/. $(WEBUI_DIST)/ + @echo "→ embedded assets: $$(find $(WEBUI_DIST) -type f | wc -l) files" + @cd $(BACKEND_DIR) && CGO_ENABLED=0 $(GO) build -trimpath -ldflags '-s -w' -o $(BINARY) . + @echo "✓ built $(BINARY) ($$(du -h $(BINARY) | cut -f1))" + +## --------------------------------------------------------------------------- +## Quality +## --------------------------------------------------------------------------- +fmt: ## gofmt the backend + @cd $(BACKEND_DIR) && $(GO) fmt ./... >/dev/null + @echo "✓ gofmt" + +vet: ## go vet the backend + @cd $(BACKEND_DIR) && $(GO) vet ./... + @echo "✓ go vet" + +tidy: ## Tidy go.mod + @cd $(BACKEND_DIR) && $(GO) mod tidy + +## --------------------------------------------------------------------------- +## Documentation generation +## --------------------------------------------------------------------------- +## Every go-* project under WORKSPACE is analysed and imported into data/docs. +## Generated pages are wrapped in markers, so a +## re-sync replaces only the machine-written part and keeps anything you wrote +## outside the markers. Pass DOCGEN_ARGS to reach the CLI directly, e.g. +## make docs-sync DOCGEN_ARGS="-apps go-tell -force" + +docs-sync: ## Analyse every go-* project under WORKSPACE and import its docs (DOCGEN_ARGS) + @mkdir -p $(DATA_DIR)/docs + @cd $(BACKEND_DIR) && $(GO) run ./scripts/docgen \ + -root $(DOCS_WORKSPACE) -data $(DATA_DIR) $(DOCGEN_ARGS) + +docs-watch: ## Keep importing documentation as projects change (Ctrl-C to stop) + @mkdir -p $(DATA_DIR)/docs + @cd $(BACKEND_DIR) && $(GO) run ./scripts/docgen \ + -root $(DOCS_WORKSPACE) -data $(DATA_DIR) -watch -interval $(DOCGEN_INTERVAL) $(DOCGEN_ARGS) + +docs-dry-run: ## Analyse and render without writing anything + @cd $(BACKEND_DIR) && $(GO) run ./scripts/docgen \ + -root $(DOCS_WORKSPACE) -data $(DATA_DIR) -dry-run $(DOCGEN_ARGS) + +docs-list: ## List the go-* projects that a sync would import + @printf 'workspace: %s\n\n' '$(DOCS_WORKSPACE)' + @ls -1 $(DOCS_WORKSPACE) 2>/dev/null | grep '^go-' | grep -v '^go-doc$$' | sed 's/^/ /' + +## --------------------------------------------------------------------------- +## Data +## --------------------------------------------------------------------------- +seed: ## Seed 120 generated documents + thousands of comments (DOCS_COUNT, COMMENTS_PER_DOC) + @cd $(BACKEND_DIR) && $(GO) run ./scripts/seed_perf_data.go \ + -data $(DATA_DIR) -docs $(DOCS_COUNT) -comments $(COMMENTS_PER_DOC) -workers $(SEED_WORKERS) + +seed-perf: seed ## Alias for seed + +## --------------------------------------------------------------------------- +## Clean +## --------------------------------------------------------------------------- +clean: ## Remove build artefacts + @rm -rf $(ROOT)/bin $(FRONTEND_DIR)/dist + @echo "✓ cleaned build artefacts (backend/webui/dist left intact)" + +distclean: clean ## Also restore the placeholder frontend bundle + @rm -rf $(WEBUI_DIST) + @mkdir -p $(WEBUI_DIST) + @cp $(BACKEND_DIR)/webui/dist.placeholder/index.html $(WEBUI_DIST)/index.html + @echo "✓ webui/dist reset to the placeholder bundle" + +## --------------------------------------------------------------------------- +## Docker +## --------------------------------------------------------------------------- +docker: ## Build the all-in-one image + docker build -t $(IMAGE):$(TAG) -f $(BACKEND_DIR)/Dockerfile . + +docker-push: ## Build and push the image to the private registry + docker tag $(IMAGE):$(TAG) $(REGISTRY_IMAGE):$(TAG) + docker push $(REGISTRY_IMAGE):$(TAG) + +docker-run: ## Run the image on :8080 + docker run --rm -p 8080:8080 --env-file $(ROOT)/.env $(IMAGE):$(TAG) + +## --------------------------------------------------------------------------- +## Helm +## --------------------------------------------------------------------------- +helm-lint: ## Lint the helm chart + $(HELM) lint $(HELM_CHART) -f $(HELM_CHART)/values-dev.yaml + $(HELM) lint $(HELM_CHART) -f $(HELM_CHART)/values-prd.yaml + $(HELM) template $(HELM_RELEASE) $(HELM_CHART) \ + -f $(HELM_CHART)/values-dev.yaml >/dev/null + $(HELM) template $(HELM_RELEASE) $(HELM_CHART) \ + -f $(HELM_CHART)/values-prd.yaml \ + --set-string goLoose.clientID=lint --set-string goLoose.clientSecret=lint >/dev/null + +# Go Loose credentials are never stored in the values files. Pass them on the +# command line, from a secret store, or with --set-file: +# make helm-deploy-production \ +# --set goLoose.clientID=$(GO_LOOSE_CLIENT_ID) \ +# --set goLoose.clientSecret=$(GO_LOOSE_CLIENT_SECRET) \ +# --set goLoose.ca.existingSecret=local-ca +helm-deploy-development: ## Deploy to development + KUBECONFIG="$(KUBECONFIG)" $(HELM) upgrade --install $(HELM_RELEASE) $(HELM_CHART) \ + --namespace $(HELM_NAMESPACE)-development --create-namespace \ + -f $(HELM_CHART)/values-dev.yaml \ + --rollback-on-failure --wait --timeout 10m + +helm-deploy-production: ## Deploy to production + set -a; [ ! -f $(ROOT)/.env ] || . $(ROOT)/.env; set +a; \ + KUBECONFIG="$(KUBECONFIG)" $(HELM) upgrade --install $(HELM_RELEASE) $(HELM_CHART) \ + --namespace go-insane-prd --create-namespace \ + -f $(HELM_CHART)/values-prd.yaml \ + $(GO_LOOSE_SET) \ + --rollback-on-failure --wait --timeout 10m + +helm-down-development: ## Tear down development release + KUBECONFIG="$(KUBECONFIG)" $(HELM) uninstall $(HELM_RELEASE) -n $(HELM_NAMESPACE)-development + +helm-down-production: ## Tear down production release + KUBECONFIG="$(KUBECONFIG)" $(HELM) uninstall $(HELM_RELEASE) -n $(HELM_NAMESPACE)-production \ No newline at end of file diff --git a/backend/Dockerfile b/backend/Dockerfile new file mode 100644 index 0000000..f682f8d --- /dev/null +++ b/backend/Dockerfile @@ -0,0 +1,18 @@ +# Development image for the Go backend. `docker-compose up` bind-mounts the +# source and runs `go run main.go`, so this image only needs the toolchain. +FROM golang:1.27-alpine + +RUN apk add --no-cache git ca-certificates tzdata wget + +ENV CGO_ENABLED=0 \ + GOFLAGS=-mod=mod + +WORKDIR /app/backend + +# Warm the module cache before the bind mount replaces the working directory. +COPY go.mod ./ +RUN go mod download || true + +EXPOSE 8080 + +CMD ["go", "run", "main.go", "-addr", "0.0.0.0:8080"] \ No newline at end of file diff --git a/backend/auth/auth.go b/backend/auth/auth.go new file mode 100644 index 0000000..78137d2 --- /dev/null +++ b/backend/auth/auth.go @@ -0,0 +1,95 @@ +package auth + +import ( + "context" + "net/http" + + "go-doc/go-loose/client" +) + +// User represents an authenticated user from go-loose +type User struct { + ID string + Email string + DisplayName string + AvatarURL string + TenantID string + TenantSlug string + ApplicationID string + Application string + Role string + Permissions []string +} + +func (u User) Can(permission string) bool { + for _, granted := range u.Permissions { + if granted == permission { + return true + } + } + return false +} + +func (u User) IsAdmin() bool { + return u.Role == "admin" || u.Role == "owner" || u.Can("console.read") +} + +type contextKey string + +const userContextKey = contextKey("go-doc-user") + +func WithUser(ctx context.Context, user User) context.Context { + return context.WithValue(ctx, userContextKey, user) +} + +func UserFromContext(ctx context.Context) (User, bool) { + user, ok := ctx.Value(userContextKey).(User) + return user, ok +} + +func TenantIDFromContext(ctx context.Context) (string, bool) { + user, ok := UserFromContext(ctx) + if !ok { + return "", false + } + return user.TenantID, true +} + +func UserIDFromContext(ctx context.Context) (string, bool) { + user, ok := UserFromContext(ctx) + if !ok { + return "", false + } + return user.ID, true +} + +func ConvertUser(looseUser client.User) User { + return User{ + ID: looseUser.ID, + Email: looseUser.Email, + DisplayName: looseUser.DisplayName, + AvatarURL: looseUser.AvatarURL, + TenantID: looseUser.TenantID, + TenantSlug: looseUser.TenantSlug, + ApplicationID: looseUser.ApplicationID, + Application: looseUser.Application, + Role: looseUser.Role, + Permissions: looseUser.Permissions, + } +} + +func BrowserAuthMiddleware(browserAuth *client.BrowserAuth) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + if browserAuth == nil { + return next + } + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + browserAuth.Middleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if user, ok := client.UserFromContext(r.Context()); ok { + r = r.WithContext(WithUser(r.Context(), ConvertUser(user))) + } + next.ServeHTTP(w, r) + })).ServeHTTP(w, r) + }) + } +} \ No newline at end of file diff --git a/backend/docgen/docgen.go b/backend/docgen/docgen.go new file mode 100644 index 0000000..7d87c39 --- /dev/null +++ b/backend/docgen/docgen.go @@ -0,0 +1,728 @@ +// Package docgen analyses a Go project and renders audience-segmented +// documentation for go-doc. +// +// The pipeline is deliberately mechanical: Analyze walks a project tree and +// extracts facts that can be read off the filesystem (module metadata, HTTP +// routes, CLI flags, environment variables, Make targets, container services, +// Helm charts, existing Markdown, language breakdown), and Render turns those +// facts into the same three-audience layout go-doc uses natively: +// +// ## For Users what the thing does and how to use it +// ## For Operators running, configuring and deploying it +// ## For Developers contracts, layout and dependencies +// +// Import wraps the rendered page in generated markers so re-running the tool +// refreshes only the machine-written region and never clobbers prose a human +// added outside it. +package docgen + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "io/fs" + "os" + "path" + "path/filepath" + "sort" + "strings" + "time" +) + +// maxReadBytes caps how much of any single file is read into memory. +const maxReadBytes = 2 << 20 + +// maxWalkFiles caps the walk so a mis-pointed root cannot spin forever. +const maxWalkFiles = 200000 + +// Route is one HTTP endpoint discovered in the source. +type Route struct { + Method string + Path string + Handler string +} + +// String renders the route the way it is registered. +func (r Route) String() string { + if r.Method == "" { + return r.Path + } + return r.Method + " " + r.Path +} + +// SortKey orders routes by path, then method, so output is stable. +func (r Route) SortKey() string { return r.Path + "\x00" + r.Method + "\x00" + r.Handler } + +// Flag is a command-line flag declared with the flag package. +type Flag struct { + Name string + Type string + Default string + Usage string +} + +// EnvVar is an environment variable the code reads, plus a note when it was +// discovered in a .env.example file rather than in source. +type EnvVar struct { + Name string + Example string + FromEnv bool +} + +// Dependency is one requirement line from a go.mod. +type Dependency struct { + Path string + Version string + Indirect bool +} + +// Module is one go.mod file found in the tree. +type Module struct { + Path string + File string + GoVersion string + Direct []Dependency + Indirect int +} + +// ComposeService is a service declared in a docker-compose file. +type ComposeService struct { + Name string + Image string + Ports string + File string +} + +// DockerInfo summarises the project's Dockerfile. +type DockerInfo struct { + File string + Stages []string + Bases []string + Exposed []string +} + +// DocFile is an existing Markdown document that ships with the project. +type DocFile struct { + Path string + Title string + Bytes int +} + +// Dir is a first-level source directory with a purpose inferred from its name. +type Dir struct { + Name string + Files int + Lines int +} + +// Language is a per-extension line count. +type Language struct { + Name string + Files int + Lines int +} + +// Stats are the headline counters shown in the generated page. +type Stats struct { + GoFiles int + TestFiles int + TestFuncs int + Components int + TotalLines int + OldestCommit string +} + +// Project is the full analysis result for one repository. +type Project struct { + Name string + Path string + Summary string + Readme string + + Modules []Module + Routes []Route + Flags []Flag + EnvVars []EnvVar + MakeTargets []string + Compose []ComposeService + Docker *DockerInfo + HelmCharts []string + Docs []DocFile + Dirs []Dir + Languages []Language + Ports []string + Stats Stats + + // Signature changes whenever any analysed input changes. Watch mode uses it + // to decide whether a regeneration is needed. + Signature string +} + +// Analyze walks root and extracts everything Render needs. +// +// A missing or partial project is not an error: a repository without a go.mod +// or without a README still yields a usable page describing whatever was found. +func Analyze(name, root string) (*Project, error) { + info, err := os.Stat(root) + if err != nil { + return nil, fmt.Errorf("docgen: stat %s: %w", root, err) + } + if !info.IsDir() { + return nil, fmt.Errorf("docgen: %s is not a directory", root) + } + + p := &Project{Name: name, Path: root} + + files, err := collectFiles(root) + if err != nil { + return nil, err + } + p.Signature = signature(files) + + for _, file := range files { + rel := file.rel + + switch { + case strings.HasSuffix(rel, ".go"): + p.Stats.GoFiles++ + p.Stats.TotalLines += file.lines + if strings.HasSuffix(rel, "_test.go") { + p.Stats.TestFiles++ + p.Stats.TestFuncs += countTestFuncs(file.text()) + } + if isComponentPath(rel) { + p.Stats.Components++ + } + analyzeGoFile(p, file) + + case filepath.Base(rel) == "go.mod": + p.Modules = append(p.Modules, parseGoMod(rel, file.text())) + + case isMarkdown(rel): + p.Docs = append(p.Docs, DocFile{ + Path: rel, + Title: markdownTitle(file.text()), + Bytes: int(file.size), + }) + + case filepath.Base(rel) == "Makefile" || filepath.Base(rel) == "makefile": + p.MakeTargets = append(p.MakeTargets, makeTargets(file.text())...) + + case isDockerfile(rel): + if p.Docker == nil { + p.Docker = parseDockerfile(rel, file.text()) + } + + case isComposeFile(rel): + p.Compose = append(p.Compose, parseCompose(rel, file.text())...) + + case strings.HasPrefix(filepath.Base(rel), ".env"): + p.EnvVars = append(p.EnvVars, envFromExample(file.text())...) + + case isHelmFile(rel): + p.HelmCharts = append(p.HelmCharts, chartRoot(rel)) + } + + p.Languages = addLanguage(p.Languages, file) + if portBearingFiles(rel) { + p.Ports = append(p.Ports, portsIn(file.text())...) + } + } + + finish(p) + return p, nil +} + +// --------------------------------------------------------------------------- +// Normalisation +// --------------------------------------------------------------------------- + +func finish(p *Project) { + p.Readme = findReadme(p) + p.Summary = summarize(p) + + p.Modules = dedupeModules(p.Modules) + p.Routes = dedupeRoutes(p.Routes) + p.Flags = dedupeFlags(p.Flags) + p.EnvVars = dedupeEnv(p.EnvVars) + p.MakeTargets = dedupeStrings(p.MakeTargets) + p.HelmCharts = dedupeStrings(p.HelmCharts) + p.Ports = dedupeStrings(p.Ports) + p.Dirs = topDirs(p) + p.Languages = rankLanguages(p.Languages) +} + +// --------------------------------------------------------------------------- +// Walking +// --------------------------------------------------------------------------- + +type scannedFile struct { + path string + rel string + size int64 + lines int +} + +func (f scannedFile) text() string { + data, err := readCapped(f.path, maxReadBytes) + if err != nil { + return "" + } + return string(data) +} + +// ignoredDirs never contain first-party source worth documenting. +var ignoredDirs = map[string]struct{}{ + "node_modules": {}, ".git": {}, ".idea": {}, ".vscode": {}, "dist": {}, + "build": {}, "bin": {}, "vendor": {}, "coverage": {}, "tmp": {}, + ".cache": {}, ".next": {}, "target": {}, "__pycache__": {}, ".venv": {}, +} + +func collectFiles(root string) ([]scannedFile, error) { + var files []scannedFile + + err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, err error) error { + if err != nil { + // An unreadable subtree must not abort the whole analysis. + if entry != nil && entry.IsDir() { + return fs.SkipDir + } + return nil + } + if len(files) >= maxWalkFiles { + return fs.SkipAll + } + + name := entry.Name() + if entry.IsDir() { + if _, skip := ignoredDirs[name]; skip && path != root { + return fs.SkipDir + } + return nil + } + if !entry.Type().IsRegular() { + return nil + } + if isNoise(name) { + return nil + } + + info, err := entry.Info() + if err != nil { + return nil + } + rel, err := filepath.Rel(root, path) + if err != nil { + rel = name + } + files = append(files, scannedFile{ + path: path, + rel: filepath.ToSlash(rel), + size: info.Size(), + lines: countLinesFast(path), + }) + return nil + }) + if err != nil { + return nil, fmt.Errorf("docgen: walk %s: %w", root, err) + } + return files, nil +} + +// isNoise filters artefacts that would pollute the analysis. +func isNoise(name string) bool { + switch filepath.Ext(name) { + case ".log", ".lock", ".sum", ".pem", ".crt", ".key", ".png", ".jpg", ".jpeg", + ".gif", ".svg", ".ico", ".woff", ".woff2", ".ttf", ".eot", ".zip", ".tar", + ".gz", ".mp4", ".sql.gz", ".db", ".sqlite", ".pyc": + return true + } + // Certificates and keys are frequently committed without extensions. + switch { + case strings.HasSuffix(name, ".crt"), strings.HasSuffix(name, ".key"): + return true + case strings.HasSuffix(name, "-chain.crt"), strings.HasSuffix(name, "-root-ca.crt"): + return true + } + return false +} + +func isMarkdown(rel string) bool { + return strings.EqualFold(filepath.Ext(rel), ".md") +} + +func isDockerfile(rel string) bool { + base := filepath.Base(rel) + return base == "Dockerfile" || strings.HasPrefix(base, "Dockerfile.") +} + +func isComposeFile(rel string) bool { + base := strings.ToLower(filepath.Base(rel)) + if !strings.HasPrefix(base, "docker-compose") && base != "compose.yaml" && base != "compose.yml" { + return false + } + return strings.HasSuffix(base, ".yml") || strings.HasSuffix(base, ".yaml") +} + +func isHelmChart(rel string) bool { + parts := strings.Split(rel, "/") + for i, part := range parts { + if part == "helm" || part == "charts" { + // A chart is a directory that owns a values.yaml or Chart.yaml. + if i+1 < len(parts) { + return true + } + } + } + return false +} + +// chartRoot collapses a chart file path to the directory that owns the chart, +// so the page lists one entry per chart rather than one per template. +func chartRoot(rel string) string { + dir := path.Dir(rel) + if dir == "." { + return "./ (repository root)" + } + return dir +} + +func isComponentPath(rel string) bool { + parts := strings.Split(rel, "/") + if len(parts) < 2 { + return false + } + switch parts[len(parts)-2] { + case "internal", "pkg", "cmd", "api", "services", "web", "ui", "frontend", "backend": + return true + } + return false +} + +func readCapped(path string, limit int64) ([]byte, error) { + file, err := os.Open(path) + if err != nil { + return nil, err + } + defer file.Close() + return io.ReadAll(io.LimitReader(file, limit)) +} + +func countLinesFast(path string) int { + file, err := os.Open(path) + if err != nil { + return 0 + } + defer file.Close() + + // Skip .md / .json / .lock style files where line counts are meaningless. + switch strings.ToLower(filepath.Ext(path)) { + case ".md", ".json", ".yaml", ".yml", ".txt", ".lock", ".css", ".html": + return 0 + } + + buf := make([]byte, 32<<10) + lines := 0 + for { + n, err := file.Read(buf) + if n > 0 { + for _, b := range buf[:n] { + if b == '\n' { + lines++ + } + } + } + if err != nil { + break + } + } + return lines +} + +// signature hashes the identity of every analysed file. It deliberately uses +// size and mtime rather than content so watch mode can poll large trees cheaply. +func signature(files []scannedFile) string { + hash := sha256.New() + for _, file := range files { + info, err := os.Stat(file.path) + if err != nil { + continue + } + fmt.Fprintf(hash, "%s|%d|%d\n", file.rel, file.size, info.ModTime().UnixNano()) + } + return hex.EncodeToString(hash.Sum(nil))[:32] +} + +func countTestFuncs(source string) int { + return strings.Count(source, "func Test") + strings.Count(source, "func Fuzz") + + strings.Count(source, "func Benchmark") +} + +// noiseExtensions are file types that carry no language signal: build +// metadata, ignore lists and editor configuration all have extensions that would +// otherwise show up as their own "language". +var noiseExtensions = map[string]struct{}{ + "mod": {}, "sum": {}, "lock": {}, "gitignore": {}, "dockerignore": {}, + "helmignore": {}, "npmignore": {}, "eslintignore": {}, "gitattributes": {}, + "env": {}, + "editorconfig": {}, "prettierignore": {}, "example": {}, "template": {}, +} + +func addLanguage(langs []Language, file scannedFile) []Language { + ext := strings.ToLower(strings.TrimPrefix(filepath.Ext(file.path), ".")) + if ext == "" { + return langs + } + if _, skip := noiseExtensions[ext]; skip { + return langs + } + name := languageName(ext) + lines := file.lines + if lines == 0 { + lines = countAllLinesFast(file.path) + } + for i := range langs { + if langs[i].Name == name { + langs[i].Files++ + langs[i].Lines += lines + return langs + } + } + return append(langs, Language{Name: name, Files: 1, Lines: lines}) +} + +func countAllLinesFast(path string) int { + data, err := readCapped(path, maxReadBytes) + if err != nil { + return 0 + } + lines := strings.Count(string(data), "\n") + if len(data) > 0 && !strings.HasSuffix(string(data), "\n") { + lines++ + } + return lines +} + +func languageName(ext string) string { + switch ext { + case "go": + return "Go" + case "js", "mjs", "cjs": + return "JavaScript" + case "jsx": + return "JSX" + case "ts", "tsx": + return "TypeScript" + case "css": + return "CSS" + case "html", "htm": + return "HTML" + case "sh", "bash": + return "Shell" + case "sql": + return "SQL" + case "yml", "yaml": + return "YAML" + case "json": + return "JSON" + case "md": + return "Markdown" + case "tf": + return "Terraform" + case "proto": + return "Protobuf" + case "tpl", "gotmpl": + return "Templates" + case "py": + return "Python" + default: + return ext + } +} + +func rankLanguages(langs []Language) []Language { + sort.Slice(langs, func(i, j int) bool { + if langs[i].Lines == langs[j].Lines { + return langs[i].Name < langs[j].Name + } + return langs[i].Lines > langs[j].Lines + }) + // A page does not benefit from a tail of one-file languages. + if len(langs) > maxLanguages { + langs = langs[:maxLanguages] + } + return langs +} + +// maxLanguages caps the language table on a generated page. +const maxLanguages = 10 + +func topDirs(p *Project) []Dir { + byName := map[string]*Dir{} + err := filepath.WalkDir(p.Path, func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return nil + } + if entry.IsDir() { + if path != p.Path { + if _, skip := ignoredDirs[entry.Name()]; skip { + return fs.SkipDir + } + // Dot directories hold tooling, not first-party source. + if strings.HasPrefix(entry.Name(), ".") { + return fs.SkipDir + } + } + return nil + } + rel, err := filepath.Rel(p.Path, path) + if err != nil { + return nil + } + parts := strings.Split(filepath.ToSlash(rel), "/") + if len(parts) < 2 { + return nil + } + dir := byName[parts[0]] + if dir == nil { + dir = &Dir{Name: parts[0]} + byName[parts[0]] = dir + } + dir.Files++ + dir.Lines += countLinesFast(path) + return nil + }) + if err != nil { + return nil + } + + dirs := make([]Dir, 0, len(byName)) + for _, dir := range byName { + dirs = append(dirs, *dir) + } + sort.Slice(dirs, func(i, j int) bool { + if dirs[i].Files == dirs[j].Files { + return dirs[i].Name < dirs[j].Name + } + return dirs[i].Files > dirs[j].Files + }) + if len(dirs) > 14 { + dirs = dirs[:14] + } + return dirs +} + +func max(a, b int) int { + if a > b { + return a + } + return b +} + +func dedupeStrings(in []string) []string { + seen := map[string]struct{}{} + out := make([]string, 0, len(in)) + for _, value := range in { + value = strings.TrimSpace(value) + if value == "" { + continue + } + if _, dup := seen[value]; dup { + continue + } + seen[value] = struct{}{} + out = append(out, value) + } + sort.Strings(out) + return out +} + +// dedupeInOrder removes duplicates while preserving first-seen order, for lists +// whose order is meaningful (Docker build stages, published ports). +func dedupeInOrder(in []string) []string { + seen := map[string]struct{}{} + out := make([]string, 0, len(in)) + for _, value := range in { + value = strings.TrimSpace(value) + if value == "" { + continue + } + if _, dup := seen[value]; dup { + continue + } + seen[value] = struct{}{} + out = append(out, value) + } + return out +} + +func dedupeModules(mods []Module) []Module { + sort.Slice(mods, func(i, j int) bool { return mods[i].File < mods[j].File }) + return mods +} + +// dedupeRoutes collapses registrations that describe the same endpoint. The +// key is method plus path, because one endpoint is often registered in several +// places (a group helper plus the router); when several carry a handler name, +// the one with a handler wins over the one without. +func dedupeRoutes(routes []Route) []Route { + best := make(map[string]Route, len(routes)) + for _, route := range routes { + if route.Path == "" { + continue + } + key := route.Method + " " + route.Path + existing, seen := best[key] + switch { + case !seen: + best[key] = route + case existing.Handler == "" && route.Handler != "": + best[key] = route + } + } + + out := make([]Route, 0, len(best)) + for _, route := range best { + out = append(out, route) + } + sort.Slice(out, func(i, j int) bool { + if out[i].Path == out[j].Path { + return out[i].Method < out[j].Method + } + return out[i].Path < out[j].Path + }) + return out +} + +func dedupeFlags(flags []Flag) []Flag { + seen := map[string]struct{}{} + out := make([]Flag, 0, len(flags)) + for _, flag := range flags { + if _, dup := seen[flag.Name]; dup { + continue + } + seen[flag.Name] = struct{}{} + out = append(out, flag) + } + sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) + return out +} + +func dedupeEnv(vars []EnvVar) []EnvVar { + seen := map[string]struct{}{} + out := make([]EnvVar, 0, len(vars)) + for _, v := range vars { + if _, dup := seen[v.Name]; dup { + continue + } + seen[v.Name] = struct{}{} + out = append(out, v) + } + sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) + return out +} + +// Timestamp is used inside generated markers so a reader can tell how fresh a +// machine-written region is. +func Timestamp() string { return time.Now().UTC().Format("2006-01-02 15:04 UTC") } diff --git a/backend/docgen/docgen_test.go b/backend/docgen/docgen_test.go new file mode 100644 index 0000000..3bea8d9 --- /dev/null +++ b/backend/docgen/docgen_test.go @@ -0,0 +1,621 @@ +package docgen + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// fixture writes a small but representative project tree: all three route +// registration styles, a module file, a Makefile, a Dockerfile, a compose file, +// an .env, a README and one unrelated Markdown document. +func fixture(t *testing.T) string { + t.Helper() + + root := t.TempDir() + files := map[string]string{ + "README.md": "# Demo Service\n\nDemo is a service that answers questions about things. " + + "It runs behind Go Loose and exposes a JSON API.\n\n" + + "## Install\n\nNot part of the summary.\n", + + "go.mod": "module github.com/danyel/demo\n\ngo 1.27.1\n\nrequire (\n" + + "\tgithub.com/go-chi/chi/v5 v5.2.3\n\tgithub.com/spf13/cobra v1.9.1\n" + + "\tgolang.org/x/sync v0.14.0 // indirect\n)\n", + + "Makefile": "build:\n\tgo build ./...\n\n.PHONY: build\n\ntest:\n\tgo test ./...\n", + + "Dockerfile": "FROM golang:1.27-alpine AS builder\nRUN go build ./...\n\n" + + "FROM alpine:3.20\nCOPY --from=builder /app/demo /demo\nEXPOSE 8080\n", + + "docker-compose.yml": "services:\n api:\n image: demo:latest\n ports:\n" + + " - \"8080:8080\"\n db:\n image: postgres:17\n environment:\n" + + " POSTGRES_PASSWORD: secret\n", + + ".env.example": "# comment\nDEMO_HTTP_PORT=8080\nDEMO_API_KEY=\n", + + "internal/server/routes.go": "package server\n\n" + + "import (\n\t\"net/http\"\n\t\"os\"\n)\n\n" + + "func Routes(mux *http.ServeMux, r chi.Router) {\n" + + "\tmux.HandleFunc(\"GET /healthz\", health)\n" + + "\tmux.HandleFunc(\"/plain\", plainHandler)\n" + + "\tregister(r, http.MethodPost, \"/api/items\", createItem)\n" + + "\tr.Get(\"/api/items/{id}\", getItem)\n" + + "\t_ = os.Getenv(\"DEMO_RUNTIME_SECRET\")\n}\n", + + "internal/server/routes_test.go": "package server\n\n" + + "func TestRoutes(t *testing.T) {\n" + + "\t_ = httptest.NewRequest(http.MethodGet, \"/api/jobs\", nil)\n}\n", + + "cmd/demo/main.go": "package main\n\nimport \"flag\"\n\nfunc main() {\n" + + "\tport := flag.String(\"port\", \"8080\", \"listen port\")\n" + + "\tverbose := flag.Bool(\"verbose\", false, \"chatty output\")\n" + + "\t_ = port\n\t_ = verbose\n}\n", + + "docs/OPERATIONS.md": "# Operations\n\nHow to run the thing.\n", + } + + for name, content := range files { + path := filepath.Join(root, filepath.FromSlash(name)) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("mkdir %s: %v", path, err) + } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatalf("write %s: %v", path, err) + } + } + + // A directory that must never be walked. + if err := os.MkdirAll(filepath.Join(root, "node_modules", "left-pad"), 0o755); err != nil { + t.Fatalf("mkdir node_modules: %v", err) + } + if err := os.WriteFile(filepath.Join(root, "node_modules", "index.js"), []byte("x\n"), 0o644); err != nil { + t.Fatalf("write node_modules file: %v", err) + } + + return root +} + +func analyzeFixture(t *testing.T) *Project { + t.Helper() + p, err := Analyze("demo", fixture(t)) + if err != nil { + t.Fatalf("Analyze: %v", err) + } + return p +} + +func TestAnalyzeExtractsModuleMetadata(t *testing.T) { + p := analyzeFixture(t) + + if len(p.Modules) != 1 { + t.Fatalf("want 1 module, got %d: %+v", len(p.Modules), p.Modules) + } + module := p.Modules[0] + if module.Path != "github.com/danyel/demo" { + t.Errorf("module path = %q", module.Path) + } + if module.GoVersion != "1.27.1" { + t.Errorf("go version = %q", module.GoVersion) + } + if len(module.Direct) != 2 { + t.Errorf("want 2 direct deps, got %d: %+v", len(module.Direct), module.Direct) + } + if module.Direct[0].Path != "github.com/go-chi/chi/v5" || module.Direct[0].Version != "v5.2.3" { + t.Errorf("unexpected first dep: %+v", module.Direct[0]) + } + if module.Indirect != 1 { + t.Errorf("want 1 indirect dep, got %d", module.Indirect) + } +} + +func TestAnalyzeExtractsAllRouteStyles(t *testing.T) { + p := analyzeFixture(t) + + want := map[string]string{ + "GET /healthz": "health", + "/plain": "plainHandler", + "POST /api/items": "createItem", + "GET /api/items/{id}": "getItem", + } + for key, handler := range want { + var found *Route + for i := range p.Routes { + if p.Routes[i].String() == key { + found = &p.Routes[i] + } + } + if found == nil { + t.Errorf("route %q not found; got %v", key, routeStrings(p.Routes)) + continue + } + if found.Handler != handler { + t.Errorf("route %q handler = %q, want %q", key, found.Handler, handler) + } + } + if len(p.Routes) != len(want) { + t.Errorf("want %d routes, got %d: %v", len(want), len(p.Routes), routeStrings(p.Routes)) + } +} + +func TestAnalyzeIgnoresRoutesInTestFiles(t *testing.T) { + p := analyzeFixture(t) + + for _, route := range p.Routes { + if route.Path == "/api/jobs" { + t.Fatalf("route from a _test.go file leaked into the analysis: %v", route) + } + } +} + +func TestAnalyzeExtractsFlagsAndEnvironment(t *testing.T) { + p := analyzeFixture(t) + + flags := map[string]Flag{} + for _, flag := range p.Flags { + flags[flag.Name] = flag + } + if flags["port"].Default != "8080" || flags["port"].Usage != "listen port" { + t.Errorf("port flag = %+v", flags["port"]) + } + if flags["verbose"].Type != "bool" { + t.Errorf("verbose flag = %+v", flags["verbose"]) + } + + env := map[string]EnvVar{} + for _, v := range p.EnvVars { + env[v.Name] = v + } + if env["DEMO_RUNTIME_SECRET"].FromEnv { + t.Error("os.Getenv should not be reported as coming from a .env file") + } + if !env["DEMO_HTTP_PORT"].FromEnv || env["DEMO_HTTP_PORT"].Example != "8080" { + t.Errorf("DEMO_HTTP_PORT = %+v", env["DEMO_HTTP_PORT"]) + } +} + +func TestAnalyzeExtractsOperationalFacts(t *testing.T) { + p := analyzeFixture(t) + + if got := p.MakeTargets; len(got) != 2 || got[0] != "build" || got[1] != "test" { + t.Errorf("make targets = %v", got) + } + if p.Docker == nil { + t.Fatal("no Dockerfile analysed") + } + if len(p.Docker.Bases) != 2 || p.Docker.Bases[0] != "golang:1.27-alpine" { + t.Errorf("docker bases = %v", p.Docker.Bases) + } + if len(p.Docker.Exposed) != 1 || p.Docker.Exposed[0] != "8080" { + t.Errorf("docker exposed = %v", p.Docker.Exposed) + } + if len(p.Compose) != 2 { + t.Fatalf("compose services = %+v", p.Compose) + } + if p.Compose[0].Name != "api" || p.Compose[0].Image != "demo:latest" { + t.Errorf("compose[0] = %+v", p.Compose[0]) + } + if len(p.Ports) == 0 { + t.Error("expected the compose file port to be picked up") + } + if p.Readme != "README.md" { + t.Errorf("readme = %q", p.Readme) + } + if len(p.Docs) != 2 { + t.Errorf("docs = %+v", p.Docs) + } +} + +func TestAnalyzeIgnoresVendoredTrees(t *testing.T) { + p := analyzeFixture(t) + + for _, lang := range p.Languages { + if lang.Name == "JavaScript" { + t.Errorf("node_modules leaked into the language table: %+v", p.Languages) + } + } + for _, dir := range p.Dirs { + if dir.Name == "node_modules" { + t.Errorf("node_modules leaked into the directory map: %+v", dir) + } + } +} + +func TestSummaryComesFromReadmeProse(t *testing.T) { + p := analyzeFixture(t) + + if !strings.HasPrefix(p.Summary, "Demo is a service") { + t.Errorf("summary = %q", p.Summary) + } + if strings.Contains(p.Summary, "Install") { + t.Errorf("summary leaked a later section: %q", p.Summary) + } +} + +func TestAnalyzeWithoutReadmeFallsBackToFacts(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "main.go"), []byte("package main\n"), 0o644); err != nil { + t.Fatal(err) + } + + p, err := Analyze("bare", root) + if err != nil { + t.Fatalf("Analyze: %v", err) + } + if p.Summary == "" { + t.Error("a project with no README still needs a summary") + } + if p.Stats.GoFiles != 1 { + t.Errorf("go files = %d", p.Stats.GoFiles) + } +} + +func TestSignatureChangesOnlyWithTheTree(t *testing.T) { + root := fixture(t) + + first, err := Analyze("demo", root) + if err != nil { + t.Fatal(err) + } + again, err := Analyze("demo", root) + if err != nil { + t.Fatal(err) + } + if first.Signature != again.Signature { + t.Errorf("signature is not stable: %q vs %q", first.Signature, again.Signature) + } + + if err := os.WriteFile(filepath.Join(root, "notes.txt"), []byte("hello\n"), 0o644); err != nil { + t.Fatal(err) + } + changed, err := Analyze("demo", root) + if err != nil { + t.Fatal(err) + } + if changed.Signature == first.Signature { + t.Error("signature did not change after a file was added") + } +} + +// --------------------------------------------------------------------------- +// Rendering +// --------------------------------------------------------------------------- + +func TestRenderProducesAudienceSections(t *testing.T) { + page, err := Render(analyzeFixture(t)) + if err != nil { + t.Fatalf("Render: %v", err) + } + + for _, want := range []string{ + "# demo", + "## For Users", + "## For Operators", + "## For Developers", + "### HTTP API", + "| Method | Path | Handler |", + "| `GET` | `/healthz` | `health` |", + "### Direct dependencies", + "`github.com/go-chi/chi/v5` v5.2.3", + "### Network surface", + "`make build`", + } { + if !strings.Contains(page, want) { + t.Errorf("rendered page is missing %q", want) + } + } +} + +func TestRenderIsWrappedInMarkers(t *testing.T) { + page, err := Render(analyzeFixture(t)) + if err != nil { + t.Fatalf("Render: %v", err) + } + + if !strings.HasPrefix(page, beginPrefix) { + t.Errorf("page does not start with the opening marker:\n%s", firstLine(page)) + } + if !strings.HasSuffix(page, EndMarker) { + t.Errorf("page does not end with the closing marker:\n%s", lastLine(page)) + } + if !FindBlock(page).Found { + t.Error("FindBlock cannot locate the markers on a freshly rendered page") + } +} + +func TestRenderKeepsMarkdownSpacingSane(t *testing.T) { + page, err := Render(analyzeFixture(t)) + if err != nil { + t.Fatalf("Render: %v", err) + } + + // Trim the marker lines: they are not part of the Markdown body. + body := page + if block := FindBlock(page); block.Found { + body = page[block.Body : block.End-len(EndMarker)] + } + + lines := strings.Split(body, "\n") + for i := 0; i < len(lines)-1; i++ { + if lines[i] == "" && lines[i+1] == "" { + t.Errorf("blank line %d: two blank lines in a row", i+1) + } + if strings.HasPrefix(lines[i], "#") && lines[i+1] != "" { + t.Errorf("heading on line %d is glued to %q", i+1, lines[i+1]) + } + } + if strings.HasPrefix(strings.TrimSpace(lines[0]), "\n\nBody.\n", + "only a closing marker": "Body.\n" + EndMarker + "\n", + "marker not at line start": "text " + beginPrefix + " -->\nBody.\n" + EndMarker + "\n", + "markers out of order": EndMarker + "\n" + beginPrefix + " -->\n", + } + + for name, content := range tests { + if block := FindBlock(content); block.Found { + t.Errorf("%s: expected no block, got %+v", name, block) + } + } +} + +func TestFindBlockReportsBoundaries(t *testing.T) { + page, err := Render(analyzeFixture(t)) + if err != nil { + t.Fatal(err) + } + + block := FindBlock(page) + if !block.Found { + t.Fatal("block not found") + } + if block.Marker != 0 { + t.Errorf("marker offset = %d, want 0", block.Marker) + } + if !strings.HasPrefix(page[block.Body:], "# demo") { + t.Errorf("body does not start after the opening marker line: %q", page[block.Body:block.Body+8]) + } + if got := page[block.End:]; got != "" { + t.Errorf("bytes after the block = %q, want none", got) + } +} + +func TestWrapNormalisesTrailingNewlines(t *testing.T) { + page := Wrap("demo", "sig", "body\n\n\n") + if strings.HasSuffix(page, "\n") { + t.Error("Wrap must not end with a newline") + } + if !strings.HasSuffix(page, "\n"+EndMarker) { + t.Errorf("the closing marker must start its own line: %q", lastLine(page)) + } +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +func routeStrings(routes []Route) []string { + out := make([]string, 0, len(routes)) + for _, route := range routes { + out = append(out, route.String()) + } + return out +} + +func firstLine(s string) string { + if i := strings.IndexByte(s, '\n'); i >= 0 { + return s[:i] + } + return s +} + +func lastLine(s string) string { + if i := strings.LastIndexByte(s, '\n'); i >= 0 { + return s[i+1:] + } + return s +} + +// TestMergeIgnoresAMinuteOldTimestamp is the regression test for a bug that +// only showed up once a minute: two renders of an unchanged project carry +// different "generated=" stamps, and a pattern that stopped at the first space +// left "15:04 UTC" in place, so every sync in a new minute rewrote all four +// pages and reported "updated" for content that had not changed at all. +func TestMergeIgnoresAMinuteOldTimestamp(t *testing.T) { + page, err := Render(analyzeFixture(t)) + if err != nil { + t.Fatal(err) + } + + // Simulate a page imported an hour ago, down to the stamp. + stale := strings.Replace(page, "generated="+Timestamp(), "generated=2001-01-01 00:00 UTC", 1) + if stale == page { + t.Fatal("could not age the timestamp; the marker format changed") + } + + merged, status, err := Merge(stale, page, false) + if err != nil { + t.Fatal(err) + } + if status != StatusUnchanged { + t.Errorf("status = %q, want unchanged for an hour-old page with identical content", status) + } + if merged != stale { + t.Error("an unchanged sync must return the file untouched, timestamp included") + } + + // A genuine content change must still be reported, even with an old stamp. + edited := strings.Replace(stale, "# demo", "# demo, revised", 1) + if edited == stale { + t.Fatal("could not edit the title") + } + merged, status, err = Merge(stale, edited, false) + if err != nil { + t.Fatal(err) + } + if status != StatusUpdated { + t.Errorf("status = %q, want updated for a real content change", status) + } + if !strings.Contains(merged, "# demo, revised") { + t.Error("the edit did not reach the merged page") + } +} + +// TestWrapStampIsFullyStripped guards the pattern itself: every byte of the +// timestamp has to be removed, not just the date. +func TestWrapStampIsFullyStripped(t *testing.T) { + page, err := Render(analyzeFixture(t)) + if err != nil { + t.Fatal(err) + } + stripped := stampPattern.ReplaceAllString(page, stampReplacement) + if strings.Contains(stripped, " UTC") { + t.Errorf("the timestamp was not fully stripped: %q", firstLineOf(stripped)) + } + if strings.Count(stripped, "-->") != strings.Count(page, "-->") { + t.Error("stripping the stamp swallowed more than the timestamp") + } +} + +func firstLineOf(s string) string { + if i := strings.IndexByte(s, '\n'); i >= 0 { + return s[:i] + } + return s +} diff --git a/backend/docgen/extract.go b/backend/docgen/extract.go new file mode 100644 index 0000000..4900206 --- /dev/null +++ b/backend/docgen/extract.go @@ -0,0 +1,560 @@ +package docgen + +import ( + "os" + "path/filepath" + "regexp" + "strconv" + "strings" +) + +// --------------------------------------------------------------------------- +// Go source extraction +// --------------------------------------------------------------------------- + +var ( + // net/http style registrations, including the Go 1.22 method-prefixed form + // mux.HandleFunc("GET /health", h). The literal is captured whole and + // validated in parseRoute, which is where the method prefix is split off. + routeHandleCall = regexp.MustCompile(`\b(?:Handle|HandleFunc)\(\s*"([^"]+)"\s*(?:,\s*([A-Za-z0-9_.]+))?`) + + // Route tables that pass the method as an http.MethodXxx constant, as + // go-guess does: register(router, spec, false, http.MethodGet, "/api/health", h) + routeMethodConst = regexp.MustCompile(`\bhttp\.Method([A-Za-z]+)\s*,\s*"(/[^"]*)"\s*(?:,\s*([A-Za-z0-9_.]+)([,)]))?`) + + // Router-value registrations: r.Get("/api/x", h.X) + routeRouterCall = regexp.MustCompile(`\b\w+\.(Get|Post|Put|Patch|Delete|Head|Options|Trace)\(\s*"(/[^"]*)"\s*(?:,\s*([A-Za-z0-9_.]+)([,)]))?`) + + // "GET /api/v1/x" as a literal argument. + methodPrefix = regexp.MustCompile(`^(GET|POST|PUT|PATCH|DELETE|HEAD|OPTIONS)\s+(\S.*)$`) + + // flag.String("name", default, "usage") + flagCall = regexp.MustCompile(`flag\.(\w+)\(\s*"([^"]+)"\s*(?:,\s*([^,)]+))?\s*(?:,\s*"([^"]*)")?`) + + // os.Getenv("X") / os.LookupEnv("X") + envCall = regexp.MustCompile(`os\.(?:Getenv|LookupEnv)\(\s*"([A-Z][A-Z0-9_]*)"`) + + // Port literals anywhere in the project. + portLiteral = regexp.MustCompile(`(?:\bPORT\b|\bport\b|EXPOSE)[^0-9]{0,20}([0-9]{2,5})\b`) + + testFuncDecl = regexp.MustCompile(`(?m)^func\s+(Test|Benchmark|Fuzz)\w*\(`) +) + +// analyzeGoFile pulls routes, flags and environment variables out of one source +// file. Each extractor is independent: a syntax oddity in one never affects the +// others because these are textual scans, not a type-checked AST walk. +// +// Test files are excluded. They build requests with httptest.NewRequest, which +// looks exactly like a route registration, and a route table is never declared +// in a _test.go file. +func analyzeGoFile(p *Project, file scannedFile) { + if strings.HasSuffix(file.rel, "_test.go") { + return + } + + source := file.text() + if source == "" { + return + } + + for _, match := range routeHandleCall.FindAllStringSubmatch(source, -1) { + if route := parseRoute(match[1], match[2]); route.Path != "" { + p.Routes = append(p.Routes, route) + } + } + for _, match := range routeMethodConst.FindAllStringSubmatch(source, -1) { + p.Routes = append(p.Routes, Route{ + Method: strings.ToUpper(match[1]), + Path: match[2], + Handler: handlerName(match[3]), + }) + } + for _, match := range routeRouterCall.FindAllStringSubmatch(source, -1) { + p.Routes = append(p.Routes, Route{ + Method: strings.ToUpper(match[1]), + Path: match[2], + Handler: handlerName(match[3]), + }) + } + + for _, match := range flagCall.FindAllStringSubmatch(source, -1) { + p.Flags = append(p.Flags, Flag{ + Name: match[2], + Type: strings.ToLower(match[1]), + Default: unquote(strings.TrimSpace(match[3])), + Usage: strings.TrimSpace(match[4]), + }) + } + + for _, match := range envCall.FindAllStringSubmatch(source, -1) { + p.EnvVars = append(p.EnvVars, EnvVar{Name: match[1]}) + } +} + +// parseRoute splits an optional method prefix and normalises the path. A +// literal that is not a route — a query key, a header name, a MIME type — is +// rejected by requiring the path to look like one. +func parseRoute(raw, handler string) Route { + method, path := "", strings.TrimSpace(raw) + if match := methodPrefix.FindStringSubmatch(raw); match != nil { + method, path = match[1], strings.TrimSpace(match[2]) + } + if !strings.HasPrefix(path, "/") { + return Route{} + } + return Route{Method: method, Path: path, Handler: handlerName(handler)} +} + +// handlerNames that are never useful in a route table: they are literal +// arguments, receivers or framework values rather than handler functions. +var noiseHandlers = map[string]struct{}{ + "nil": {}, "true": {}, "false": {}, "spec": {}, "router": {}, "mux": {}, + "r": {}, "g": {}, "chi": {}, "group": {}, "protected": {}, "public": {}, + "api": {}, "v1": {}, "sub": {}, "app": {}, "server": {}, "srv": {}, + "ctx": {}, "c": {}, "w": {}, "req": {}, "request": {}, "http": {}, +} + +// handlerName reduces a handler expression to a readable name, dropping the +// receiver and rejecting anything that is not plainly a function reference. +func handlerName(handler string) string { + handler = strings.TrimSpace(handler) + if handler == "" { + return "" + } + if i := strings.LastIndex(handler, "."); i >= 0 { + handler = handler[i+1:] + } + handler = strings.TrimSuffix(handler, ".func1") + if _, noise := noiseHandlers[handler]; noise { + return "" + } + if handler == "HandlerFunc" || strings.Contains(handler, "(") { + return "" + } + return handler +} + +// --------------------------------------------------------------------------- +// go.mod +// --------------------------------------------------------------------------- + +var ( + modLine = regexp.MustCompile(`^module\s+(\S+)`) + goVersionLine = regexp.MustCompile(`^go\s+(\S+)`) + requireLine = regexp.MustCompile(`^\s*(\S+)\s+(v\S+)(.*)$`) +) + +// parseGoMod reads a go.mod outside any require block. +func parseGoMod(rel, source string) Module { + module := Module{File: rel} + inBlock := false + + for _, raw := range strings.Split(source, "\n") { + line := strings.TrimRight(raw, " \t") + trimmed := strings.TrimSpace(line) + + switch { + case strings.HasPrefix(trimmed, "module "): + if module.Path == "" { + module.Path = strings.TrimSpace(strings.TrimPrefix(trimmed, "module ")) + } + case strings.HasPrefix(trimmed, "go "): + if module.GoVersion == "" { + if match := goVersionLine.FindStringSubmatch(trimmed); match != nil { + module.GoVersion = match[1] + } + } + case trimmed == "require (": + inBlock = true + case inBlock && trimmed == ")": + inBlock = false + case inBlock && trimmed != "" && !strings.HasPrefix(trimmed, "//"): + match := requireLine.FindStringSubmatch(line) + if match != nil { + addDependency(&module, match[1], match[2], strings.Contains(match[3], "// indirect")) + } + case strings.HasPrefix(trimmed, "require ") && !inBlock: + rest := strings.TrimSpace(strings.TrimPrefix(trimmed, "require ")) + if match := requireLine.FindStringSubmatch(" " + rest); match != nil { + addDependency(&module, match[1], match[2], strings.Contains(match[3], "// indirect")) + } + } + } + return module +} + +func addDependency(module *Module, path, version string, indirect bool) { + if path == "" || strings.HasPrefix(path, "//") { + return + } + if indirect { + module.Indirect++ + return + } + module.Direct = append(module.Direct, Dependency{Path: path, Version: version}) +} + +// --------------------------------------------------------------------------- +// Makefile +// --------------------------------------------------------------------------- + +var makeTargetLine = regexp.MustCompile(`^([A-Za-z0-9][A-Za-z0-9_.-]*):`) + +// makeTargets extracts invocable targets, skipping .PHONY declarations and +// variable assignments. +func makeTargets(source string) []string { + var targets []string + phony := map[string]struct{}{} + + for _, raw := range strings.Split(source, "\n") { + if strings.HasPrefix(raw, "\t") || strings.HasPrefix(raw, " ") { + continue + } + line := strings.TrimRight(raw, " \t") + if strings.HasPrefix(line, ".PHONY") { + for _, name := range strings.Fields(line) { + if name == ".PHONY:" || name == ".PHONY" { + continue + } + phony[strings.TrimSuffix(name, ":")] = struct{}{} + } + continue + } + match := makeTargetLine.FindStringSubmatch(line) + if match == nil { + continue + } + name := match[1] + if strings.HasPrefix(name, ".") { + continue + } + if _, isPhony := phony[name]; isPhony { + continue + } + targets = append(targets, name) + } + return targets +} + +// --------------------------------------------------------------------------- +// Docker +// --------------------------------------------------------------------------- + +var ( + dockerFrom = regexp.MustCompile(`(?mi)^\s*FROM\s+(\S+)(?:\s+AS\s+(\S+))?`) + dockerExp = regexp.MustCompile(`(?mi)^\s*EXPOSE\s+(\S+)`) +) + +func parseDockerfile(rel, source string) *DockerInfo { + info := &DockerInfo{File: rel} + for _, match := range dockerFrom.FindAllStringSubmatch(source, -1) { + info.Bases = append(info.Bases, match[1]) + if match[2] != "" { + info.Stages = append(info.Stages, match[2]) + } + } + for _, match := range dockerExp.FindAllStringSubmatch(source, -1) { + info.Exposed = append(info.Exposed, match[1]) + } + // Order matters: the first FROM is the build stage and the last is the + // runtime image, so these lists are deduplicated but never sorted. + info.Bases = dedupeInOrder(info.Bases) + info.Stages = dedupeInOrder(info.Stages) + info.Exposed = dedupeInOrder(info.Exposed) + return info +} + +// --------------------------------------------------------------------------- +// docker-compose +// --------------------------------------------------------------------------- + +// parseCompose reads service names, images and published ports out of a compose +// file. It is a line scanner rather than a YAML parser: compose files in these +// projects are consistently indented two spaces per level, and avoiding a YAML +// dependency keeps the module dependency-free. +func parseCompose(rel, source string) []ComposeService { + var ( + services []ComposeService + inBlock bool + current int = -1 + ) + + for _, raw := range strings.Split(source, "\n") { + line := strings.TrimRight(raw, " \t") + trimmed := strings.TrimSpace(line) + if trimmed == "" || strings.HasPrefix(trimmed, "#") { + continue + } + + indent := len(line) - len(strings.TrimLeft(line, " ")) + + if indent == 0 { + inBlock = trimmed == "services:" + current = -1 + continue + } + if !inBlock { + continue + } + + if indent == 2 && strings.HasSuffix(trimmed, ":") { + services = append(services, ComposeService{Name: strings.TrimSuffix(trimmed, ":"), File: rel}) + current = len(services) - 1 + continue + } + if current < 0 || indent != 4 { + continue + } + + key, value, found := strings.Cut(trimmed, ":") + if !found { + continue + } + switch strings.TrimSpace(key) { + case "image": + services[current].Image = strings.Trim(strings.TrimSpace(value), `"'`) + case "ports": + if services[current].Ports == "" { + services[current].Ports = strings.Trim(strings.TrimSpace(value), `"'`) + } else { + services[current].Ports += ", " + strings.Trim(strings.TrimSpace(value), `"'`) + } + } + } + + // Drop the well-known internal networks and volume declarations that the + // indentation scanner otherwise mistakes for services. + names := map[string]struct{}{} + for _, name := range []string{ + "networks", "volumes", "configs", "secrets", "services", + } { + names[name] = struct{}{} + } + + out := services[:0] + for _, service := range services { + if _, skip := names[service.Name]; skip { + continue + } + out = append(out, service) + } + return out +} + +// --------------------------------------------------------------------------- +// .env files +// --------------------------------------------------------------------------- + +// unquote strips the quoting Go string literals carry around literal values. +func unquote(value string) string { + if len(value) >= 2 { + if (value[0] == '"' && value[len(value)-1] == '"') || + (value[0] == '`' && value[len(value)-1] == '`') { + return value[1 : len(value)-1] + } + } + return value +} + +// envFromExample reads KEY=VALUE pairs out of a .env or .env.example file. +func envFromExample(source string) []EnvVar { + var out []EnvVar + for _, raw := range strings.Split(source, "\n") { + line := strings.TrimSpace(raw) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + key, value, found := strings.Cut(line, "=") + if !found { + continue + } + key = strings.TrimSpace(key) + if key == "" || !strings.ContainsAny(key, "ABCDEFGHIJKLMNOPQRSTUVWXYZ") { + continue + } + out = append(out, EnvVar{Name: key, Example: strings.Trim(strings.TrimSpace(value), `"'`), FromEnv: true}) + } + return out +} + +// --------------------------------------------------------------------------- +// Misc +// --------------------------------------------------------------------------- + +// portsIn extracts plausible TCP port literals from any file. +func portsIn(source string) []string { + if source == "" { + return nil + } + var out []string + for _, match := range portLiteral.FindAllStringSubmatch(source, -1) { + port, err := strconv.Atoi(match[1]) + if err != nil || port < 1024 || port > 65535 { + continue + } + out = append(out, match[1]) + } + return out +} + +// portBearingFiles reports whether a file is a plausible source of a declared +// port. Scanning every source file for port literals produces noise (version +// numbers, row counts), so only operational files are considered. +func portBearingFiles(rel string) bool { + base := strings.ToLower(filepath.Base(rel)) + switch { + case isDockerfile(rel), isComposeFile(rel), isHelmFile(rel): + return true + case base == "makefile", base == "makefile.am": + return true + case base == "readme.md": + return true + case strings.HasPrefix(base, ".env"): + return true + case strings.HasSuffix(base, ".service"), strings.HasSuffix(base, ".env.example"): + return true + default: + return false + } +} + +// isHelmFile reports whether a path is part of a Helm chart, anchored on the two +// files that define one. +func isHelmFile(rel string) bool { + base := strings.ToLower(filepath.Base(rel)) + return base == "chart.yaml" || base == "values.yaml" +} + +// markdownTitle returns the first heading of a Markdown document. +func markdownTitle(source string) string { + for _, raw := range strings.Split(source, "\n") { + line := strings.TrimSpace(raw) + if strings.HasPrefix(line, "#") { + return strings.TrimSpace(strings.TrimLeft(line, "#")) + } + if line != "" { + return "" + } + } + return "" +} + +// findReadme locates the project's README, preferring the repository root. +func findReadme(p *Project) string { + for _, candidate := range []string{"README.md", "readme.md", "README.markdown", "Readme.md"} { + if info, err := os.Stat(filepath.Join(p.Path, candidate)); err == nil && !info.IsDir() { + return candidate + } + } + for _, doc := range p.Docs { + if strings.EqualFold(filepath.Base(doc.Path), "readme.md") { + return doc.Path + } + } + return "" +} + +// summarize produces the one-paragraph preamble shown above the audience tabs. +// It prefers the README's opening prose and falls back to a description derived +// from what the analyser found, so a project without a README still gets one. +func summarize(p *Project) string { + if p.Readme != "" { + if data, err := readCapped(p.Path+"/"+p.Readme, maxReadBytes); err == nil { + if summary := firstParagraph(string(data)); summary != "" { + return collapse(summary, 460) + } + } + } + + parts := []string{} + if module := primaryModule(p); module != "" { + parts = append(parts, "Go module `"+module+"`") + } + if p.Stats.GoFiles > 0 { + parts = append(parts, plural(p.Stats.GoFiles, "Go source file", "Go source files")) + } + if len(p.Routes) > 0 { + parts = append(parts, plural(len(p.Routes), "HTTP endpoint", "HTTP endpoints")) + } + if len(p.EnvVars) > 0 { + parts = append(parts, plural(len(p.EnvVars), "environment variable", "environment variables")) + } + if len(parts) == 0 { + return "This project has no README yet. The sections below were derived from its structure on disk." + } + return "This project has no README yet. The analyser found " + joinList(parts) + "." +} + +// firstParagraph returns the opening prose of a document: the first run of +// non-empty lines that is not a heading, badge, table or HTML block. +func firstParagraph(source string) string { + var lines []string + + for _, raw := range strings.Split(source, "\n") { + trimmed := strings.TrimSpace(raw) + + if strings.HasPrefix(trimmed, "```") { + break // prose never starts after a code fence in these READMEs + } + switch { + case trimmed == "": + if len(lines) > 0 { + return strings.Join(lines, " ") + } + case strings.HasPrefix(trimmed, "#"): + continue + case strings.HasPrefix(trimmed, "!"), strings.HasPrefix(trimmed, "[!"), + strings.HasPrefix(trimmed, "<"), strings.HasPrefix(trimmed, "|"): + continue // badge, HTML or table row + default: + lines = append(lines, trimmed) + } + } + return strings.Join(lines, " ") +} + +// collapse folds whitespace and truncates on a word boundary. +func collapse(text string, limit int) string { + fields := strings.Fields(text) + if len(fields) == 0 { + return "" + } + out := strings.Join(fields, " ") + if len(out) <= limit { + return out + } + cut := out[:limit] + if i := strings.LastIndexByte(cut, ' '); i > limit/2 { + cut = cut[:i] + } + return strings.TrimRight(cut, " ,;:.") + "…" +} + +func primaryModule(p *Project) string { + for _, module := range p.Modules { + if module.Path != "" { + return module.Path + } + } + return "" +} + +func plural(n int, singular, many string) string { + if n == 1 { + return "1 " + singular + } + return strconv.Itoa(n) + " " + many +} + +func joinList(parts []string) string { + switch len(parts) { + case 0: + return "" + case 1: + return parts[0] + case 2: + return parts[0] + " and " + parts[1] + default: + return strings.Join(parts[:len(parts)-1], ", ") + " and " + parts[len(parts)-1] + } +} diff --git a/backend/docgen/merge.go b/backend/docgen/merge.go new file mode 100644 index 0000000..bdf3d80 --- /dev/null +++ b/backend/docgen/merge.go @@ -0,0 +1,143 @@ +package docgen + +import ( + "fmt" + "regexp" + "strings" +) + +// beginPrefix opens the machine-written region of an imported page. Everything +// between begin and EndMarker is replaced on every sync; anything outside it is +// preserved exactly as written. +const beginPrefix = "" + +// stampPattern matches the generated=