initial commit

backend/docgen/docgen.go

@@ -0,0 +1,728 @@
+// Package docgen analyses a Go project and renders audience-segmented
+// documentation for go-doc.
+//
+// The pipeline is deliberately mechanical: Analyze walks a project tree and
+// extracts facts that can be read off the filesystem (module metadata, HTTP
+// routes, CLI flags, environment variables, Make targets, container services,
+// Helm charts, existing Markdown, language breakdown), and Render turns those
+// facts into the same three-audience layout go-doc uses natively:
+//
+//	## For Users      what the thing does and how to use it
+//	## For Operators  running, configuring and deploying it
+//	## For Developers  contracts, layout and dependencies
+//
+// Import wraps the rendered page in generated markers so re-running the tool
+// refreshes only the machine-written region and never clobbers prose a human
+// added outside it.
+package docgen
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"path"
+	"path/filepath"
+	"sort"
+	"strings"
+	"time"
+)
+
+// maxReadBytes caps how much of any single file is read into memory.
+const maxReadBytes = 2 << 20
+
+// maxWalkFiles caps the walk so a mis-pointed root cannot spin forever.
+const maxWalkFiles = 200000
+
+// Route is one HTTP endpoint discovered in the source.
+type Route struct {
+	Method  string
+	Path    string
+	Handler string
+}
+
+// String renders the route the way it is registered.
+func (r Route) String() string {
+	if r.Method == "" {
+		return r.Path
+	}
+	return r.Method + " " + r.Path
+}
+
+// SortKey orders routes by path, then method, so output is stable.
+func (r Route) SortKey() string { return r.Path + "\x00" + r.Method + "\x00" + r.Handler }
+
+// Flag is a command-line flag declared with the flag package.
+type Flag struct {
+	Name    string
+	Type    string
+	Default string
+	Usage   string
+}
+
+// EnvVar is an environment variable the code reads, plus a note when it was
+// discovered in a .env.example file rather than in source.
+type EnvVar struct {
+	Name    string
+	Example string
+	FromEnv bool
+}
+
+// Dependency is one requirement line from a go.mod.
+type Dependency struct {
+	Path     string
+	Version  string
+	Indirect bool
+}
+
+// Module is one go.mod file found in the tree.
+type Module struct {
+	Path      string
+	File      string
+	GoVersion string
+	Direct    []Dependency
+	Indirect  int
+}
+
+// ComposeService is a service declared in a docker-compose file.
+type ComposeService struct {
+	Name  string
+	Image string
+	Ports string
+	File  string
+}
+
+// DockerInfo summarises the project's Dockerfile.
+type DockerInfo struct {
+	File    string
+	Stages  []string
+	Bases   []string
+	Exposed []string
+}
+
+// DocFile is an existing Markdown document that ships with the project.
+type DocFile struct {
+	Path  string
+	Title string
+	Bytes int
+}
+
+// Dir is a first-level source directory with a purpose inferred from its name.
+type Dir struct {
+	Name  string
+	Files int
+	Lines int
+}
+
+// Language is a per-extension line count.
+type Language struct {
+	Name  string
+	Files int
+	Lines int
+}
+
+// Stats are the headline counters shown in the generated page.
+type Stats struct {
+	GoFiles      int
+	TestFiles    int
+	TestFuncs    int
+	Components   int
+	TotalLines   int
+	OldestCommit string
+}
+
+// Project is the full analysis result for one repository.
+type Project struct {
+	Name    string
+	Path    string
+	Summary string
+	Readme  string
+
+	Modules     []Module
+	Routes      []Route
+	Flags       []Flag
+	EnvVars     []EnvVar
+	MakeTargets []string
+	Compose     []ComposeService
+	Docker      *DockerInfo
+	HelmCharts  []string
+	Docs        []DocFile
+	Dirs        []Dir
+	Languages   []Language
+	Ports       []string
+	Stats       Stats
+
+	// Signature changes whenever any analysed input changes. Watch mode uses it
+	// to decide whether a regeneration is needed.
+	Signature string
+}
+
+// Analyze walks root and extracts everything Render needs.
+//
+// A missing or partial project is not an error: a repository without a go.mod
+// or without a README still yields a usable page describing whatever was found.
+func Analyze(name, root string) (*Project, error) {
+	info, err := os.Stat(root)
+	if err != nil {
+		return nil, fmt.Errorf("docgen: stat %s: %w", root, err)
+	}
+	if !info.IsDir() {
+		return nil, fmt.Errorf("docgen: %s is not a directory", root)
+	}
+
+	p := &Project{Name: name, Path: root}
+
+	files, err := collectFiles(root)
+	if err != nil {
+		return nil, err
+	}
+	p.Signature = signature(files)
+
+	for _, file := range files {
+		rel := file.rel
+
+		switch {
+		case strings.HasSuffix(rel, ".go"):
+			p.Stats.GoFiles++
+			p.Stats.TotalLines += file.lines
+			if strings.HasSuffix(rel, "_test.go") {
+				p.Stats.TestFiles++
+				p.Stats.TestFuncs += countTestFuncs(file.text())
+			}
+			if isComponentPath(rel) {
+				p.Stats.Components++
+			}
+			analyzeGoFile(p, file)
+
+		case filepath.Base(rel) == "go.mod":
+			p.Modules = append(p.Modules, parseGoMod(rel, file.text()))
+
+		case isMarkdown(rel):
+			p.Docs = append(p.Docs, DocFile{
+				Path:  rel,
+				Title: markdownTitle(file.text()),
+				Bytes: int(file.size),
+			})
+
+		case filepath.Base(rel) == "Makefile" || filepath.Base(rel) == "makefile":
+			p.MakeTargets = append(p.MakeTargets, makeTargets(file.text())...)
+
+		case isDockerfile(rel):
+			if p.Docker == nil {
+				p.Docker = parseDockerfile(rel, file.text())
+			}
+
+		case isComposeFile(rel):
+			p.Compose = append(p.Compose, parseCompose(rel, file.text())...)
+
+		case strings.HasPrefix(filepath.Base(rel), ".env"):
+			p.EnvVars = append(p.EnvVars, envFromExample(file.text())...)
+
+		case isHelmFile(rel):
+			p.HelmCharts = append(p.HelmCharts, chartRoot(rel))
+		}
+
+		p.Languages = addLanguage(p.Languages, file)
+		if portBearingFiles(rel) {
+			p.Ports = append(p.Ports, portsIn(file.text())...)
+		}
+	}
+
+	finish(p)
+	return p, nil
+}
+
+// ---------------------------------------------------------------------------
+// Normalisation
+// ---------------------------------------------------------------------------
+
+func finish(p *Project) {
+	p.Readme = findReadme(p)
+	p.Summary = summarize(p)
+
+	p.Modules = dedupeModules(p.Modules)
+	p.Routes = dedupeRoutes(p.Routes)
+	p.Flags = dedupeFlags(p.Flags)
+	p.EnvVars = dedupeEnv(p.EnvVars)
+	p.MakeTargets = dedupeStrings(p.MakeTargets)
+	p.HelmCharts = dedupeStrings(p.HelmCharts)
+	p.Ports = dedupeStrings(p.Ports)
+	p.Dirs = topDirs(p)
+	p.Languages = rankLanguages(p.Languages)
+}
+
+// ---------------------------------------------------------------------------
+// Walking
+// ---------------------------------------------------------------------------
+
+type scannedFile struct {
+	path  string
+	rel   string
+	size  int64
+	lines int
+}
+
+func (f scannedFile) text() string {
+	data, err := readCapped(f.path, maxReadBytes)
+	if err != nil {
+		return ""
+	}
+	return string(data)
+}
+
+// ignoredDirs never contain first-party source worth documenting.
+var ignoredDirs = map[string]struct{}{
+	"node_modules": {}, ".git": {}, ".idea": {}, ".vscode": {}, "dist": {},
+	"build": {}, "bin": {}, "vendor": {}, "coverage": {}, "tmp": {},
+	".cache": {}, ".next": {}, "target": {}, "__pycache__": {}, ".venv": {},
+}
+
+func collectFiles(root string) ([]scannedFile, error) {
+	var files []scannedFile
+
+	err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, err error) error {
+		if err != nil {
+			// An unreadable subtree must not abort the whole analysis.
+			if entry != nil && entry.IsDir() {
+				return fs.SkipDir
+			}
+			return nil
+		}
+		if len(files) >= maxWalkFiles {
+			return fs.SkipAll
+		}
+
+		name := entry.Name()
+		if entry.IsDir() {
+			if _, skip := ignoredDirs[name]; skip && path != root {
+				return fs.SkipDir
+			}
+			return nil
+		}
+		if !entry.Type().IsRegular() {
+			return nil
+		}
+		if isNoise(name) {
+			return nil
+		}
+
+		info, err := entry.Info()
+		if err != nil {
+			return nil
+		}
+		rel, err := filepath.Rel(root, path)
+		if err != nil {
+			rel = name
+		}
+		files = append(files, scannedFile{
+			path:  path,
+			rel:   filepath.ToSlash(rel),
+			size:  info.Size(),
+			lines: countLinesFast(path),
+		})
+		return nil
+	})
+	if err != nil {
+		return nil, fmt.Errorf("docgen: walk %s: %w", root, err)
+	}
+	return files, nil
+}
+
+// isNoise filters artefacts that would pollute the analysis.
+func isNoise(name string) bool {
+	switch filepath.Ext(name) {
+	case ".log", ".lock", ".sum", ".pem", ".crt", ".key", ".png", ".jpg", ".jpeg",
+		".gif", ".svg", ".ico", ".woff", ".woff2", ".ttf", ".eot", ".zip", ".tar",
+		".gz", ".mp4", ".sql.gz", ".db", ".sqlite", ".pyc":
+		return true
+	}
+	// Certificates and keys are frequently committed without extensions.
+	switch {
+	case strings.HasSuffix(name, ".crt"), strings.HasSuffix(name, ".key"):
+		return true
+	case strings.HasSuffix(name, "-chain.crt"), strings.HasSuffix(name, "-root-ca.crt"):
+		return true
+	}
+	return false
+}
+
+func isMarkdown(rel string) bool {
+	return strings.EqualFold(filepath.Ext(rel), ".md")
+}
+
+func isDockerfile(rel string) bool {
+	base := filepath.Base(rel)
+	return base == "Dockerfile" || strings.HasPrefix(base, "Dockerfile.")
+}
+
+func isComposeFile(rel string) bool {
+	base := strings.ToLower(filepath.Base(rel))
+	if !strings.HasPrefix(base, "docker-compose") && base != "compose.yaml" && base != "compose.yml" {
+		return false
+	}
+	return strings.HasSuffix(base, ".yml") || strings.HasSuffix(base, ".yaml")
+}
+
+func isHelmChart(rel string) bool {
+	parts := strings.Split(rel, "/")
+	for i, part := range parts {
+		if part == "helm" || part == "charts" {
+			// A chart is a directory that owns a values.yaml or Chart.yaml.
+			if i+1 < len(parts) {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+// chartRoot collapses a chart file path to the directory that owns the chart,
+// so the page lists one entry per chart rather than one per template.
+func chartRoot(rel string) string {
+	dir := path.Dir(rel)
+	if dir == "." {
+		return "./ (repository root)"
+	}
+	return dir
+}
+
+func isComponentPath(rel string) bool {
+	parts := strings.Split(rel, "/")
+	if len(parts) < 2 {
+		return false
+	}
+	switch parts[len(parts)-2] {
+	case "internal", "pkg", "cmd", "api", "services", "web", "ui", "frontend", "backend":
+		return true
+	}
+	return false
+}
+
+func readCapped(path string, limit int64) ([]byte, error) {
+	file, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+	return io.ReadAll(io.LimitReader(file, limit))
+}
+
+func countLinesFast(path string) int {
+	file, err := os.Open(path)
+	if err != nil {
+		return 0
+	}
+	defer file.Close()
+
+	// Skip .md / .json / .lock style files where line counts are meaningless.
+	switch strings.ToLower(filepath.Ext(path)) {
+	case ".md", ".json", ".yaml", ".yml", ".txt", ".lock", ".css", ".html":
+		return 0
+	}
+
+	buf := make([]byte, 32<<10)
+	lines := 0
+	for {
+		n, err := file.Read(buf)
+		if n > 0 {
+			for _, b := range buf[:n] {
+				if b == '\n' {
+					lines++
+				}
+			}
+		}
+		if err != nil {
+			break
+		}
+	}
+	return lines
+}
+
+// signature hashes the identity of every analysed file. It deliberately uses
+// size and mtime rather than content so watch mode can poll large trees cheaply.
+func signature(files []scannedFile) string {
+	hash := sha256.New()
+	for _, file := range files {
+		info, err := os.Stat(file.path)
+		if err != nil {
+			continue
+		}
+		fmt.Fprintf(hash, "%s|%d|%d\n", file.rel, file.size, info.ModTime().UnixNano())
+	}
+	return hex.EncodeToString(hash.Sum(nil))[:32]
+}
+
+func countTestFuncs(source string) int {
+	return strings.Count(source, "func Test") + strings.Count(source, "func Fuzz") +
+		strings.Count(source, "func Benchmark")
+}
+
+// noiseExtensions are file types that carry no language signal: build
+// metadata, ignore lists and editor configuration all have extensions that would
+// otherwise show up as their own "language".
+var noiseExtensions = map[string]struct{}{
+	"mod": {}, "sum": {}, "lock": {}, "gitignore": {}, "dockerignore": {},
+	"helmignore": {}, "npmignore": {}, "eslintignore": {}, "gitattributes": {},
+	"env":          {},
+	"editorconfig": {}, "prettierignore": {}, "example": {}, "template": {},
+}
+
+func addLanguage(langs []Language, file scannedFile) []Language {
+	ext := strings.ToLower(strings.TrimPrefix(filepath.Ext(file.path), "."))
+	if ext == "" {
+		return langs
+	}
+	if _, skip := noiseExtensions[ext]; skip {
+		return langs
+	}
+	name := languageName(ext)
+	lines := file.lines
+	if lines == 0 {
+		lines = countAllLinesFast(file.path)
+	}
+	for i := range langs {
+		if langs[i].Name == name {
+			langs[i].Files++
+			langs[i].Lines += lines
+			return langs
+		}
+	}
+	return append(langs, Language{Name: name, Files: 1, Lines: lines})
+}
+
+func countAllLinesFast(path string) int {
+	data, err := readCapped(path, maxReadBytes)
+	if err != nil {
+		return 0
+	}
+	lines := strings.Count(string(data), "\n")
+	if len(data) > 0 && !strings.HasSuffix(string(data), "\n") {
+		lines++
+	}
+	return lines
+}
+
+func languageName(ext string) string {
+	switch ext {
+	case "go":
+		return "Go"
+	case "js", "mjs", "cjs":
+		return "JavaScript"
+	case "jsx":
+		return "JSX"
+	case "ts", "tsx":
+		return "TypeScript"
+	case "css":
+		return "CSS"
+	case "html", "htm":
+		return "HTML"
+	case "sh", "bash":
+		return "Shell"
+	case "sql":
+		return "SQL"
+	case "yml", "yaml":
+		return "YAML"
+	case "json":
+		return "JSON"
+	case "md":
+		return "Markdown"
+	case "tf":
+		return "Terraform"
+	case "proto":
+		return "Protobuf"
+	case "tpl", "gotmpl":
+		return "Templates"
+	case "py":
+		return "Python"
+	default:
+		return ext
+	}
+}
+
+func rankLanguages(langs []Language) []Language {
+	sort.Slice(langs, func(i, j int) bool {
+		if langs[i].Lines == langs[j].Lines {
+			return langs[i].Name < langs[j].Name
+		}
+		return langs[i].Lines > langs[j].Lines
+	})
+	// A page does not benefit from a tail of one-file languages.
+	if len(langs) > maxLanguages {
+		langs = langs[:maxLanguages]
+	}
+	return langs
+}
+
+// maxLanguages caps the language table on a generated page.
+const maxLanguages = 10
+
+func topDirs(p *Project) []Dir {
+	byName := map[string]*Dir{}
+	err := filepath.WalkDir(p.Path, func(path string, entry fs.DirEntry, err error) error {
+		if err != nil {
+			return nil
+		}
+		if entry.IsDir() {
+			if path != p.Path {
+				if _, skip := ignoredDirs[entry.Name()]; skip {
+					return fs.SkipDir
+				}
+				// Dot directories hold tooling, not first-party source.
+				if strings.HasPrefix(entry.Name(), ".") {
+					return fs.SkipDir
+				}
+			}
+			return nil
+		}
+		rel, err := filepath.Rel(p.Path, path)
+		if err != nil {
+			return nil
+		}
+		parts := strings.Split(filepath.ToSlash(rel), "/")
+		if len(parts) < 2 {
+			return nil
+		}
+		dir := byName[parts[0]]
+		if dir == nil {
+			dir = &Dir{Name: parts[0]}
+			byName[parts[0]] = dir
+		}
+		dir.Files++
+		dir.Lines += countLinesFast(path)
+		return nil
+	})
+	if err != nil {
+		return nil
+	}
+
+	dirs := make([]Dir, 0, len(byName))
+	for _, dir := range byName {
+		dirs = append(dirs, *dir)
+	}
+	sort.Slice(dirs, func(i, j int) bool {
+		if dirs[i].Files == dirs[j].Files {
+			return dirs[i].Name < dirs[j].Name
+		}
+		return dirs[i].Files > dirs[j].Files
+	})
+	if len(dirs) > 14 {
+		dirs = dirs[:14]
+	}
+	return dirs
+}
+
+func max(a, b int) int {
+	if a > b {
+		return a
+	}
+	return b
+}
+
+func dedupeStrings(in []string) []string {
+	seen := map[string]struct{}{}
+	out := make([]string, 0, len(in))
+	for _, value := range in {
+		value = strings.TrimSpace(value)
+		if value == "" {
+			continue
+		}
+		if _, dup := seen[value]; dup {
+			continue
+		}
+		seen[value] = struct{}{}
+		out = append(out, value)
+	}
+	sort.Strings(out)
+	return out
+}
+
+// dedupeInOrder removes duplicates while preserving first-seen order, for lists
+// whose order is meaningful (Docker build stages, published ports).
+func dedupeInOrder(in []string) []string {
+	seen := map[string]struct{}{}
+	out := make([]string, 0, len(in))
+	for _, value := range in {
+		value = strings.TrimSpace(value)
+		if value == "" {
+			continue
+		}
+		if _, dup := seen[value]; dup {
+			continue
+		}
+		seen[value] = struct{}{}
+		out = append(out, value)
+	}
+	return out
+}
+
+func dedupeModules(mods []Module) []Module {
+	sort.Slice(mods, func(i, j int) bool { return mods[i].File < mods[j].File })
+	return mods
+}
+
+// dedupeRoutes collapses registrations that describe the same endpoint. The
+// key is method plus path, because one endpoint is often registered in several
+// places (a group helper plus the router); when several carry a handler name,
+// the one with a handler wins over the one without.
+func dedupeRoutes(routes []Route) []Route {
+	best := make(map[string]Route, len(routes))
+	for _, route := range routes {
+		if route.Path == "" {
+			continue
+		}
+		key := route.Method + " " + route.Path
+		existing, seen := best[key]
+		switch {
+		case !seen:
+			best[key] = route
+		case existing.Handler == "" && route.Handler != "":
+			best[key] = route
+		}
+	}
+
+	out := make([]Route, 0, len(best))
+	for _, route := range best {
+		out = append(out, route)
+	}
+	sort.Slice(out, func(i, j int) bool {
+		if out[i].Path == out[j].Path {
+			return out[i].Method < out[j].Method
+		}
+		return out[i].Path < out[j].Path
+	})
+	return out
+}
+
+func dedupeFlags(flags []Flag) []Flag {
+	seen := map[string]struct{}{}
+	out := make([]Flag, 0, len(flags))
+	for _, flag := range flags {
+		if _, dup := seen[flag.Name]; dup {
+			continue
+		}
+		seen[flag.Name] = struct{}{}
+		out = append(out, flag)
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name })
+	return out
+}
+
+func dedupeEnv(vars []EnvVar) []EnvVar {
+	seen := map[string]struct{}{}
+	out := make([]EnvVar, 0, len(vars))
+	for _, v := range vars {
+		if _, dup := seen[v.Name]; dup {
+			continue
+		}
+		seen[v.Name] = struct{}{}
+		out = append(out, v)
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name })
+	return out
+}
+
+// Timestamp is used inside generated markers so a reader can tell how fresh a
+// machine-written region is.
+func Timestamp() string { return time.Now().UTC().Format("2006-01-02 15:04 UTC") }