go-doc/backend/docgen/docgen.go

// Package docgen analyses a Go project and renders audience-segmented
// documentation for go-doc.
//
// The pipeline is deliberately mechanical: Analyze walks a project tree and
// extracts facts that can be read off the filesystem (module metadata, HTTP
// routes, CLI flags, environment variables, Make targets, container services,
// Helm charts, existing Markdown, language breakdown), and Render turns those
// facts into the same three-audience layout go-doc uses natively:
//
//	## For Users      what the thing does and how to use it
//	## For Operators  running, configuring and deploying it
//	## For Developers  contracts, layout and dependencies
//
// Import wraps the rendered page in generated markers so re-running the tool
// refreshes only the machine-written region and never clobbers prose a human
// added outside it.
package docgen

import (
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"io"
	"io/fs"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strings"
	"time"
)

// maxReadBytes caps how much of any single file is read into memory.
const maxReadBytes = 2 << 20

// maxWalkFiles caps the walk so a mis-pointed root cannot spin forever.
const maxWalkFiles = 200000

// Route is one HTTP endpoint discovered in the source.
type Route struct {
	Method  string
	Path    string
	Handler string
}

// String renders the route the way it is registered.
func (r Route) String() string {
	if r.Method == "" {
		return r.Path
	}
	return r.Method + " " + r.Path
}

// SortKey orders routes by path, then method, so output is stable.
func (r Route) SortKey() string { return r.Path + "\x00" + r.Method + "\x00" + r.Handler }

// Flag is a command-line flag declared with the flag package.
type Flag struct {
	Name    string
	Type    string
	Default string
	Usage   string
}

// EnvVar is an environment variable the code reads, plus a note when it was
// discovered in a .env.example file rather than in source.
type EnvVar struct {
	Name    string
	Example string
	FromEnv bool
}

// Dependency is one requirement line from a go.mod.
type Dependency struct {
	Path     string
	Version  string
	Indirect bool
}

// Module is one go.mod file found in the tree.
type Module struct {
	Path      string
	File      string
	GoVersion string
	Direct    []Dependency
	Indirect  int
}

// ComposeService is a service declared in a docker-compose file.
type ComposeService struct {
	Name  string
	Image string
	Ports string
	File  string
}

// DockerInfo summarises the project's Dockerfile.
type DockerInfo struct {
	File    string
	Stages  []string
	Bases   []string
	Exposed []string
}

// DocFile is an existing Markdown document that ships with the project.
type DocFile struct {
	Path  string
	Title string
	Bytes int
}

// Dir is a first-level source directory with a purpose inferred from its name.
type Dir struct {
	Name  string
	Files int
	Lines int
}

// Language is a per-extension line count.
type Language struct {
	Name  string
	Files int
	Lines int
}

// Stats are the headline counters shown in the generated page.
type Stats struct {
	GoFiles      int
	TestFiles    int
	TestFuncs    int
	Components   int
	TotalLines   int
	OldestCommit string
}

// Project is the full analysis result for one repository.
type Project struct {
	Name    string
	Path    string
	Summary string
	Readme  string

	Modules     []Module
	Routes      []Route
	Flags       []Flag
	EnvVars     []EnvVar
	MakeTargets []string
	Compose     []ComposeService
	Docker      *DockerInfo
	HelmCharts  []string
	Docs        []DocFile
	Dirs        []Dir
	Languages   []Language
	Ports       []string
	Stats       Stats

	// Signature changes whenever any analysed input changes. Watch mode uses it
	// to decide whether a regeneration is needed.
	Signature string
}

// Analyze walks root and extracts everything Render needs.
//
// A missing or partial project is not an error: a repository without a go.mod
// or without a README still yields a usable page describing whatever was found.
func Analyze(name, root string) (*Project, error) {
	info, err := os.Stat(root)
	if err != nil {
		return nil, fmt.Errorf("docgen: stat %s: %w", root, err)
	}
	if !info.IsDir() {
		return nil, fmt.Errorf("docgen: %s is not a directory", root)
	}

	p := &Project{Name: name, Path: root}

	files, err := collectFiles(root)
	if err != nil {
		return nil, err
	}
	p.Signature = signature(files)

	for _, file := range files {
		rel := file.rel

		switch {
		case strings.HasSuffix(rel, ".go"):
			p.Stats.GoFiles++
			p.Stats.TotalLines += file.lines
			if strings.HasSuffix(rel, "_test.go") {
				p.Stats.TestFiles++
				p.Stats.TestFuncs += countTestFuncs(file.text())
			}
			if isComponentPath(rel) {
				p.Stats.Components++
			}
			analyzeGoFile(p, file)

		case filepath.Base(rel) == "go.mod":
			p.Modules = append(p.Modules, parseGoMod(rel, file.text()))

		case isMarkdown(rel):
			p.Docs = append(p.Docs, DocFile{
				Path:  rel,
				Title: markdownTitle(file.text()),
				Bytes: int(file.size),
			})

		case filepath.Base(rel) == "Makefile" || filepath.Base(rel) == "makefile":
			p.MakeTargets = append(p.MakeTargets, makeTargets(file.text())...)

		case isDockerfile(rel):
			if p.Docker == nil {
				p.Docker = parseDockerfile(rel, file.text())
			}

		case isComposeFile(rel):
			p.Compose = append(p.Compose, parseCompose(rel, file.text())...)

		case strings.HasPrefix(filepath.Base(rel), ".env"):
			p.EnvVars = append(p.EnvVars, envFromExample(file.text())...)

		case isHelmFile(rel):
			p.HelmCharts = append(p.HelmCharts, chartRoot(rel))
		}

		p.Languages = addLanguage(p.Languages, file)
		if portBearingFiles(rel) {
			p.Ports = append(p.Ports, portsIn(file.text())...)
		}
	}

	finish(p)
	return p, nil
}

// ---------------------------------------------------------------------------
// Normalisation
// ---------------------------------------------------------------------------

func finish(p *Project) {
	p.Readme = findReadme(p)
	p.Summary = summarize(p)

	p.Modules = dedupeModules(p.Modules)
	p.Routes = dedupeRoutes(p.Routes)
	p.Flags = dedupeFlags(p.Flags)
	p.EnvVars = dedupeEnv(p.EnvVars)
	p.MakeTargets = dedupeStrings(p.MakeTargets)
	p.HelmCharts = dedupeStrings(p.HelmCharts)
	p.Ports = dedupeStrings(p.Ports)
	p.Dirs = topDirs(p)
	p.Languages = rankLanguages(p.Languages)
}

// ---------------------------------------------------------------------------
// Walking
// ---------------------------------------------------------------------------

type scannedFile struct {
	path  string
	rel   string
	size  int64
	lines int
}

func (f scannedFile) text() string {
	data, err := readCapped(f.path, maxReadBytes)
	if err != nil {
		return ""
	}
	return string(data)
}

// ignoredDirs never contain first-party source worth documenting.
var ignoredDirs = map[string]struct{}{
	"node_modules": {}, ".git": {}, ".idea": {}, ".vscode": {}, "dist": {},
	"build": {}, "bin": {}, "vendor": {}, "coverage": {}, "tmp": {},
	".cache": {}, ".next": {}, "target": {}, "__pycache__": {}, ".venv": {},
}

func collectFiles(root string) ([]scannedFile, error) {
	var files []scannedFile

	err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, err error) error {
		if err != nil {
			// An unreadable subtree must not abort the whole analysis.
			if entry != nil && entry.IsDir() {
				return fs.SkipDir
			}
			return nil
		}
		if len(files) >= maxWalkFiles {
			return fs.SkipAll
		}

		name := entry.Name()
		if entry.IsDir() {
			if _, skip := ignoredDirs[name]; skip && path != root {
				return fs.SkipDir
			}
			return nil
		}
		if !entry.Type().IsRegular() {
			return nil
		}
		if isNoise(name) {
			return nil
		}

		info, err := entry.Info()
		if err != nil {
			return nil
		}
		rel, err := filepath.Rel(root, path)
		if err != nil {
			rel = name
		}
		files = append(files, scannedFile{
			path:  path,
			rel:   filepath.ToSlash(rel),
			size:  info.Size(),
			lines: countLinesFast(path),
		})
		return nil
	})
	if err != nil {
		return nil, fmt.Errorf("docgen: walk %s: %w", root, err)
	}
	return files, nil
}

// isNoise filters artefacts that would pollute the analysis.
func isNoise(name string) bool {
	switch filepath.Ext(name) {
	case ".log", ".lock", ".sum", ".pem", ".crt", ".key", ".png", ".jpg", ".jpeg",
		".gif", ".svg", ".ico", ".woff", ".woff2", ".ttf", ".eot", ".zip", ".tar",
		".gz", ".mp4", ".sql.gz", ".db", ".sqlite", ".pyc":
		return true
	}
	// Certificates and keys are frequently committed without extensions.
	switch {
	case strings.HasSuffix(name, ".crt"), strings.HasSuffix(name, ".key"):
		return true
	case strings.HasSuffix(name, "-chain.crt"), strings.HasSuffix(name, "-root-ca.crt"):
		return true
	}
	return false
}

func isMarkdown(rel string) bool {
	return strings.EqualFold(filepath.Ext(rel), ".md")
}

func isDockerfile(rel string) bool {
	base := filepath.Base(rel)
	return base == "Dockerfile" || strings.HasPrefix(base, "Dockerfile.")
}

func isComposeFile(rel string) bool {
	base := strings.ToLower(filepath.Base(rel))
	if !strings.HasPrefix(base, "docker-compose") && base != "compose.yaml" && base != "compose.yml" {
		return false
	}
	return strings.HasSuffix(base, ".yml") || strings.HasSuffix(base, ".yaml")
}

func isHelmChart(rel string) bool {
	parts := strings.Split(rel, "/")
	for i, part := range parts {
		if part == "helm" || part == "charts" {
			// A chart is a directory that owns a values.yaml or Chart.yaml.
			if i+1 < len(parts) {
				return true
			}
		}
	}
	return false
}

// chartRoot collapses a chart file path to the directory that owns the chart,
// so the page lists one entry per chart rather than one per template.
func chartRoot(rel string) string {
	dir := path.Dir(rel)
	if dir == "." {
		return "./ (repository root)"
	}
	return dir
}

func isComponentPath(rel string) bool {
	parts := strings.Split(rel, "/")
	if len(parts) < 2 {
		return false
	}
	switch parts[len(parts)-2] {
	case "internal", "pkg", "cmd", "api", "services", "web", "ui", "frontend", "backend":
		return true
	}
	return false
}

func readCapped(path string, limit int64) ([]byte, error) {
	file, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer file.Close()
	return io.ReadAll(io.LimitReader(file, limit))
}

func countLinesFast(path string) int {
	file, err := os.Open(path)
	if err != nil {
		return 0
	}
	defer file.Close()

	// Skip .md / .json / .lock style files where line counts are meaningless.
	switch strings.ToLower(filepath.Ext(path)) {
	case ".md", ".json", ".yaml", ".yml", ".txt", ".lock", ".css", ".html":
		return 0
	}

	buf := make([]byte, 32<<10)
	lines := 0
	for {
		n, err := file.Read(buf)
		if n > 0 {
			for _, b := range buf[:n] {
				if b == '\n' {
					lines++
				}
			}
		}
		if err != nil {
			break
		}
	}
	return lines
}

// signature hashes the identity of every analysed file. It deliberately uses
// size and mtime rather than content so watch mode can poll large trees cheaply.
func signature(files []scannedFile) string {
	hash := sha256.New()
	for _, file := range files {
		info, err := os.Stat(file.path)
		if err != nil {
			continue
		}
		fmt.Fprintf(hash, "%s|%d|%d\n", file.rel, file.size, info.ModTime().UnixNano())
	}
	return hex.EncodeToString(hash.Sum(nil))[:32]
}

func countTestFuncs(source string) int {
	return strings.Count(source, "func Test") + strings.Count(source, "func Fuzz") +
		strings.Count(source, "func Benchmark")
}

// noiseExtensions are file types that carry no language signal: build
// metadata, ignore lists and editor configuration all have extensions that would
// otherwise show up as their own "language".
var noiseExtensions = map[string]struct{}{
	"mod": {}, "sum": {}, "lock": {}, "gitignore": {}, "dockerignore": {},
	"helmignore": {}, "npmignore": {}, "eslintignore": {}, "gitattributes": {},
	"env":          {},
	"editorconfig": {}, "prettierignore": {}, "example": {}, "template": {},
}

func addLanguage(langs []Language, file scannedFile) []Language {
	ext := strings.ToLower(strings.TrimPrefix(filepath.Ext(file.path), "."))
	if ext == "" {
		return langs
	}
	if _, skip := noiseExtensions[ext]; skip {
		return langs
	}
	name := languageName(ext)
	lines := file.lines
	if lines == 0 {
		lines = countAllLinesFast(file.path)
	}
	for i := range langs {
		if langs[i].Name == name {
			langs[i].Files++
			langs[i].Lines += lines
			return langs
		}
	}
	return append(langs, Language{Name: name, Files: 1, Lines: lines})
}

func countAllLinesFast(path string) int {
	data, err := readCapped(path, maxReadBytes)
	if err != nil {
		return 0
	}
	lines := strings.Count(string(data), "\n")
	if len(data) > 0 && !strings.HasSuffix(string(data), "\n") {
		lines++
	}
	return lines
}

func languageName(ext string) string {
	switch ext {
	case "go":
		return "Go"
	case "js", "mjs", "cjs":
		return "JavaScript"
	case "jsx":
		return "JSX"
	case "ts", "tsx":
		return "TypeScript"
	case "css":
		return "CSS"
	case "html", "htm":
		return "HTML"
	case "sh", "bash":
		return "Shell"
	case "sql":
		return "SQL"
	case "yml", "yaml":
		return "YAML"
	case "json":
		return "JSON"
	case "md":
		return "Markdown"
	case "tf":
		return "Terraform"
	case "proto":
		return "Protobuf"
	case "tpl", "gotmpl":
		return "Templates"
	case "py":
		return "Python"
	default:
		return ext
	}
}

func rankLanguages(langs []Language) []Language {
	sort.Slice(langs, func(i, j int) bool {
		if langs[i].Lines == langs[j].Lines {
			return langs[i].Name < langs[j].Name
		}
		return langs[i].Lines > langs[j].Lines
	})
	// A page does not benefit from a tail of one-file languages.
	if len(langs) > maxLanguages {
		langs = langs[:maxLanguages]
	}
	return langs
}

// maxLanguages caps the language table on a generated page.
const maxLanguages = 10

func topDirs(p *Project) []Dir {
	byName := map[string]*Dir{}
	err := filepath.WalkDir(p.Path, func(path string, entry fs.DirEntry, err error) error {
		if err != nil {
			return nil
		}
		if entry.IsDir() {
			if path != p.Path {
				if _, skip := ignoredDirs[entry.Name()]; skip {
					return fs.SkipDir
				}
				// Dot directories hold tooling, not first-party source.
				if strings.HasPrefix(entry.Name(), ".") {
					return fs.SkipDir
				}
			}
			return nil
		}
		rel, err := filepath.Rel(p.Path, path)
		if err != nil {
			return nil
		}
		parts := strings.Split(filepath.ToSlash(rel), "/")
		if len(parts) < 2 {
			return nil
		}
		dir := byName[parts[0]]
		if dir == nil {
			dir = &Dir{Name: parts[0]}
			byName[parts[0]] = dir
		}
		dir.Files++
		dir.Lines += countLinesFast(path)
		return nil
	})
	if err != nil {
		return nil
	}

	dirs := make([]Dir, 0, len(byName))
	for _, dir := range byName {
		dirs = append(dirs, *dir)
	}
	sort.Slice(dirs, func(i, j int) bool {
		if dirs[i].Files == dirs[j].Files {
			return dirs[i].Name < dirs[j].Name
		}
		return dirs[i].Files > dirs[j].Files
	})
	if len(dirs) > 14 {
		dirs = dirs[:14]
	}
	return dirs
}

func max(a, b int) int {
	if a > b {
		return a
	}
	return b
}

func dedupeStrings(in []string) []string {
	seen := map[string]struct{}{}
	out := make([]string, 0, len(in))
	for _, value := range in {
		value = strings.TrimSpace(value)
		if value == "" {
			continue
		}
		if _, dup := seen[value]; dup {
			continue
		}
		seen[value] = struct{}{}
		out = append(out, value)
	}
	sort.Strings(out)
	return out
}

// dedupeInOrder removes duplicates while preserving first-seen order, for lists
// whose order is meaningful (Docker build stages, published ports).
func dedupeInOrder(in []string) []string {
	seen := map[string]struct{}{}
	out := make([]string, 0, len(in))
	for _, value := range in {
		value = strings.TrimSpace(value)
		if value == "" {
			continue
		}
		if _, dup := seen[value]; dup {
			continue
		}
		seen[value] = struct{}{}
		out = append(out, value)
	}
	return out
}

func dedupeModules(mods []Module) []Module {
	sort.Slice(mods, func(i, j int) bool { return mods[i].File < mods[j].File })
	return mods
}

// dedupeRoutes collapses registrations that describe the same endpoint. The
// key is method plus path, because one endpoint is often registered in several
// places (a group helper plus the router); when several carry a handler name,
// the one with a handler wins over the one without.
func dedupeRoutes(routes []Route) []Route {
	best := make(map[string]Route, len(routes))
	for _, route := range routes {
		if route.Path == "" {
			continue
		}
		key := route.Method + " " + route.Path
		existing, seen := best[key]
		switch {
		case !seen:
			best[key] = route
		case existing.Handler == "" && route.Handler != "":
			best[key] = route
		}
	}

	out := make([]Route, 0, len(best))
	for _, route := range best {
		out = append(out, route)
	}
	sort.Slice(out, func(i, j int) bool {
		if out[i].Path == out[j].Path {
			return out[i].Method < out[j].Method
		}
		return out[i].Path < out[j].Path
	})
	return out
}

func dedupeFlags(flags []Flag) []Flag {
	seen := map[string]struct{}{}
	out := make([]Flag, 0, len(flags))
	for _, flag := range flags {
		if _, dup := seen[flag.Name]; dup {
			continue
		}
		seen[flag.Name] = struct{}{}
		out = append(out, flag)
	}
	sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name })
	return out
}

func dedupeEnv(vars []EnvVar) []EnvVar {
	seen := map[string]struct{}{}
	out := make([]EnvVar, 0, len(vars))
	for _, v := range vars {
		if _, dup := seen[v.Name]; dup {
			continue
		}
		seen[v.Name] = struct{}{}
		out = append(out, v)
	}
	sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name })
	return out
}

// Timestamp is used inside generated markers so a reader can tell how fresh a
// machine-written region is.
func Timestamp() string { return time.Now().UTC().Format("2006-01-02 15:04 UTC") }