go-doc/backend/docgen/extract.go

package docgen

import (
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
)

// ---------------------------------------------------------------------------
// Go source extraction
// ---------------------------------------------------------------------------

var (
	// net/http style registrations, including the Go 1.22 method-prefixed form
	// mux.HandleFunc("GET /health", h). The literal is captured whole and
	// validated in parseRoute, which is where the method prefix is split off.
	routeHandleCall = regexp.MustCompile(`\b(?:Handle|HandleFunc)\(\s*"([^"]+)"\s*(?:,\s*([A-Za-z0-9_.]+))?`)

	// Route tables that pass the method as an http.MethodXxx constant, as
	// go-guess does: register(router, spec, false, http.MethodGet, "/api/health", h)
	routeMethodConst = regexp.MustCompile(`\bhttp\.Method([A-Za-z]+)\s*,\s*"(/[^"]*)"\s*(?:,\s*([A-Za-z0-9_.]+)([,)]))?`)

	// Router-value registrations: r.Get("/api/x", h.X)
	routeRouterCall = regexp.MustCompile(`\b\w+\.(Get|Post|Put|Patch|Delete|Head|Options|Trace)\(\s*"(/[^"]*)"\s*(?:,\s*([A-Za-z0-9_.]+)([,)]))?`)

	// "GET /api/v1/x" as a literal argument.
	methodPrefix = regexp.MustCompile(`^(GET|POST|PUT|PATCH|DELETE|HEAD|OPTIONS)\s+(\S.*)$`)

	// flag.String("name", default, "usage")
	flagCall = regexp.MustCompile(`flag\.(\w+)\(\s*"([^"]+)"\s*(?:,\s*([^,)]+))?\s*(?:,\s*"([^"]*)")?`)

	// os.Getenv("X") / os.LookupEnv("X")
	envCall = regexp.MustCompile(`os\.(?:Getenv|LookupEnv)\(\s*"([A-Z][A-Z0-9_]*)"`)

	// Port literals anywhere in the project.
	portLiteral = regexp.MustCompile(`(?:\bPORT\b|\bport\b|EXPOSE)[^0-9]{0,20}([0-9]{2,5})\b`)

	testFuncDecl = regexp.MustCompile(`(?m)^func\s+(Test|Benchmark|Fuzz)\w*\(`)
)

// analyzeGoFile pulls routes, flags and environment variables out of one source
// file. Each extractor is independent: a syntax oddity in one never affects the
// others because these are textual scans, not a type-checked AST walk.
//
// Test files are excluded. They build requests with httptest.NewRequest, which
// looks exactly like a route registration, and a route table is never declared
// in a _test.go file.
func analyzeGoFile(p *Project, file scannedFile) {
	if strings.HasSuffix(file.rel, "_test.go") {
		return
	}

	source := file.text()
	if source == "" {
		return
	}

	for _, match := range routeHandleCall.FindAllStringSubmatch(source, -1) {
		if route := parseRoute(match[1], match[2]); route.Path != "" {
			p.Routes = append(p.Routes, route)
		}
	}
	for _, match := range routeMethodConst.FindAllStringSubmatch(source, -1) {
		p.Routes = append(p.Routes, Route{
			Method:  strings.ToUpper(match[1]),
			Path:    match[2],
			Handler: handlerName(match[3]),
		})
	}
	for _, match := range routeRouterCall.FindAllStringSubmatch(source, -1) {
		p.Routes = append(p.Routes, Route{
			Method:  strings.ToUpper(match[1]),
			Path:    match[2],
			Handler: handlerName(match[3]),
		})
	}

	for _, match := range flagCall.FindAllStringSubmatch(source, -1) {
		p.Flags = append(p.Flags, Flag{
			Name:    match[2],
			Type:    strings.ToLower(match[1]),
			Default: unquote(strings.TrimSpace(match[3])),
			Usage:   strings.TrimSpace(match[4]),
		})
	}

	for _, match := range envCall.FindAllStringSubmatch(source, -1) {
		p.EnvVars = append(p.EnvVars, EnvVar{Name: match[1]})
	}
}

// parseRoute splits an optional method prefix and normalises the path. A
// literal that is not a route — a query key, a header name, a MIME type — is
// rejected by requiring the path to look like one.
func parseRoute(raw, handler string) Route {
	method, path := "", strings.TrimSpace(raw)
	if match := methodPrefix.FindStringSubmatch(raw); match != nil {
		method, path = match[1], strings.TrimSpace(match[2])
	}
	if !strings.HasPrefix(path, "/") {
		return Route{}
	}
	return Route{Method: method, Path: path, Handler: handlerName(handler)}
}

// handlerNames that are never useful in a route table: they are literal
// arguments, receivers or framework values rather than handler functions.
var noiseHandlers = map[string]struct{}{
	"nil": {}, "true": {}, "false": {}, "spec": {}, "router": {}, "mux": {},
	"r": {}, "g": {}, "chi": {}, "group": {}, "protected": {}, "public": {},
	"api": {}, "v1": {}, "sub": {}, "app": {}, "server": {}, "srv": {},
	"ctx": {}, "c": {}, "w": {}, "req": {}, "request": {}, "http": {},
}

// handlerName reduces a handler expression to a readable name, dropping the
// receiver and rejecting anything that is not plainly a function reference.
func handlerName(handler string) string {
	handler = strings.TrimSpace(handler)
	if handler == "" {
		return ""
	}
	if i := strings.LastIndex(handler, "."); i >= 0 {
		handler = handler[i+1:]
	}
	handler = strings.TrimSuffix(handler, ".func1")
	if _, noise := noiseHandlers[handler]; noise {
		return ""
	}
	if handler == "HandlerFunc" || strings.Contains(handler, "(") {
		return ""
	}
	return handler
}

// ---------------------------------------------------------------------------
// go.mod
// ---------------------------------------------------------------------------

var (
	modLine       = regexp.MustCompile(`^module\s+(\S+)`)
	goVersionLine = regexp.MustCompile(`^go\s+(\S+)`)
	requireLine   = regexp.MustCompile(`^\s*(\S+)\s+(v\S+)(.*)$`)
)

// parseGoMod reads a go.mod outside any require block.
func parseGoMod(rel, source string) Module {
	module := Module{File: rel}
	inBlock := false

	for _, raw := range strings.Split(source, "\n") {
		line := strings.TrimRight(raw, " \t")
		trimmed := strings.TrimSpace(line)

		switch {
		case strings.HasPrefix(trimmed, "module "):
			if module.Path == "" {
				module.Path = strings.TrimSpace(strings.TrimPrefix(trimmed, "module "))
			}
		case strings.HasPrefix(trimmed, "go "):
			if module.GoVersion == "" {
				if match := goVersionLine.FindStringSubmatch(trimmed); match != nil {
					module.GoVersion = match[1]
				}
			}
		case trimmed == "require (":
			inBlock = true
		case inBlock && trimmed == ")":
			inBlock = false
		case inBlock && trimmed != "" && !strings.HasPrefix(trimmed, "//"):
			match := requireLine.FindStringSubmatch(line)
			if match != nil {
				addDependency(&module, match[1], match[2], strings.Contains(match[3], "// indirect"))
			}
		case strings.HasPrefix(trimmed, "require ") && !inBlock:
			rest := strings.TrimSpace(strings.TrimPrefix(trimmed, "require "))
			if match := requireLine.FindStringSubmatch(" " + rest); match != nil {
				addDependency(&module, match[1], match[2], strings.Contains(match[3], "// indirect"))
			}
		}
	}
	return module
}

func addDependency(module *Module, path, version string, indirect bool) {
	if path == "" || strings.HasPrefix(path, "//") {
		return
	}
	if indirect {
		module.Indirect++
		return
	}
	module.Direct = append(module.Direct, Dependency{Path: path, Version: version})
}

// ---------------------------------------------------------------------------
// Makefile
// ---------------------------------------------------------------------------

var makeTargetLine = regexp.MustCompile(`^([A-Za-z0-9][A-Za-z0-9_.-]*):`)

// makeTargets extracts invocable targets, skipping .PHONY declarations and
// variable assignments.
func makeTargets(source string) []string {
	var targets []string
	phony := map[string]struct{}{}

	for _, raw := range strings.Split(source, "\n") {
		if strings.HasPrefix(raw, "\t") || strings.HasPrefix(raw, " ") {
			continue
		}
		line := strings.TrimRight(raw, " \t")
		if strings.HasPrefix(line, ".PHONY") {
			for _, name := range strings.Fields(line) {
				if name == ".PHONY:" || name == ".PHONY" {
					continue
				}
				phony[strings.TrimSuffix(name, ":")] = struct{}{}
			}
			continue
		}
		match := makeTargetLine.FindStringSubmatch(line)
		if match == nil {
			continue
		}
		name := match[1]
		if strings.HasPrefix(name, ".") {
			continue
		}
		if _, isPhony := phony[name]; isPhony {
			continue
		}
		targets = append(targets, name)
	}
	return targets
}

// ---------------------------------------------------------------------------
// Docker
// ---------------------------------------------------------------------------

var (
	dockerFrom = regexp.MustCompile(`(?mi)^\s*FROM\s+(\S+)(?:\s+AS\s+(\S+))?`)
	dockerExp  = regexp.MustCompile(`(?mi)^\s*EXPOSE\s+(\S+)`)
)

func parseDockerfile(rel, source string) *DockerInfo {
	info := &DockerInfo{File: rel}
	for _, match := range dockerFrom.FindAllStringSubmatch(source, -1) {
		info.Bases = append(info.Bases, match[1])
		if match[2] != "" {
			info.Stages = append(info.Stages, match[2])
		}
	}
	for _, match := range dockerExp.FindAllStringSubmatch(source, -1) {
		info.Exposed = append(info.Exposed, match[1])
	}
	// Order matters: the first FROM is the build stage and the last is the
	// runtime image, so these lists are deduplicated but never sorted.
	info.Bases = dedupeInOrder(info.Bases)
	info.Stages = dedupeInOrder(info.Stages)
	info.Exposed = dedupeInOrder(info.Exposed)
	return info
}

// ---------------------------------------------------------------------------
// docker-compose
// ---------------------------------------------------------------------------

// parseCompose reads service names, images and published ports out of a compose
// file. It is a line scanner rather than a YAML parser: compose files in these
// projects are consistently indented two spaces per level, and avoiding a YAML
// dependency keeps the module dependency-free.
func parseCompose(rel, source string) []ComposeService {
	var (
		services []ComposeService
		inBlock  bool
		current  int = -1
	)

	for _, raw := range strings.Split(source, "\n") {
		line := strings.TrimRight(raw, " \t")
		trimmed := strings.TrimSpace(line)
		if trimmed == "" || strings.HasPrefix(trimmed, "#") {
			continue
		}

		indent := len(line) - len(strings.TrimLeft(line, " "))

		if indent == 0 {
			inBlock = trimmed == "services:"
			current = -1
			continue
		}
		if !inBlock {
			continue
		}

		if indent == 2 && strings.HasSuffix(trimmed, ":") {
			services = append(services, ComposeService{Name: strings.TrimSuffix(trimmed, ":"), File: rel})
			current = len(services) - 1
			continue
		}
		if current < 0 || indent != 4 {
			continue
		}

		key, value, found := strings.Cut(trimmed, ":")
		if !found {
			continue
		}
		switch strings.TrimSpace(key) {
		case "image":
			services[current].Image = strings.Trim(strings.TrimSpace(value), `"'`)
		case "ports":
			if services[current].Ports == "" {
				services[current].Ports = strings.Trim(strings.TrimSpace(value), `"'`)
			} else {
				services[current].Ports += ", " + strings.Trim(strings.TrimSpace(value), `"'`)
			}
		}
	}

	// Drop the well-known internal networks and volume declarations that the
	// indentation scanner otherwise mistakes for services.
	names := map[string]struct{}{}
	for _, name := range []string{
		"networks", "volumes", "configs", "secrets", "services",
	} {
		names[name] = struct{}{}
	}

	out := services[:0]
	for _, service := range services {
		if _, skip := names[service.Name]; skip {
			continue
		}
		out = append(out, service)
	}
	return out
}

// ---------------------------------------------------------------------------
// .env files
// ---------------------------------------------------------------------------

// unquote strips the quoting Go string literals carry around literal values.
func unquote(value string) string {
	if len(value) >= 2 {
		if (value[0] == '"' && value[len(value)-1] == '"') ||
			(value[0] == '`' && value[len(value)-1] == '`') {
			return value[1 : len(value)-1]
		}
	}
	return value
}

// envFromExample reads KEY=VALUE pairs out of a .env or .env.example file.
func envFromExample(source string) []EnvVar {
	var out []EnvVar
	for _, raw := range strings.Split(source, "\n") {
		line := strings.TrimSpace(raw)
		if line == "" || strings.HasPrefix(line, "#") {
			continue
		}
		key, value, found := strings.Cut(line, "=")
		if !found {
			continue
		}
		key = strings.TrimSpace(key)
		if key == "" || !strings.ContainsAny(key, "ABCDEFGHIJKLMNOPQRSTUVWXYZ") {
			continue
		}
		out = append(out, EnvVar{Name: key, Example: strings.Trim(strings.TrimSpace(value), `"'`), FromEnv: true})
	}
	return out
}

// ---------------------------------------------------------------------------
// Misc
// ---------------------------------------------------------------------------

// portsIn extracts plausible TCP port literals from any file.
func portsIn(source string) []string {
	if source == "" {
		return nil
	}
	var out []string
	for _, match := range portLiteral.FindAllStringSubmatch(source, -1) {
		port, err := strconv.Atoi(match[1])
		if err != nil || port < 1024 || port > 65535 {
			continue
		}
		out = append(out, match[1])
	}
	return out
}

// portBearingFiles reports whether a file is a plausible source of a declared
// port. Scanning every source file for port literals produces noise (version
// numbers, row counts), so only operational files are considered.
func portBearingFiles(rel string) bool {
	base := strings.ToLower(filepath.Base(rel))
	switch {
	case isDockerfile(rel), isComposeFile(rel), isHelmFile(rel):
		return true
	case base == "makefile", base == "makefile.am":
		return true
	case base == "readme.md":
		return true
	case strings.HasPrefix(base, ".env"):
		return true
	case strings.HasSuffix(base, ".service"), strings.HasSuffix(base, ".env.example"):
		return true
	default:
		return false
	}
}

// isHelmFile reports whether a path is part of a Helm chart, anchored on the two
// files that define one.
func isHelmFile(rel string) bool {
	base := strings.ToLower(filepath.Base(rel))
	return base == "chart.yaml" || base == "values.yaml"
}

// markdownTitle returns the first heading of a Markdown document.
func markdownTitle(source string) string {
	for _, raw := range strings.Split(source, "\n") {
		line := strings.TrimSpace(raw)
		if strings.HasPrefix(line, "#") {
			return strings.TrimSpace(strings.TrimLeft(line, "#"))
		}
		if line != "" {
			return ""
		}
	}
	return ""
}

// findReadme locates the project's README, preferring the repository root.
func findReadme(p *Project) string {
	for _, candidate := range []string{"README.md", "readme.md", "README.markdown", "Readme.md"} {
		if info, err := os.Stat(filepath.Join(p.Path, candidate)); err == nil && !info.IsDir() {
			return candidate
		}
	}
	for _, doc := range p.Docs {
		if strings.EqualFold(filepath.Base(doc.Path), "readme.md") {
			return doc.Path
		}
	}
	return ""
}

// summarize produces the one-paragraph preamble shown above the audience tabs.
// It prefers the README's opening prose and falls back to a description derived
// from what the analyser found, so a project without a README still gets one.
func summarize(p *Project) string {
	if p.Readme != "" {
		if data, err := readCapped(p.Path+"/"+p.Readme, maxReadBytes); err == nil {
			if summary := firstParagraph(string(data)); summary != "" {
				return collapse(summary, 460)
			}
		}
	}

	parts := []string{}
	if module := primaryModule(p); module != "" {
		parts = append(parts, "Go module `"+module+"`")
	}
	if p.Stats.GoFiles > 0 {
		parts = append(parts, plural(p.Stats.GoFiles, "Go source file", "Go source files"))
	}
	if len(p.Routes) > 0 {
		parts = append(parts, plural(len(p.Routes), "HTTP endpoint", "HTTP endpoints"))
	}
	if len(p.EnvVars) > 0 {
		parts = append(parts, plural(len(p.EnvVars), "environment variable", "environment variables"))
	}
	if len(parts) == 0 {
		return "This project has no README yet. The sections below were derived from its structure on disk."
	}
	return "This project has no README yet. The analyser found " + joinList(parts) + "."
}

// firstParagraph returns the opening prose of a document: the first run of
// non-empty lines that is not a heading, badge, table or HTML block.
func firstParagraph(source string) string {
	var lines []string

	for _, raw := range strings.Split(source, "\n") {
		trimmed := strings.TrimSpace(raw)

		if strings.HasPrefix(trimmed, "```") {
			break // prose never starts after a code fence in these READMEs
		}
		switch {
		case trimmed == "":
			if len(lines) > 0 {
				return strings.Join(lines, " ")
			}
		case strings.HasPrefix(trimmed, "#"):
			continue
		case strings.HasPrefix(trimmed, "!"), strings.HasPrefix(trimmed, "[!"),
			strings.HasPrefix(trimmed, "<"), strings.HasPrefix(trimmed, "|"):
			continue // badge, HTML or table row
		default:
			lines = append(lines, trimmed)
		}
	}
	return strings.Join(lines, " ")
}

// collapse folds whitespace and truncates on a word boundary.
func collapse(text string, limit int) string {
	fields := strings.Fields(text)
	if len(fields) == 0 {
		return ""
	}
	out := strings.Join(fields, " ")
	if len(out) <= limit {
		return out
	}
	cut := out[:limit]
	if i := strings.LastIndexByte(cut, ' '); i > limit/2 {
		cut = cut[:i]
	}
	return strings.TrimRight(cut, " ,;:.") + "…"
}

func primaryModule(p *Project) string {
	for _, module := range p.Modules {
		if module.Path != "" {
			return module.Path
		}
	}
	return ""
}

func plural(n int, singular, many string) string {
	if n == 1 {
		return "1 " + singular
	}
	return strconv.Itoa(n) + " " + many
}

func joinList(parts []string) string {
	switch len(parts) {
	case 0:
		return ""
	case 1:
		return parts[0]
	case 2:
		return parts[0] + " and " + parts[1]
	default:
		return strings.Join(parts[:len(parts)-1], ", ") + " and " + parts[len(parts)-1]
	}
}