go-doc/backend/scripts/docgen/main.go

// Command docgen analyses every go-* project under a workspace root and imports
// audience-segmented documentation for each one into go-doc.
//
// One-shot sync:
//
//	go run ./scripts/docgen -root /home/dnoulet/go -data ../data
//
// Continuous sync, re-importing whenever a project's docs or source change:
//
//	go run ./scripts/docgen -root /home/dnoulet/go -data ../data -watch
//
// Imported pages are wrapped in generated markers. Re-running replaces only the
// machine-written region, so prose written outside the markers survives, and a
// page that was never generated is left untouched unless -force is passed.
package main

import (
	"flag"
	"fmt"
	"log/slog"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"time"

	"go-doc/docgen"
	"go-doc/storage"
)

func main() {
	if err := run(); err != nil {
		slog.Error("fatal", "error", err)
		os.Exit(1)
	}
}

type options struct {
	root         string
	dataDir      string
	apps         string
	skip         string
	watch        bool
	interval     time.Duration
	dryRun       bool
	force        bool
	importSubdoc bool
	verbose      bool
}

func run() error {
	var opts options

	flag.StringVar(&opts.root, "root", defaultRoot(), "directory containing the go-* projects")
	flag.StringVar(&opts.dataDir, "data", storage.ResolveRoot(), "go-doc data directory to import into")
	flag.StringVar(&opts.apps, "apps", "", "comma-separated project names to limit the run (default: all)")
	flag.StringVar(&opts.skip, "skip", "go-doc", "comma-separated project names to ignore")
	flag.BoolVar(&opts.watch, "watch", false, "keep running and re-import when a project changes")
	flag.DurationVar(&opts.interval, "interval", 5*time.Second, "poll interval for -watch")
	flag.BoolVar(&opts.dryRun, "dry-run", false, "analyse and render without writing anything")
	flag.BoolVar(&opts.force, "force", false, "overwrite pages that carry no generated markers")
	flag.BoolVar(&opts.importSubdoc, "import-subdocs", false, "also import each project's own Markdown as separate go-doc pages")
	flag.BoolVar(&opts.verbose, "verbose", false, "log at debug level")
	flag.Parse()

	level := slog.LevelInfo
	if opts.verbose {
		level = slog.LevelDebug
	}
	logger := slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: level}))

	engine, err := storage.New(opts.dataDir)
	if err != nil {
		return fmt.Errorf("cannot open the go-doc data directory: %w", err)
	}

	im := newImporter(opts, engine, logger)

	if !opts.watch {
		result, err := im.SyncOnce()
		if err != nil {
			return err
		}
		result.report(logger)
		if result.Failed > 0 {
			return fmt.Errorf("%d project(s) failed to import", result.Failed)
		}
		return nil
	}

	logger.Info("watching for documentation changes",
		"root", opts.root,
		"interval", opts.interval.String(),
		"data", engine.Root(),
		"hint", "Ctrl-C to stop",
	)

	ticker := time.NewTicker(opts.interval)
	defer ticker.Stop()

	for {
		changed, err := im.Scan()
		switch {
		case err != nil:
			logger.Error("scan failed", "error", err)
		case len(changed) == 0:
			logger.Debug("no changes")
		default:
			logger.Info("change detected", "projects", strings.Join(sortedKeys(changed), ", "))
			result, syncErr := im.Import(sortedKeys(changed))
			if syncErr != nil {
				logger.Error("sync failed", "error", syncErr)
			}
			result.report(logger)
		}

		<-ticker.C
	}
}

func defaultRoot() string {
	if v := strings.TrimSpace(os.Getenv("GO_DOC_WORKSPACE")); v != "" {
		return v
	}
	return "/home/dnoulet/go"
}

// Result summarises one sync pass.
type Result struct {
	Created   int
	Updated   int
	Unchanged int
	Skipped   int
	Failed    int
	Subdocs   int
	Pages     []pageResult
}

func (r Result) report(logger *slog.Logger) {
	logger.Info("sync complete",
		"created", r.Created,
		"updated", r.Updated,
		"unchanged", r.Unchanged,
		"skipped", r.Skipped,
		"subdocs", r.Subdocs,
		"failed", r.Failed,
	)
	for _, page := range r.Pages {
		attrs := []any{"project", page.Project, "slug", page.Slug, "status", string(page.Status)}
		if len(page.Notes) > 0 {
			attrs = append(attrs, "notes", strings.Join(page.Notes, ", "))
		}
		if page.Sub {
			attrs = append(attrs, "subdocument", true)
		}
		if page.Status == docgen.StatusSkipped {
			logger.Warn("skipped", attrs...)
			continue
		}
		logger.Info("imported", attrs...)
	}
}

// pageResult is the outcome for one written page.
type pageResult struct {
	Project string
	Slug    string
	Status  docgen.Status
	Notes   []string
	Sub     bool
}

// ---------------------------------------------------------------------------
// Import pipeline
// ---------------------------------------------------------------------------

// importer holds the sync state shared by the one-shot and watch paths.
type importer struct {
	opts   options
	engine *storage.Engine
	logger *slog.Logger

	only    map[string]struct{}
	skipSet map[string]struct{}

	// seen maps a project name to the signature observed on the previous poll,
	// which is what makes "did this change?" a cheap map lookup.
	seen map[string]string
	mu   sync.Mutex
}

func newImporter(opts options, engine *storage.Engine, logger *slog.Logger) *importer {
	return &importer{
		opts:    opts,
		engine:  engine,
		logger:  logger,
		only:    splitSet(opts.apps),
		skipSet: splitSet(opts.skip),
		seen:    map[string]string{},
	}
}

func splitSet(raw string) map[string]struct{} {
	out := map[string]struct{}{}
	for _, part := range strings.Split(raw, ",") {
		if trimmed := strings.TrimSpace(part); trimmed != "" {
			out[trimmed] = struct{}{}
		}
	}
	return out
}

// Discover returns the project names selected by -apps and -skip.
func (im *importer) Discover() ([]string, error) {
	entries, err := os.ReadDir(im.opts.root)
	if err != nil {
		return nil, fmt.Errorf("cannot read workspace root %s: %w", im.opts.root, err)
	}

	var names []string
	for _, entry := range entries {
		name := entry.Name()
		if !entry.IsDir() || !strings.HasPrefix(name, "go-") {
			continue
		}
		if _, skip := im.skipSet[name]; skip {
			continue
		}
		if len(im.only) > 0 {
			if _, wanted := im.only[name]; !wanted {
				continue
			}
		}
		names = append(names, name)
	}
	sort.Strings(names)
	return names, nil
}

// Signatures returns the current source signature per project.
func (im *importer) Signatures() (map[string]string, error) {
	names, err := im.Discover()
	if err != nil {
		return nil, err
	}
	out := make(map[string]string, len(names))
	for _, name := range names {
		project, err := docgen.Analyze(name, filepath.Join(im.opts.root, name))
		if err != nil {
			im.logger.Debug("cannot analyse project", "project", name, "error", err)
			continue
		}
		out[name] = project.Signature
	}
	return out, nil
}

// SyncOnce analyses and imports every discovered project.
func (im *importer) SyncOnce() (Result, error) {
	names, err := im.Discover()
	if err != nil {
		return Result{}, err
	}
	if len(names) == 0 {
		im.logger.Warn("no go-* projects discovered", "root", im.opts.root)
	}
	return im.Import(names)
}

// Scan returns the projects whose signature changed since the previous poll,
// including any project seen for the first time. Remembering every signature —
// not just the changed ones — keeps a project from re-importing forever once it
// has been synced. A project that has left the workspace is forgotten and
// reported: its page stays on disk untouched, because a sync that silently
// deletes documentation is worse than one that leaves it stale.
func (im *importer) Scan() (map[string]string, error) {
	sigs, err := im.Signatures()
	if err != nil {
		return nil, err
	}

	im.mu.Lock()
	defer im.mu.Unlock()

	changed := map[string]string{}
	for name, sig := range sigs {
		if previous, seen := im.seen[name]; !seen || previous != sig {
			changed[name] = sig
		}
	}
	for name, sig := range sigs {
		im.seen[name] = sig
	}

	var departed []string
	for name := range im.seen {
		if _, present := sigs[name]; !present {
			departed = append(departed, name)
		}
	}
	sort.Strings(departed)
	for _, name := range departed {
		delete(im.seen, name)
		im.logger.Info("project left the workspace; its page was left as is",
			"project", name,
			"hint", "delete the page by hand if the project is gone for good",
		)
	}

	return changed, nil
}

// Import analyses each named project and writes its page through the storage
// engine, so the result is slugified, atomically written and immediately
// visible to the API and the frontend.
func (im *importer) Import(names []string) (Result, error) {
	var (
		result Result
		mu     sync.Mutex
		wg     sync.WaitGroup
	)

	for _, name := range names {
		wg.Add(1)
		go func(name string) {
			defer wg.Done()

			pages, subdocs, err := im.importOne(name)

			mu.Lock()
			defer mu.Unlock()
			if err != nil {
				result.Failed++
				im.logger.Error("import failed", "project", name, "error", err)
				return
			}
			result.Subdocs += subdocs
			result.Pages = append(result.Pages, pages...)
			for _, page := range pages {
				if page.Sub {
					continue
				}
				switch page.Status {
				case docgen.StatusCreated:
					result.Created++
				case docgen.StatusUpdated:
					result.Updated++
				case docgen.StatusUnchanged:
					result.Unchanged++
				case docgen.StatusSkipped:
					result.Skipped++
				}
			}
		}(name)
	}
	wg.Wait()

	sort.Slice(result.Pages, func(i, j int) bool { return result.Pages[i].Slug < result.Pages[j].Slug })
	return result, nil
}

// importOne builds and writes the page for a single project, plus any sub-pages
// when -import-subdocs is set.
func (im *importer) importOne(name string) ([]pageResult, int, error) {
	root := filepath.Join(im.opts.root, name)

	project, err := docgen.Analyze(name, root)
	if err != nil {
		return nil, 0, err
	}
	page, err := docgen.Render(project)
	if err != nil {
		return nil, 0, err
	}

	slug := storage.Slugify(name)

	status, err := im.write(slug, page)
	if err != nil {
		return nil, 0, err
	}

	out := []pageResult{{
		Project: name,
		Slug:    slug,
		Status:  status,
		Notes:   notes(project),
	}}

	if !im.opts.importSubdoc {
		return out, 0, nil
	}

	subpages := im.subdocPages(project)
	for _, sub := range subpages {
		subStatus, err := im.write(sub.slug, sub.body)
		if err != nil {
			im.logger.Warn("subdocument import failed",
				"project", name, "path", sub.source, "error", err)
			continue
		}
		out = append(out, pageResult{
			Project: name,
			Slug:    sub.slug,
			Status:  subStatus,
			Sub:     true,
		})
	}
	return out, len(subpages), nil
}

type subdocPage struct {
	source string
	slug   string
	body   string
}

// subdocPages turns each Markdown file the project already owns into its own
// go-doc page, wrapped so a later sync refreshes it instead of orphaning it.
func (im *importer) subdocPages(project *docgen.Project) []subdocPage {
	base := storage.Slugify(project.Name)
	var out []subdocPage

	for _, doc := range project.Docs {
		if doc.Path == project.Readme {
			continue // already the summary of the main page
		}
		stem := strings.TrimSuffix(filepath.Base(doc.Path), filepath.Ext(doc.Path))
		slug := storage.Slugify(project.Name + "-" + stem)
		if slug == "" || slug == base {
			continue
		}

		data, err := os.ReadFile(filepath.Join(project.Path, filepath.FromSlash(doc.Path)))
		if err != nil {
			continue
		}

		body := fmt.Sprintf(
			"Imported from %s in the %s repository.\n\n---\n\n%s",
			doc.Path, project.Name, strings.TrimRight(string(data), "\n"),
		)
		out = append(out, subdocPage{
			source: doc.Path,
			slug:   slug,
			body:   docgen.Wrap(project.Name+":"+doc.Path, project.Signature, body),
		})
	}
	return out
}

// write merges a generated page with whatever is already on disk. Existing prose
// outside the markers survives; a page with no markers is skipped unless force.
func (im *importer) write(slug, page string) (docgen.Status, error) {
	var existing string
	if data, err := os.ReadFile(filepath.Join(im.engine.DocsDir(), slug+".md")); err == nil {
		existing = string(data)
	}

	merged, status, err := docgen.Merge(existing, page, im.opts.force)
	if err != nil {
		return "", err
	}

	if im.opts.dryRun {
		im.logger.Debug("dry run", "slug", slug, "status", string(status), "bytes", len(merged))
		return status, nil
	}
	if _, err := im.engine.SaveDoc(slug, merged); err != nil {
		return "", err
	}
	return status, nil
}

// notes surfaces anything a reader should know about a generated page.
func notes(project *docgen.Project) []string {
	var out []string
	if project.Readme == "" {
		out = append(out, "no README found")
	}
	if len(project.Routes) == 0 {
		out = append(out, "no HTTP routes detected")
	}
	if len(project.Modules) == 0 {
		out = append(out, "no go.mod found")
	}
	return out
}

func sortedKeys(m map[string]string) []string {
	keys := make([]string, 0, len(m))
	for key := range m {
		keys = append(keys, key)
	}
	sort.Strings(keys)
	return keys
}