initial commit

This commit is contained in:
Daniel Noulet 2026-10-07 15:03:50 +02:00
commit 23816d00f7
71 changed files with 14497 additions and 0 deletions

View file

@ -0,0 +1,498 @@
// Command docgen analyses every go-* project under a workspace root and imports
// audience-segmented documentation for each one into go-doc.
//
// One-shot sync:
//
// go run ./scripts/docgen -root /home/dnoulet/go -data ../data
//
// Continuous sync, re-importing whenever a project's docs or source change:
//
// go run ./scripts/docgen -root /home/dnoulet/go -data ../data -watch
//
// Imported pages are wrapped in generated markers. Re-running replaces only the
// machine-written region, so prose written outside the markers survives, and a
// page that was never generated is left untouched unless -force is passed.
package main
import (
"flag"
"fmt"
"log/slog"
"os"
"path/filepath"
"sort"
"strings"
"sync"
"time"
"go-doc/docgen"
"go-doc/storage"
)
func main() {
if err := run(); err != nil {
slog.Error("fatal", "error", err)
os.Exit(1)
}
}
type options struct {
root string
dataDir string
apps string
skip string
watch bool
interval time.Duration
dryRun bool
force bool
importSubdoc bool
verbose bool
}
func run() error {
var opts options
flag.StringVar(&opts.root, "root", defaultRoot(), "directory containing the go-* projects")
flag.StringVar(&opts.dataDir, "data", storage.ResolveRoot(), "go-doc data directory to import into")
flag.StringVar(&opts.apps, "apps", "", "comma-separated project names to limit the run (default: all)")
flag.StringVar(&opts.skip, "skip", "go-doc", "comma-separated project names to ignore")
flag.BoolVar(&opts.watch, "watch", false, "keep running and re-import when a project changes")
flag.DurationVar(&opts.interval, "interval", 5*time.Second, "poll interval for -watch")
flag.BoolVar(&opts.dryRun, "dry-run", false, "analyse and render without writing anything")
flag.BoolVar(&opts.force, "force", false, "overwrite pages that carry no generated markers")
flag.BoolVar(&opts.importSubdoc, "import-subdocs", false, "also import each project's own Markdown as separate go-doc pages")
flag.BoolVar(&opts.verbose, "verbose", false, "log at debug level")
flag.Parse()
level := slog.LevelInfo
if opts.verbose {
level = slog.LevelDebug
}
logger := slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: level}))
engine, err := storage.New(opts.dataDir)
if err != nil {
return fmt.Errorf("cannot open the go-doc data directory: %w", err)
}
im := newImporter(opts, engine, logger)
if !opts.watch {
result, err := im.SyncOnce()
if err != nil {
return err
}
result.report(logger)
if result.Failed > 0 {
return fmt.Errorf("%d project(s) failed to import", result.Failed)
}
return nil
}
logger.Info("watching for documentation changes",
"root", opts.root,
"interval", opts.interval.String(),
"data", engine.Root(),
"hint", "Ctrl-C to stop",
)
ticker := time.NewTicker(opts.interval)
defer ticker.Stop()
for {
changed, err := im.Scan()
switch {
case err != nil:
logger.Error("scan failed", "error", err)
case len(changed) == 0:
logger.Debug("no changes")
default:
logger.Info("change detected", "projects", strings.Join(sortedKeys(changed), ", "))
result, syncErr := im.Import(sortedKeys(changed))
if syncErr != nil {
logger.Error("sync failed", "error", syncErr)
}
result.report(logger)
}
<-ticker.C
}
}
func defaultRoot() string {
if v := strings.TrimSpace(os.Getenv("GO_DOC_WORKSPACE")); v != "" {
return v
}
return "/home/dnoulet/go"
}
// Result summarises one sync pass.
type Result struct {
Created int
Updated int
Unchanged int
Skipped int
Failed int
Subdocs int
Pages []pageResult
}
func (r Result) report(logger *slog.Logger) {
logger.Info("sync complete",
"created", r.Created,
"updated", r.Updated,
"unchanged", r.Unchanged,
"skipped", r.Skipped,
"subdocs", r.Subdocs,
"failed", r.Failed,
)
for _, page := range r.Pages {
attrs := []any{"project", page.Project, "slug", page.Slug, "status", string(page.Status)}
if len(page.Notes) > 0 {
attrs = append(attrs, "notes", strings.Join(page.Notes, ", "))
}
if page.Sub {
attrs = append(attrs, "subdocument", true)
}
if page.Status == docgen.StatusSkipped {
logger.Warn("skipped", attrs...)
continue
}
logger.Info("imported", attrs...)
}
}
// pageResult is the outcome for one written page.
type pageResult struct {
Project string
Slug string
Status docgen.Status
Notes []string
Sub bool
}
// ---------------------------------------------------------------------------
// Import pipeline
// ---------------------------------------------------------------------------
// importer holds the sync state shared by the one-shot and watch paths.
type importer struct {
opts options
engine *storage.Engine
logger *slog.Logger
only map[string]struct{}
skipSet map[string]struct{}
// seen maps a project name to the signature observed on the previous poll,
// which is what makes "did this change?" a cheap map lookup.
seen map[string]string
mu sync.Mutex
}
func newImporter(opts options, engine *storage.Engine, logger *slog.Logger) *importer {
return &importer{
opts: opts,
engine: engine,
logger: logger,
only: splitSet(opts.apps),
skipSet: splitSet(opts.skip),
seen: map[string]string{},
}
}
func splitSet(raw string) map[string]struct{} {
out := map[string]struct{}{}
for _, part := range strings.Split(raw, ",") {
if trimmed := strings.TrimSpace(part); trimmed != "" {
out[trimmed] = struct{}{}
}
}
return out
}
// Discover returns the project names selected by -apps and -skip.
func (im *importer) Discover() ([]string, error) {
entries, err := os.ReadDir(im.opts.root)
if err != nil {
return nil, fmt.Errorf("cannot read workspace root %s: %w", im.opts.root, err)
}
var names []string
for _, entry := range entries {
name := entry.Name()
if !entry.IsDir() || !strings.HasPrefix(name, "go-") {
continue
}
if _, skip := im.skipSet[name]; skip {
continue
}
if len(im.only) > 0 {
if _, wanted := im.only[name]; !wanted {
continue
}
}
names = append(names, name)
}
sort.Strings(names)
return names, nil
}
// Signatures returns the current source signature per project.
func (im *importer) Signatures() (map[string]string, error) {
names, err := im.Discover()
if err != nil {
return nil, err
}
out := make(map[string]string, len(names))
for _, name := range names {
project, err := docgen.Analyze(name, filepath.Join(im.opts.root, name))
if err != nil {
im.logger.Debug("cannot analyse project", "project", name, "error", err)
continue
}
out[name] = project.Signature
}
return out, nil
}
// SyncOnce analyses and imports every discovered project.
func (im *importer) SyncOnce() (Result, error) {
names, err := im.Discover()
if err != nil {
return Result{}, err
}
if len(names) == 0 {
im.logger.Warn("no go-* projects discovered", "root", im.opts.root)
}
return im.Import(names)
}
// Scan returns the projects whose signature changed since the previous poll,
// including any project seen for the first time. Remembering every signature —
// not just the changed ones — keeps a project from re-importing forever once it
// has been synced. A project that has left the workspace is forgotten and
// reported: its page stays on disk untouched, because a sync that silently
// deletes documentation is worse than one that leaves it stale.
func (im *importer) Scan() (map[string]string, error) {
sigs, err := im.Signatures()
if err != nil {
return nil, err
}
im.mu.Lock()
defer im.mu.Unlock()
changed := map[string]string{}
for name, sig := range sigs {
if previous, seen := im.seen[name]; !seen || previous != sig {
changed[name] = sig
}
}
for name, sig := range sigs {
im.seen[name] = sig
}
var departed []string
for name := range im.seen {
if _, present := sigs[name]; !present {
departed = append(departed, name)
}
}
sort.Strings(departed)
for _, name := range departed {
delete(im.seen, name)
im.logger.Info("project left the workspace; its page was left as is",
"project", name,
"hint", "delete the page by hand if the project is gone for good",
)
}
return changed, nil
}
// Import analyses each named project and writes its page through the storage
// engine, so the result is slugified, atomically written and immediately
// visible to the API and the frontend.
func (im *importer) Import(names []string) (Result, error) {
var (
result Result
mu sync.Mutex
wg sync.WaitGroup
)
for _, name := range names {
wg.Add(1)
go func(name string) {
defer wg.Done()
pages, subdocs, err := im.importOne(name)
mu.Lock()
defer mu.Unlock()
if err != nil {
result.Failed++
im.logger.Error("import failed", "project", name, "error", err)
return
}
result.Subdocs += subdocs
result.Pages = append(result.Pages, pages...)
for _, page := range pages {
if page.Sub {
continue
}
switch page.Status {
case docgen.StatusCreated:
result.Created++
case docgen.StatusUpdated:
result.Updated++
case docgen.StatusUnchanged:
result.Unchanged++
case docgen.StatusSkipped:
result.Skipped++
}
}
}(name)
}
wg.Wait()
sort.Slice(result.Pages, func(i, j int) bool { return result.Pages[i].Slug < result.Pages[j].Slug })
return result, nil
}
// importOne builds and writes the page for a single project, plus any sub-pages
// when -import-subdocs is set.
func (im *importer) importOne(name string) ([]pageResult, int, error) {
root := filepath.Join(im.opts.root, name)
project, err := docgen.Analyze(name, root)
if err != nil {
return nil, 0, err
}
page, err := docgen.Render(project)
if err != nil {
return nil, 0, err
}
slug := storage.Slugify(name)
status, err := im.write(slug, page)
if err != nil {
return nil, 0, err
}
out := []pageResult{{
Project: name,
Slug: slug,
Status: status,
Notes: notes(project),
}}
if !im.opts.importSubdoc {
return out, 0, nil
}
subpages := im.subdocPages(project)
for _, sub := range subpages {
subStatus, err := im.write(sub.slug, sub.body)
if err != nil {
im.logger.Warn("subdocument import failed",
"project", name, "path", sub.source, "error", err)
continue
}
out = append(out, pageResult{
Project: name,
Slug: sub.slug,
Status: subStatus,
Sub: true,
})
}
return out, len(subpages), nil
}
type subdocPage struct {
source string
slug string
body string
}
// subdocPages turns each Markdown file the project already owns into its own
// go-doc page, wrapped so a later sync refreshes it instead of orphaning it.
func (im *importer) subdocPages(project *docgen.Project) []subdocPage {
base := storage.Slugify(project.Name)
var out []subdocPage
for _, doc := range project.Docs {
if doc.Path == project.Readme {
continue // already the summary of the main page
}
stem := strings.TrimSuffix(filepath.Base(doc.Path), filepath.Ext(doc.Path))
slug := storage.Slugify(project.Name + "-" + stem)
if slug == "" || slug == base {
continue
}
data, err := os.ReadFile(filepath.Join(project.Path, filepath.FromSlash(doc.Path)))
if err != nil {
continue
}
body := fmt.Sprintf(
"Imported from %s in the %s repository.\n\n---\n\n%s",
doc.Path, project.Name, strings.TrimRight(string(data), "\n"),
)
out = append(out, subdocPage{
source: doc.Path,
slug: slug,
body: docgen.Wrap(project.Name+":"+doc.Path, project.Signature, body),
})
}
return out
}
// write merges a generated page with whatever is already on disk. Existing prose
// outside the markers survives; a page with no markers is skipped unless force.
func (im *importer) write(slug, page string) (docgen.Status, error) {
var existing string
if data, err := os.ReadFile(filepath.Join(im.engine.DocsDir(), slug+".md")); err == nil {
existing = string(data)
}
merged, status, err := docgen.Merge(existing, page, im.opts.force)
if err != nil {
return "", err
}
if im.opts.dryRun {
im.logger.Debug("dry run", "slug", slug, "status", string(status), "bytes", len(merged))
return status, nil
}
if _, err := im.engine.SaveDoc(slug, merged); err != nil {
return "", err
}
return status, nil
}
// notes surfaces anything a reader should know about a generated page.
func notes(project *docgen.Project) []string {
var out []string
if project.Readme == "" {
out = append(out, "no README found")
}
if len(project.Routes) == 0 {
out = append(out, "no HTTP routes detected")
}
if len(project.Modules) == 0 {
out = append(out, "no go.mod found")
}
return out
}
func sortedKeys(m map[string]string) []string {
keys := make([]string, 0, len(m))
for key := range m {
keys = append(keys, key)
}
sort.Strings(keys)
return keys
}

View file

@ -0,0 +1,172 @@
package main
import (
"io"
"log/slog"
"os"
"path/filepath"
"sort"
"strings"
"testing"
"go-doc/storage"
)
func TestScanDetectsNewProjectsAndEditsButSettlesWhenIdle(t *testing.T) {
t.Parallel()
root := t.TempDir()
writeProject(t, filepath.Join(root, "go-alpha"))
im := newImporter(options{root: root}, newEngine(t), discardLogger())
// A project nobody has seen counts as changed.
changed := scan(t, im)
if want := []string{"go-alpha"}; !equal(changed, want) {
t.Fatalf("first scan = %v, want %v", changed, want)
}
// An untouched project must not re-import on every poll.
if changed := scan(t, im); len(changed) != 0 {
t.Errorf("idle scan reported %v, want no changes", changed)
}
// A project that appears mid-run is picked up.
writeProject(t, filepath.Join(root, "go-beta"))
if changed := scan(t, im); !equal(changed, []string{"go-beta"}) {
t.Errorf("scan after a new project = %v, want [go-beta]", changed)
}
// An edit to an existing project is picked up, once.
writeProject(t, filepath.Join(root, "go-alpha"))
if changed := scan(t, im); !equal(changed, []string{"go-alpha"}) {
t.Errorf("scan after an edit = %v, want [go-alpha]", changed)
}
if changed := scan(t, im); len(changed) != 0 {
t.Errorf("the edit was reported twice: %v", changed)
}
}
func TestScanForgetsProjectsThatLeaveTheWorkspace(t *testing.T) {
t.Parallel()
root := t.TempDir()
project := filepath.Join(root, "go-alpha")
writeProject(t, project)
var logs strings.Builder
im := newImporter(options{root: root}, newEngine(t),
slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelDebug})))
if changed := scan(t, im); len(changed) != 1 {
t.Fatalf("first scan = %v, want one project", changed)
}
logs.Reset()
if err := os.RemoveAll(project); err != nil {
t.Fatalf("remove project: %v", err)
}
if changed := scan(t, im); len(changed) != 0 {
t.Errorf("a departed project was treated as changed: %v", changed)
}
if !strings.Contains(logs.String(), "project left the workspace") {
t.Errorf("a departed project was not reported:\n%s", logs.String())
}
// Coming back is a change again rather than a silent no-op.
writeProject(t, project)
if changed := scan(t, im); !equal(changed, []string{"go-alpha"}) {
t.Errorf("scan after the project returned = %v, want [go-alpha]", changed)
}
}
func TestDiscoverHonoursAppsAndSkip(t *testing.T) {
t.Parallel()
root := t.TempDir()
for _, name := range []string{"go-alpha", "go-beta", "go-doc", "not-a-go-app"} {
writeProject(t, filepath.Join(root, name))
}
cases := []struct {
name string
opts options
want []string
}{
{name: "default skips go-doc", opts: options{root: root, skip: "go-doc"}, want: []string{"go-alpha", "go-beta"}},
{name: "-apps narrows the run", opts: options{root: root, skip: "", apps: "go-beta"}, want: []string{"go-beta"}},
{name: "no skip at all", opts: options{root: root}, want: []string{"go-alpha", "go-beta", "go-doc"}},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
im := newImporter(tc.opts, newEngine(t), discardLogger())
got, err := im.Discover()
if err != nil {
t.Fatalf("discover: %v", err)
}
if strings.Join(got, ",") != strings.Join(tc.want, ",") {
t.Errorf("discover = %v, want %v", got, tc.want)
}
})
}
}
func TestDiscoverReportsAMissingWorkspace(t *testing.T) {
t.Parallel()
im := newImporter(options{root: filepath.Join(t.TempDir(), "absent")}, newEngine(t), discardLogger())
if _, err := im.Discover(); err == nil {
t.Fatal("a missing workspace should be reported, not silently ignored")
}
}
func scan(t *testing.T, im *importer) []string {
t.Helper()
changed, err := im.Scan()
if err != nil {
t.Fatalf("scan: %v", err)
}
names := make([]string, 0, len(changed))
for name := range changed {
names = append(names, name)
}
sort.Strings(names)
return names
}
func equal(got, want []string) bool {
return strings.Join(got, ",") == strings.Join(want, ",")
}
func newEngine(t *testing.T) *storage.Engine {
t.Helper()
engine, err := storage.New(t.TempDir())
if err != nil {
t.Fatalf("provision engine: %v", err)
}
return engine
}
func discardLogger() *slog.Logger {
return slog.New(slog.NewTextHandler(io.Discard, &slog.HandlerOptions{Level: slog.LevelDebug}))
}
func writeProject(t *testing.T, root string) {
t.Helper()
if err := os.MkdirAll(filepath.Join(root, "internal"), 0o755); err != nil {
t.Fatalf("mkdir %s: %v", root, err)
}
files := map[string]string{
"go.mod": "module example.com/" + filepath.Base(root) + "\n\ngo 1.27.1\n",
"README.md": "# " + filepath.Base(root) + "\n\nIt serves an HTTP API.\n",
"internal/app.go": "package internal\n\n// Run starts the app.\nfunc Run() error { return nil }\n",
"internal/app_t.go": "placeholder\n",
}
for name, content := range files {
if err := os.WriteFile(filepath.Join(root, filepath.FromSlash(name)), []byte(content), 0o644); err != nil {
t.Fatalf("write %s: %v", name, err)
}
}
}