498 lines
13 KiB
Go
498 lines
13 KiB
Go
// Command docgen analyses every go-* project under a workspace root and imports
|
|
// audience-segmented documentation for each one into go-doc.
|
|
//
|
|
// One-shot sync:
|
|
//
|
|
// go run ./scripts/docgen -root /home/dnoulet/go -data ../data
|
|
//
|
|
// Continuous sync, re-importing whenever a project's docs or source change:
|
|
//
|
|
// go run ./scripts/docgen -root /home/dnoulet/go -data ../data -watch
|
|
//
|
|
// Imported pages are wrapped in generated markers. Re-running replaces only the
|
|
// machine-written region, so prose written outside the markers survives, and a
|
|
// page that was never generated is left untouched unless -force is passed.
|
|
package main
|
|
|
|
import (
|
|
"flag"
|
|
"fmt"
|
|
"log/slog"
|
|
"os"
|
|
"path/filepath"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"go-doc/docgen"
|
|
"go-doc/storage"
|
|
)
|
|
|
|
func main() {
|
|
if err := run(); err != nil {
|
|
slog.Error("fatal", "error", err)
|
|
os.Exit(1)
|
|
}
|
|
}
|
|
|
|
type options struct {
|
|
root string
|
|
dataDir string
|
|
apps string
|
|
skip string
|
|
watch bool
|
|
interval time.Duration
|
|
dryRun bool
|
|
force bool
|
|
importSubdoc bool
|
|
verbose bool
|
|
}
|
|
|
|
func run() error {
|
|
var opts options
|
|
|
|
flag.StringVar(&opts.root, "root", defaultRoot(), "directory containing the go-* projects")
|
|
flag.StringVar(&opts.dataDir, "data", storage.ResolveRoot(), "go-doc data directory to import into")
|
|
flag.StringVar(&opts.apps, "apps", "", "comma-separated project names to limit the run (default: all)")
|
|
flag.StringVar(&opts.skip, "skip", "go-doc", "comma-separated project names to ignore")
|
|
flag.BoolVar(&opts.watch, "watch", false, "keep running and re-import when a project changes")
|
|
flag.DurationVar(&opts.interval, "interval", 5*time.Second, "poll interval for -watch")
|
|
flag.BoolVar(&opts.dryRun, "dry-run", false, "analyse and render without writing anything")
|
|
flag.BoolVar(&opts.force, "force", false, "overwrite pages that carry no generated markers")
|
|
flag.BoolVar(&opts.importSubdoc, "import-subdocs", false, "also import each project's own Markdown as separate go-doc pages")
|
|
flag.BoolVar(&opts.verbose, "verbose", false, "log at debug level")
|
|
flag.Parse()
|
|
|
|
level := slog.LevelInfo
|
|
if opts.verbose {
|
|
level = slog.LevelDebug
|
|
}
|
|
logger := slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: level}))
|
|
|
|
engine, err := storage.New(opts.dataDir)
|
|
if err != nil {
|
|
return fmt.Errorf("cannot open the go-doc data directory: %w", err)
|
|
}
|
|
|
|
im := newImporter(opts, engine, logger)
|
|
|
|
if !opts.watch {
|
|
result, err := im.SyncOnce()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
result.report(logger)
|
|
if result.Failed > 0 {
|
|
return fmt.Errorf("%d project(s) failed to import", result.Failed)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
logger.Info("watching for documentation changes",
|
|
"root", opts.root,
|
|
"interval", opts.interval.String(),
|
|
"data", engine.Root(),
|
|
"hint", "Ctrl-C to stop",
|
|
)
|
|
|
|
ticker := time.NewTicker(opts.interval)
|
|
defer ticker.Stop()
|
|
|
|
for {
|
|
changed, err := im.Scan()
|
|
switch {
|
|
case err != nil:
|
|
logger.Error("scan failed", "error", err)
|
|
case len(changed) == 0:
|
|
logger.Debug("no changes")
|
|
default:
|
|
logger.Info("change detected", "projects", strings.Join(sortedKeys(changed), ", "))
|
|
result, syncErr := im.Import(sortedKeys(changed))
|
|
if syncErr != nil {
|
|
logger.Error("sync failed", "error", syncErr)
|
|
}
|
|
result.report(logger)
|
|
}
|
|
|
|
<-ticker.C
|
|
}
|
|
}
|
|
|
|
func defaultRoot() string {
|
|
if v := strings.TrimSpace(os.Getenv("GO_DOC_WORKSPACE")); v != "" {
|
|
return v
|
|
}
|
|
return "/home/dnoulet/go"
|
|
}
|
|
|
|
// Result summarises one sync pass.
|
|
type Result struct {
|
|
Created int
|
|
Updated int
|
|
Unchanged int
|
|
Skipped int
|
|
Failed int
|
|
Subdocs int
|
|
Pages []pageResult
|
|
}
|
|
|
|
func (r Result) report(logger *slog.Logger) {
|
|
logger.Info("sync complete",
|
|
"created", r.Created,
|
|
"updated", r.Updated,
|
|
"unchanged", r.Unchanged,
|
|
"skipped", r.Skipped,
|
|
"subdocs", r.Subdocs,
|
|
"failed", r.Failed,
|
|
)
|
|
for _, page := range r.Pages {
|
|
attrs := []any{"project", page.Project, "slug", page.Slug, "status", string(page.Status)}
|
|
if len(page.Notes) > 0 {
|
|
attrs = append(attrs, "notes", strings.Join(page.Notes, ", "))
|
|
}
|
|
if page.Sub {
|
|
attrs = append(attrs, "subdocument", true)
|
|
}
|
|
if page.Status == docgen.StatusSkipped {
|
|
logger.Warn("skipped", attrs...)
|
|
continue
|
|
}
|
|
logger.Info("imported", attrs...)
|
|
}
|
|
}
|
|
|
|
// pageResult is the outcome for one written page.
|
|
type pageResult struct {
|
|
Project string
|
|
Slug string
|
|
Status docgen.Status
|
|
Notes []string
|
|
Sub bool
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Import pipeline
|
|
// ---------------------------------------------------------------------------
|
|
|
|
// importer holds the sync state shared by the one-shot and watch paths.
|
|
type importer struct {
|
|
opts options
|
|
engine *storage.Engine
|
|
logger *slog.Logger
|
|
|
|
only map[string]struct{}
|
|
skipSet map[string]struct{}
|
|
|
|
// seen maps a project name to the signature observed on the previous poll,
|
|
// which is what makes "did this change?" a cheap map lookup.
|
|
seen map[string]string
|
|
mu sync.Mutex
|
|
}
|
|
|
|
func newImporter(opts options, engine *storage.Engine, logger *slog.Logger) *importer {
|
|
return &importer{
|
|
opts: opts,
|
|
engine: engine,
|
|
logger: logger,
|
|
only: splitSet(opts.apps),
|
|
skipSet: splitSet(opts.skip),
|
|
seen: map[string]string{},
|
|
}
|
|
}
|
|
|
|
func splitSet(raw string) map[string]struct{} {
|
|
out := map[string]struct{}{}
|
|
for _, part := range strings.Split(raw, ",") {
|
|
if trimmed := strings.TrimSpace(part); trimmed != "" {
|
|
out[trimmed] = struct{}{}
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// Discover returns the project names selected by -apps and -skip.
|
|
func (im *importer) Discover() ([]string, error) {
|
|
entries, err := os.ReadDir(im.opts.root)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("cannot read workspace root %s: %w", im.opts.root, err)
|
|
}
|
|
|
|
var names []string
|
|
for _, entry := range entries {
|
|
name := entry.Name()
|
|
if !entry.IsDir() || !strings.HasPrefix(name, "go-") {
|
|
continue
|
|
}
|
|
if _, skip := im.skipSet[name]; skip {
|
|
continue
|
|
}
|
|
if len(im.only) > 0 {
|
|
if _, wanted := im.only[name]; !wanted {
|
|
continue
|
|
}
|
|
}
|
|
names = append(names, name)
|
|
}
|
|
sort.Strings(names)
|
|
return names, nil
|
|
}
|
|
|
|
// Signatures returns the current source signature per project.
|
|
func (im *importer) Signatures() (map[string]string, error) {
|
|
names, err := im.Discover()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out := make(map[string]string, len(names))
|
|
for _, name := range names {
|
|
project, err := docgen.Analyze(name, filepath.Join(im.opts.root, name))
|
|
if err != nil {
|
|
im.logger.Debug("cannot analyse project", "project", name, "error", err)
|
|
continue
|
|
}
|
|
out[name] = project.Signature
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// SyncOnce analyses and imports every discovered project.
|
|
func (im *importer) SyncOnce() (Result, error) {
|
|
names, err := im.Discover()
|
|
if err != nil {
|
|
return Result{}, err
|
|
}
|
|
if len(names) == 0 {
|
|
im.logger.Warn("no go-* projects discovered", "root", im.opts.root)
|
|
}
|
|
return im.Import(names)
|
|
}
|
|
|
|
// Scan returns the projects whose signature changed since the previous poll,
|
|
// including any project seen for the first time. Remembering every signature —
|
|
// not just the changed ones — keeps a project from re-importing forever once it
|
|
// has been synced. A project that has left the workspace is forgotten and
|
|
// reported: its page stays on disk untouched, because a sync that silently
|
|
// deletes documentation is worse than one that leaves it stale.
|
|
func (im *importer) Scan() (map[string]string, error) {
|
|
sigs, err := im.Signatures()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
im.mu.Lock()
|
|
defer im.mu.Unlock()
|
|
|
|
changed := map[string]string{}
|
|
for name, sig := range sigs {
|
|
if previous, seen := im.seen[name]; !seen || previous != sig {
|
|
changed[name] = sig
|
|
}
|
|
}
|
|
for name, sig := range sigs {
|
|
im.seen[name] = sig
|
|
}
|
|
|
|
var departed []string
|
|
for name := range im.seen {
|
|
if _, present := sigs[name]; !present {
|
|
departed = append(departed, name)
|
|
}
|
|
}
|
|
sort.Strings(departed)
|
|
for _, name := range departed {
|
|
delete(im.seen, name)
|
|
im.logger.Info("project left the workspace; its page was left as is",
|
|
"project", name,
|
|
"hint", "delete the page by hand if the project is gone for good",
|
|
)
|
|
}
|
|
|
|
return changed, nil
|
|
}
|
|
|
|
// Import analyses each named project and writes its page through the storage
|
|
// engine, so the result is slugified, atomically written and immediately
|
|
// visible to the API and the frontend.
|
|
func (im *importer) Import(names []string) (Result, error) {
|
|
var (
|
|
result Result
|
|
mu sync.Mutex
|
|
wg sync.WaitGroup
|
|
)
|
|
|
|
for _, name := range names {
|
|
wg.Add(1)
|
|
go func(name string) {
|
|
defer wg.Done()
|
|
|
|
pages, subdocs, err := im.importOne(name)
|
|
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
if err != nil {
|
|
result.Failed++
|
|
im.logger.Error("import failed", "project", name, "error", err)
|
|
return
|
|
}
|
|
result.Subdocs += subdocs
|
|
result.Pages = append(result.Pages, pages...)
|
|
for _, page := range pages {
|
|
if page.Sub {
|
|
continue
|
|
}
|
|
switch page.Status {
|
|
case docgen.StatusCreated:
|
|
result.Created++
|
|
case docgen.StatusUpdated:
|
|
result.Updated++
|
|
case docgen.StatusUnchanged:
|
|
result.Unchanged++
|
|
case docgen.StatusSkipped:
|
|
result.Skipped++
|
|
}
|
|
}
|
|
}(name)
|
|
}
|
|
wg.Wait()
|
|
|
|
sort.Slice(result.Pages, func(i, j int) bool { return result.Pages[i].Slug < result.Pages[j].Slug })
|
|
return result, nil
|
|
}
|
|
|
|
// importOne builds and writes the page for a single project, plus any sub-pages
|
|
// when -import-subdocs is set.
|
|
func (im *importer) importOne(name string) ([]pageResult, int, error) {
|
|
root := filepath.Join(im.opts.root, name)
|
|
|
|
project, err := docgen.Analyze(name, root)
|
|
if err != nil {
|
|
return nil, 0, err
|
|
}
|
|
page, err := docgen.Render(project)
|
|
if err != nil {
|
|
return nil, 0, err
|
|
}
|
|
|
|
slug := storage.Slugify(name)
|
|
|
|
status, err := im.write(slug, page)
|
|
if err != nil {
|
|
return nil, 0, err
|
|
}
|
|
|
|
out := []pageResult{{
|
|
Project: name,
|
|
Slug: slug,
|
|
Status: status,
|
|
Notes: notes(project),
|
|
}}
|
|
|
|
if !im.opts.importSubdoc {
|
|
return out, 0, nil
|
|
}
|
|
|
|
subpages := im.subdocPages(project)
|
|
for _, sub := range subpages {
|
|
subStatus, err := im.write(sub.slug, sub.body)
|
|
if err != nil {
|
|
im.logger.Warn("subdocument import failed",
|
|
"project", name, "path", sub.source, "error", err)
|
|
continue
|
|
}
|
|
out = append(out, pageResult{
|
|
Project: name,
|
|
Slug: sub.slug,
|
|
Status: subStatus,
|
|
Sub: true,
|
|
})
|
|
}
|
|
return out, len(subpages), nil
|
|
}
|
|
|
|
type subdocPage struct {
|
|
source string
|
|
slug string
|
|
body string
|
|
}
|
|
|
|
// subdocPages turns each Markdown file the project already owns into its own
|
|
// go-doc page, wrapped so a later sync refreshes it instead of orphaning it.
|
|
func (im *importer) subdocPages(project *docgen.Project) []subdocPage {
|
|
base := storage.Slugify(project.Name)
|
|
var out []subdocPage
|
|
|
|
for _, doc := range project.Docs {
|
|
if doc.Path == project.Readme {
|
|
continue // already the summary of the main page
|
|
}
|
|
stem := strings.TrimSuffix(filepath.Base(doc.Path), filepath.Ext(doc.Path))
|
|
slug := storage.Slugify(project.Name + "-" + stem)
|
|
if slug == "" || slug == base {
|
|
continue
|
|
}
|
|
|
|
data, err := os.ReadFile(filepath.Join(project.Path, filepath.FromSlash(doc.Path)))
|
|
if err != nil {
|
|
continue
|
|
}
|
|
|
|
body := fmt.Sprintf(
|
|
"Imported from %s in the %s repository.\n\n---\n\n%s",
|
|
doc.Path, project.Name, strings.TrimRight(string(data), "\n"),
|
|
)
|
|
out = append(out, subdocPage{
|
|
source: doc.Path,
|
|
slug: slug,
|
|
body: docgen.Wrap(project.Name+":"+doc.Path, project.Signature, body),
|
|
})
|
|
}
|
|
return out
|
|
}
|
|
|
|
// write merges a generated page with whatever is already on disk. Existing prose
|
|
// outside the markers survives; a page with no markers is skipped unless force.
|
|
func (im *importer) write(slug, page string) (docgen.Status, error) {
|
|
var existing string
|
|
if data, err := os.ReadFile(filepath.Join(im.engine.DocsDir(), slug+".md")); err == nil {
|
|
existing = string(data)
|
|
}
|
|
|
|
merged, status, err := docgen.Merge(existing, page, im.opts.force)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
if im.opts.dryRun {
|
|
im.logger.Debug("dry run", "slug", slug, "status", string(status), "bytes", len(merged))
|
|
return status, nil
|
|
}
|
|
if _, err := im.engine.SaveDoc(slug, merged); err != nil {
|
|
return "", err
|
|
}
|
|
return status, nil
|
|
}
|
|
|
|
// notes surfaces anything a reader should know about a generated page.
|
|
func notes(project *docgen.Project) []string {
|
|
var out []string
|
|
if project.Readme == "" {
|
|
out = append(out, "no README found")
|
|
}
|
|
if len(project.Routes) == 0 {
|
|
out = append(out, "no HTTP routes detected")
|
|
}
|
|
if len(project.Modules) == 0 {
|
|
out = append(out, "no go.mod found")
|
|
}
|
|
return out
|
|
}
|
|
|
|
func sortedKeys(m map[string]string) []string {
|
|
keys := make([]string, 0, len(m))
|
|
for key := range m {
|
|
keys = append(keys, key)
|
|
}
|
|
sort.Strings(keys)
|
|
return keys
|
|
}
|