// Command docgen analyses every go-* project under a workspace root and imports // audience-segmented documentation for each one into go-doc. // // One-shot sync: // // go run ./scripts/docgen -root /home/dnoulet/go -data ../data // // Continuous sync, re-importing whenever a project's docs or source change: // // go run ./scripts/docgen -root /home/dnoulet/go -data ../data -watch // // Imported pages are wrapped in generated markers. Re-running replaces only the // machine-written region, so prose written outside the markers survives, and a // page that was never generated is left untouched unless -force is passed. package main import ( "flag" "fmt" "log/slog" "os" "path/filepath" "sort" "strings" "sync" "time" "go-doc/docgen" "go-doc/storage" ) func main() { if err := run(); err != nil { slog.Error("fatal", "error", err) os.Exit(1) } } type options struct { root string dataDir string apps string skip string watch bool interval time.Duration dryRun bool force bool importSubdoc bool verbose bool } func run() error { var opts options flag.StringVar(&opts.root, "root", defaultRoot(), "directory containing the go-* projects") flag.StringVar(&opts.dataDir, "data", storage.ResolveRoot(), "go-doc data directory to import into") flag.StringVar(&opts.apps, "apps", "", "comma-separated project names to limit the run (default: all)") flag.StringVar(&opts.skip, "skip", "go-doc", "comma-separated project names to ignore") flag.BoolVar(&opts.watch, "watch", false, "keep running and re-import when a project changes") flag.DurationVar(&opts.interval, "interval", 5*time.Second, "poll interval for -watch") flag.BoolVar(&opts.dryRun, "dry-run", false, "analyse and render without writing anything") flag.BoolVar(&opts.force, "force", false, "overwrite pages that carry no generated markers") flag.BoolVar(&opts.importSubdoc, "import-subdocs", false, "also import each project's own Markdown as separate go-doc pages") flag.BoolVar(&opts.verbose, "verbose", false, "log at debug level") flag.Parse() level := slog.LevelInfo if opts.verbose { level = slog.LevelDebug } logger := slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: level})) engine, err := storage.New(opts.dataDir) if err != nil { return fmt.Errorf("cannot open the go-doc data directory: %w", err) } im := newImporter(opts, engine, logger) if !opts.watch { result, err := im.SyncOnce() if err != nil { return err } result.report(logger) if result.Failed > 0 { return fmt.Errorf("%d project(s) failed to import", result.Failed) } return nil } logger.Info("watching for documentation changes", "root", opts.root, "interval", opts.interval.String(), "data", engine.Root(), "hint", "Ctrl-C to stop", ) ticker := time.NewTicker(opts.interval) defer ticker.Stop() for { changed, err := im.Scan() switch { case err != nil: logger.Error("scan failed", "error", err) case len(changed) == 0: logger.Debug("no changes") default: logger.Info("change detected", "projects", strings.Join(sortedKeys(changed), ", ")) result, syncErr := im.Import(sortedKeys(changed)) if syncErr != nil { logger.Error("sync failed", "error", syncErr) } result.report(logger) } <-ticker.C } } func defaultRoot() string { if v := strings.TrimSpace(os.Getenv("GO_DOC_WORKSPACE")); v != "" { return v } return "/home/dnoulet/go" } // Result summarises one sync pass. type Result struct { Created int Updated int Unchanged int Skipped int Failed int Subdocs int Pages []pageResult } func (r Result) report(logger *slog.Logger) { logger.Info("sync complete", "created", r.Created, "updated", r.Updated, "unchanged", r.Unchanged, "skipped", r.Skipped, "subdocs", r.Subdocs, "failed", r.Failed, ) for _, page := range r.Pages { attrs := []any{"project", page.Project, "slug", page.Slug, "status", string(page.Status)} if len(page.Notes) > 0 { attrs = append(attrs, "notes", strings.Join(page.Notes, ", ")) } if page.Sub { attrs = append(attrs, "subdocument", true) } if page.Status == docgen.StatusSkipped { logger.Warn("skipped", attrs...) continue } logger.Info("imported", attrs...) } } // pageResult is the outcome for one written page. type pageResult struct { Project string Slug string Status docgen.Status Notes []string Sub bool } // --------------------------------------------------------------------------- // Import pipeline // --------------------------------------------------------------------------- // importer holds the sync state shared by the one-shot and watch paths. type importer struct { opts options engine *storage.Engine logger *slog.Logger only map[string]struct{} skipSet map[string]struct{} // seen maps a project name to the signature observed on the previous poll, // which is what makes "did this change?" a cheap map lookup. seen map[string]string mu sync.Mutex } func newImporter(opts options, engine *storage.Engine, logger *slog.Logger) *importer { return &importer{ opts: opts, engine: engine, logger: logger, only: splitSet(opts.apps), skipSet: splitSet(opts.skip), seen: map[string]string{}, } } func splitSet(raw string) map[string]struct{} { out := map[string]struct{}{} for _, part := range strings.Split(raw, ",") { if trimmed := strings.TrimSpace(part); trimmed != "" { out[trimmed] = struct{}{} } } return out } // Discover returns the project names selected by -apps and -skip. func (im *importer) Discover() ([]string, error) { entries, err := os.ReadDir(im.opts.root) if err != nil { return nil, fmt.Errorf("cannot read workspace root %s: %w", im.opts.root, err) } var names []string for _, entry := range entries { name := entry.Name() if !entry.IsDir() || !strings.HasPrefix(name, "go-") { continue } if _, skip := im.skipSet[name]; skip { continue } if len(im.only) > 0 { if _, wanted := im.only[name]; !wanted { continue } } names = append(names, name) } sort.Strings(names) return names, nil } // Signatures returns the current source signature per project. func (im *importer) Signatures() (map[string]string, error) { names, err := im.Discover() if err != nil { return nil, err } out := make(map[string]string, len(names)) for _, name := range names { project, err := docgen.Analyze(name, filepath.Join(im.opts.root, name)) if err != nil { im.logger.Debug("cannot analyse project", "project", name, "error", err) continue } out[name] = project.Signature } return out, nil } // SyncOnce analyses and imports every discovered project. func (im *importer) SyncOnce() (Result, error) { names, err := im.Discover() if err != nil { return Result{}, err } if len(names) == 0 { im.logger.Warn("no go-* projects discovered", "root", im.opts.root) } return im.Import(names) } // Scan returns the projects whose signature changed since the previous poll, // including any project seen for the first time. Remembering every signature — // not just the changed ones — keeps a project from re-importing forever once it // has been synced. A project that has left the workspace is forgotten and // reported: its page stays on disk untouched, because a sync that silently // deletes documentation is worse than one that leaves it stale. func (im *importer) Scan() (map[string]string, error) { sigs, err := im.Signatures() if err != nil { return nil, err } im.mu.Lock() defer im.mu.Unlock() changed := map[string]string{} for name, sig := range sigs { if previous, seen := im.seen[name]; !seen || previous != sig { changed[name] = sig } } for name, sig := range sigs { im.seen[name] = sig } var departed []string for name := range im.seen { if _, present := sigs[name]; !present { departed = append(departed, name) } } sort.Strings(departed) for _, name := range departed { delete(im.seen, name) im.logger.Info("project left the workspace; its page was left as is", "project", name, "hint", "delete the page by hand if the project is gone for good", ) } return changed, nil } // Import analyses each named project and writes its page through the storage // engine, so the result is slugified, atomically written and immediately // visible to the API and the frontend. func (im *importer) Import(names []string) (Result, error) { var ( result Result mu sync.Mutex wg sync.WaitGroup ) for _, name := range names { wg.Add(1) go func(name string) { defer wg.Done() pages, subdocs, err := im.importOne(name) mu.Lock() defer mu.Unlock() if err != nil { result.Failed++ im.logger.Error("import failed", "project", name, "error", err) return } result.Subdocs += subdocs result.Pages = append(result.Pages, pages...) for _, page := range pages { if page.Sub { continue } switch page.Status { case docgen.StatusCreated: result.Created++ case docgen.StatusUpdated: result.Updated++ case docgen.StatusUnchanged: result.Unchanged++ case docgen.StatusSkipped: result.Skipped++ } } }(name) } wg.Wait() sort.Slice(result.Pages, func(i, j int) bool { return result.Pages[i].Slug < result.Pages[j].Slug }) return result, nil } // importOne builds and writes the page for a single project, plus any sub-pages // when -import-subdocs is set. func (im *importer) importOne(name string) ([]pageResult, int, error) { root := filepath.Join(im.opts.root, name) project, err := docgen.Analyze(name, root) if err != nil { return nil, 0, err } page, err := docgen.Render(project) if err != nil { return nil, 0, err } slug := storage.Slugify(name) status, err := im.write(slug, page) if err != nil { return nil, 0, err } out := []pageResult{{ Project: name, Slug: slug, Status: status, Notes: notes(project), }} if !im.opts.importSubdoc { return out, 0, nil } subpages := im.subdocPages(project) for _, sub := range subpages { subStatus, err := im.write(sub.slug, sub.body) if err != nil { im.logger.Warn("subdocument import failed", "project", name, "path", sub.source, "error", err) continue } out = append(out, pageResult{ Project: name, Slug: sub.slug, Status: subStatus, Sub: true, }) } return out, len(subpages), nil } type subdocPage struct { source string slug string body string } // subdocPages turns each Markdown file the project already owns into its own // go-doc page, wrapped so a later sync refreshes it instead of orphaning it. func (im *importer) subdocPages(project *docgen.Project) []subdocPage { base := storage.Slugify(project.Name) var out []subdocPage for _, doc := range project.Docs { if doc.Path == project.Readme { continue // already the summary of the main page } stem := strings.TrimSuffix(filepath.Base(doc.Path), filepath.Ext(doc.Path)) slug := storage.Slugify(project.Name + "-" + stem) if slug == "" || slug == base { continue } data, err := os.ReadFile(filepath.Join(project.Path, filepath.FromSlash(doc.Path))) if err != nil { continue } body := fmt.Sprintf( "Imported from %s in the %s repository.\n\n---\n\n%s", doc.Path, project.Name, strings.TrimRight(string(data), "\n"), ) out = append(out, subdocPage{ source: doc.Path, slug: slug, body: docgen.Wrap(project.Name+":"+doc.Path, project.Signature, body), }) } return out } // write merges a generated page with whatever is already on disk. Existing prose // outside the markers survives; a page with no markers is skipped unless force. func (im *importer) write(slug, page string) (docgen.Status, error) { var existing string if data, err := os.ReadFile(filepath.Join(im.engine.DocsDir(), slug+".md")); err == nil { existing = string(data) } merged, status, err := docgen.Merge(existing, page, im.opts.force) if err != nil { return "", err } if im.opts.dryRun { im.logger.Debug("dry run", "slug", slug, "status", string(status), "bytes", len(merged)) return status, nil } if _, err := im.engine.SaveDoc(slug, merged); err != nil { return "", err } return status, nil } // notes surfaces anything a reader should know about a generated page. func notes(project *docgen.Project) []string { var out []string if project.Readme == "" { out = append(out, "no README found") } if len(project.Routes) == 0 { out = append(out, "no HTTP routes detected") } if len(project.Modules) == 0 { out = append(out, "no go.mod found") } return out } func sortedKeys(m map[string]string) []string { keys := make([]string, 0, len(m)) for key := range m { keys = append(keys, key) } sort.Strings(keys) return keys }