initial commit
This commit is contained in:
Daniel Noulet
commit
23816d00f7
71 changed files with 14497 additions and 0 deletions
498
backend/scripts/docgen/main.go
Normal file
498
backend/scripts/docgen/main.go
Normal file
|
|
@ -0,0 +1,498 @@
|
|||
// Command docgen analyses every go-* project under a workspace root and imports
|
||||
// audience-segmented documentation for each one into go-doc.
|
||||
//
|
||||
// One-shot sync:
|
||||
//
|
||||
// go run ./scripts/docgen -root /home/dnoulet/go -data ../data
|
||||
//
|
||||
// Continuous sync, re-importing whenever a project's docs or source change:
|
||||
//
|
||||
// go run ./scripts/docgen -root /home/dnoulet/go -data ../data -watch
|
||||
//
|
||||
// Imported pages are wrapped in generated markers. Re-running replaces only the
|
||||
// machine-written region, so prose written outside the markers survives, and a
|
||||
// page that was never generated is left untouched unless -force is passed.
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"go-doc/docgen"
|
||||
"go-doc/storage"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if err := run(); err != nil {
|
||||
slog.Error("fatal", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
type options struct {
|
||||
root string
|
||||
dataDir string
|
||||
apps string
|
||||
skip string
|
||||
watch bool
|
||||
interval time.Duration
|
||||
dryRun bool
|
||||
force bool
|
||||
importSubdoc bool
|
||||
verbose bool
|
||||
}
|
||||
|
||||
func run() error {
|
||||
var opts options
|
||||
|
||||
flag.StringVar(&opts.root, "root", defaultRoot(), "directory containing the go-* projects")
|
||||
flag.StringVar(&opts.dataDir, "data", storage.ResolveRoot(), "go-doc data directory to import into")
|
||||
flag.StringVar(&opts.apps, "apps", "", "comma-separated project names to limit the run (default: all)")
|
||||
flag.StringVar(&opts.skip, "skip", "go-doc", "comma-separated project names to ignore")
|
||||
flag.BoolVar(&opts.watch, "watch", false, "keep running and re-import when a project changes")
|
||||
flag.DurationVar(&opts.interval, "interval", 5*time.Second, "poll interval for -watch")
|
||||
flag.BoolVar(&opts.dryRun, "dry-run", false, "analyse and render without writing anything")
|
||||
flag.BoolVar(&opts.force, "force", false, "overwrite pages that carry no generated markers")
|
||||
flag.BoolVar(&opts.importSubdoc, "import-subdocs", false, "also import each project's own Markdown as separate go-doc pages")
|
||||
flag.BoolVar(&opts.verbose, "verbose", false, "log at debug level")
|
||||
flag.Parse()
|
||||
|
||||
level := slog.LevelInfo
|
||||
if opts.verbose {
|
||||
level = slog.LevelDebug
|
||||
}
|
||||
logger := slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: level}))
|
||||
|
||||
engine, err := storage.New(opts.dataDir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot open the go-doc data directory: %w", err)
|
||||
}
|
||||
|
||||
im := newImporter(opts, engine, logger)
|
||||
|
||||
if !opts.watch {
|
||||
result, err := im.SyncOnce()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
result.report(logger)
|
||||
if result.Failed > 0 {
|
||||
return fmt.Errorf("%d project(s) failed to import", result.Failed)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
logger.Info("watching for documentation changes",
|
||||
"root", opts.root,
|
||||
"interval", opts.interval.String(),
|
||||
"data", engine.Root(),
|
||||
"hint", "Ctrl-C to stop",
|
||||
)
|
||||
|
||||
ticker := time.NewTicker(opts.interval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
changed, err := im.Scan()
|
||||
switch {
|
||||
case err != nil:
|
||||
logger.Error("scan failed", "error", err)
|
||||
case len(changed) == 0:
|
||||
logger.Debug("no changes")
|
||||
default:
|
||||
logger.Info("change detected", "projects", strings.Join(sortedKeys(changed), ", "))
|
||||
result, syncErr := im.Import(sortedKeys(changed))
|
||||
if syncErr != nil {
|
||||
logger.Error("sync failed", "error", syncErr)
|
||||
}
|
||||
result.report(logger)
|
||||
}
|
||||
|
||||
<-ticker.C
|
||||
}
|
||||
}
|
||||
|
||||
func defaultRoot() string {
|
||||
if v := strings.TrimSpace(os.Getenv("GO_DOC_WORKSPACE")); v != "" {
|
||||
return v
|
||||
}
|
||||
return "/home/dnoulet/go"
|
||||
}
|
||||
|
||||
// Result summarises one sync pass.
|
||||
type Result struct {
|
||||
Created int
|
||||
Updated int
|
||||
Unchanged int
|
||||
Skipped int
|
||||
Failed int
|
||||
Subdocs int
|
||||
Pages []pageResult
|
||||
}
|
||||
|
||||
func (r Result) report(logger *slog.Logger) {
|
||||
logger.Info("sync complete",
|
||||
"created", r.Created,
|
||||
"updated", r.Updated,
|
||||
"unchanged", r.Unchanged,
|
||||
"skipped", r.Skipped,
|
||||
"subdocs", r.Subdocs,
|
||||
"failed", r.Failed,
|
||||
)
|
||||
for _, page := range r.Pages {
|
||||
attrs := []any{"project", page.Project, "slug", page.Slug, "status", string(page.Status)}
|
||||
if len(page.Notes) > 0 {
|
||||
attrs = append(attrs, "notes", strings.Join(page.Notes, ", "))
|
||||
}
|
||||
if page.Sub {
|
||||
attrs = append(attrs, "subdocument", true)
|
||||
}
|
||||
if page.Status == docgen.StatusSkipped {
|
||||
logger.Warn("skipped", attrs...)
|
||||
continue
|
||||
}
|
||||
logger.Info("imported", attrs...)
|
||||
}
|
||||
}
|
||||
|
||||
// pageResult is the outcome for one written page.
|
||||
type pageResult struct {
|
||||
Project string
|
||||
Slug string
|
||||
Status docgen.Status
|
||||
Notes []string
|
||||
Sub bool
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Import pipeline
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// importer holds the sync state shared by the one-shot and watch paths.
|
||||
type importer struct {
|
||||
opts options
|
||||
engine *storage.Engine
|
||||
logger *slog.Logger
|
||||
|
||||
only map[string]struct{}
|
||||
skipSet map[string]struct{}
|
||||
|
||||
// seen maps a project name to the signature observed on the previous poll,
|
||||
// which is what makes "did this change?" a cheap map lookup.
|
||||
seen map[string]string
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func newImporter(opts options, engine *storage.Engine, logger *slog.Logger) *importer {
|
||||
return &importer{
|
||||
opts: opts,
|
||||
engine: engine,
|
||||
logger: logger,
|
||||
only: splitSet(opts.apps),
|
||||
skipSet: splitSet(opts.skip),
|
||||
seen: map[string]string{},
|
||||
}
|
||||
}
|
||||
|
||||
func splitSet(raw string) map[string]struct{} {
|
||||
out := map[string]struct{}{}
|
||||
for _, part := range strings.Split(raw, ",") {
|
||||
if trimmed := strings.TrimSpace(part); trimmed != "" {
|
||||
out[trimmed] = struct{}{}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Discover returns the project names selected by -apps and -skip.
|
||||
func (im *importer) Discover() ([]string, error) {
|
||||
entries, err := os.ReadDir(im.opts.root)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot read workspace root %s: %w", im.opts.root, err)
|
||||
}
|
||||
|
||||
var names []string
|
||||
for _, entry := range entries {
|
||||
name := entry.Name()
|
||||
if !entry.IsDir() || !strings.HasPrefix(name, "go-") {
|
||||
continue
|
||||
}
|
||||
if _, skip := im.skipSet[name]; skip {
|
||||
continue
|
||||
}
|
||||
if len(im.only) > 0 {
|
||||
if _, wanted := im.only[name]; !wanted {
|
||||
continue
|
||||
}
|
||||
}
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
return names, nil
|
||||
}
|
||||
|
||||
// Signatures returns the current source signature per project.
|
||||
func (im *importer) Signatures() (map[string]string, error) {
|
||||
names, err := im.Discover()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make(map[string]string, len(names))
|
||||
for _, name := range names {
|
||||
project, err := docgen.Analyze(name, filepath.Join(im.opts.root, name))
|
||||
if err != nil {
|
||||
im.logger.Debug("cannot analyse project", "project", name, "error", err)
|
||||
continue
|
||||
}
|
||||
out[name] = project.Signature
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// SyncOnce analyses and imports every discovered project.
|
||||
func (im *importer) SyncOnce() (Result, error) {
|
||||
names, err := im.Discover()
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
if len(names) == 0 {
|
||||
im.logger.Warn("no go-* projects discovered", "root", im.opts.root)
|
||||
}
|
||||
return im.Import(names)
|
||||
}
|
||||
|
||||
// Scan returns the projects whose signature changed since the previous poll,
|
||||
// including any project seen for the first time. Remembering every signature —
|
||||
// not just the changed ones — keeps a project from re-importing forever once it
|
||||
// has been synced. A project that has left the workspace is forgotten and
|
||||
// reported: its page stays on disk untouched, because a sync that silently
|
||||
// deletes documentation is worse than one that leaves it stale.
|
||||
func (im *importer) Scan() (map[string]string, error) {
|
||||
sigs, err := im.Signatures()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
im.mu.Lock()
|
||||
defer im.mu.Unlock()
|
||||
|
||||
changed := map[string]string{}
|
||||
for name, sig := range sigs {
|
||||
if previous, seen := im.seen[name]; !seen || previous != sig {
|
||||
changed[name] = sig
|
||||
}
|
||||
}
|
||||
for name, sig := range sigs {
|
||||
im.seen[name] = sig
|
||||
}
|
||||
|
||||
var departed []string
|
||||
for name := range im.seen {
|
||||
if _, present := sigs[name]; !present {
|
||||
departed = append(departed, name)
|
||||
}
|
||||
}
|
||||
sort.Strings(departed)
|
||||
for _, name := range departed {
|
||||
delete(im.seen, name)
|
||||
im.logger.Info("project left the workspace; its page was left as is",
|
||||
"project", name,
|
||||
"hint", "delete the page by hand if the project is gone for good",
|
||||
)
|
||||
}
|
||||
|
||||
return changed, nil
|
||||
}
|
||||
|
||||
// Import analyses each named project and writes its page through the storage
|
||||
// engine, so the result is slugified, atomically written and immediately
|
||||
// visible to the API and the frontend.
|
||||
func (im *importer) Import(names []string) (Result, error) {
|
||||
var (
|
||||
result Result
|
||||
mu sync.Mutex
|
||||
wg sync.WaitGroup
|
||||
)
|
||||
|
||||
for _, name := range names {
|
||||
wg.Add(1)
|
||||
go func(name string) {
|
||||
defer wg.Done()
|
||||
|
||||
pages, subdocs, err := im.importOne(name)
|
||||
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if err != nil {
|
||||
result.Failed++
|
||||
im.logger.Error("import failed", "project", name, "error", err)
|
||||
return
|
||||
}
|
||||
result.Subdocs += subdocs
|
||||
result.Pages = append(result.Pages, pages...)
|
||||
for _, page := range pages {
|
||||
if page.Sub {
|
||||
continue
|
||||
}
|
||||
switch page.Status {
|
||||
case docgen.StatusCreated:
|
||||
result.Created++
|
||||
case docgen.StatusUpdated:
|
||||
result.Updated++
|
||||
case docgen.StatusUnchanged:
|
||||
result.Unchanged++
|
||||
case docgen.StatusSkipped:
|
||||
result.Skipped++
|
||||
}
|
||||
}
|
||||
}(name)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
sort.Slice(result.Pages, func(i, j int) bool { return result.Pages[i].Slug < result.Pages[j].Slug })
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// importOne builds and writes the page for a single project, plus any sub-pages
|
||||
// when -import-subdocs is set.
|
||||
func (im *importer) importOne(name string) ([]pageResult, int, error) {
|
||||
root := filepath.Join(im.opts.root, name)
|
||||
|
||||
project, err := docgen.Analyze(name, root)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
page, err := docgen.Render(project)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
slug := storage.Slugify(name)
|
||||
|
||||
status, err := im.write(slug, page)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
out := []pageResult{{
|
||||
Project: name,
|
||||
Slug: slug,
|
||||
Status: status,
|
||||
Notes: notes(project),
|
||||
}}
|
||||
|
||||
if !im.opts.importSubdoc {
|
||||
return out, 0, nil
|
||||
}
|
||||
|
||||
subpages := im.subdocPages(project)
|
||||
for _, sub := range subpages {
|
||||
subStatus, err := im.write(sub.slug, sub.body)
|
||||
if err != nil {
|
||||
im.logger.Warn("subdocument import failed",
|
||||
"project", name, "path", sub.source, "error", err)
|
||||
continue
|
||||
}
|
||||
out = append(out, pageResult{
|
||||
Project: name,
|
||||
Slug: sub.slug,
|
||||
Status: subStatus,
|
||||
Sub: true,
|
||||
})
|
||||
}
|
||||
return out, len(subpages), nil
|
||||
}
|
||||
|
||||
type subdocPage struct {
|
||||
source string
|
||||
slug string
|
||||
body string
|
||||
}
|
||||
|
||||
// subdocPages turns each Markdown file the project already owns into its own
|
||||
// go-doc page, wrapped so a later sync refreshes it instead of orphaning it.
|
||||
func (im *importer) subdocPages(project *docgen.Project) []subdocPage {
|
||||
base := storage.Slugify(project.Name)
|
||||
var out []subdocPage
|
||||
|
||||
for _, doc := range project.Docs {
|
||||
if doc.Path == project.Readme {
|
||||
continue // already the summary of the main page
|
||||
}
|
||||
stem := strings.TrimSuffix(filepath.Base(doc.Path), filepath.Ext(doc.Path))
|
||||
slug := storage.Slugify(project.Name + "-" + stem)
|
||||
if slug == "" || slug == base {
|
||||
continue
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(filepath.Join(project.Path, filepath.FromSlash(doc.Path)))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
body := fmt.Sprintf(
|
||||
"Imported from %s in the %s repository.\n\n---\n\n%s",
|
||||
doc.Path, project.Name, strings.TrimRight(string(data), "\n"),
|
||||
)
|
||||
out = append(out, subdocPage{
|
||||
source: doc.Path,
|
||||
slug: slug,
|
||||
body: docgen.Wrap(project.Name+":"+doc.Path, project.Signature, body),
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// write merges a generated page with whatever is already on disk. Existing prose
|
||||
// outside the markers survives; a page with no markers is skipped unless force.
|
||||
func (im *importer) write(slug, page string) (docgen.Status, error) {
|
||||
var existing string
|
||||
if data, err := os.ReadFile(filepath.Join(im.engine.DocsDir(), slug+".md")); err == nil {
|
||||
existing = string(data)
|
||||
}
|
||||
|
||||
merged, status, err := docgen.Merge(existing, page, im.opts.force)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if im.opts.dryRun {
|
||||
im.logger.Debug("dry run", "slug", slug, "status", string(status), "bytes", len(merged))
|
||||
return status, nil
|
||||
}
|
||||
if _, err := im.engine.SaveDoc(slug, merged); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return status, nil
|
||||
}
|
||||
|
||||
// notes surfaces anything a reader should know about a generated page.
|
||||
func notes(project *docgen.Project) []string {
|
||||
var out []string
|
||||
if project.Readme == "" {
|
||||
out = append(out, "no README found")
|
||||
}
|
||||
if len(project.Routes) == 0 {
|
||||
out = append(out, "no HTTP routes detected")
|
||||
}
|
||||
if len(project.Modules) == 0 {
|
||||
out = append(out, "no go.mod found")
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func sortedKeys(m map[string]string) []string {
|
||||
keys := make([]string, 0, len(m))
|
||||
for key := range m {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
return keys
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue