go-doc/backend/storage/engine.go

// Package storage implements the flat-file persistence engine behind go-doc.
//
// Documents live on disk as raw Markdown files at data/docs/<slug>.md and
// audience feedback is stored as JSON arrays at data/comments/<slug>-comments.json.
//
// Every mutation is written with a temp-file + fsync + rename sequence so a
// concurrent reader can never observe a half-written file, and read-modify-write
// cycles on the same document are serialised by a keyed mutex. That combination
// is what lets the engine stay correct while the seed script floods it with
// thousands of concurrent comments.
package storage

import (
	"errors"
	"fmt"
	"io/fs"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
	"sync"
	"time"
	"unicode"
	"unicode/utf8"
)

const (
	// DocExt is the file extension used for every Markdown document.
	DocExt = ".md"

	// maxSlugLen caps generated slugs so a pathological title cannot produce an
	// unusable filename.
	maxSlugLen = 120

	// maxTitleScanBytes limits how much of a document is scanned when extracting
	// its title from the leading heading.
	maxTitleScanBytes = 64 << 10

	docsDirName     = "docs"
	commentsDirName = "comments"
)

// Errors returned by the engine. Callers map these onto HTTP status codes.
var (
	// ErrNotFound is returned when a document or comment does not exist.
	ErrNotFound = errors.New("storage: not found")
	// ErrInvalidSlug is returned for slugs that cannot be safely slugified.
	ErrInvalidSlug = errors.New("storage: invalid slug")
	// ErrInvalidRole is returned when a comment carries an unknown role.
	ErrInvalidRole = errors.New("storage: invalid role")
	// ErrInvalidComment is returned when a comment payload is unusable.
	ErrInvalidComment = errors.New("storage: invalid comment")
)

// Engine owns a data directory and all flat-file operations against it.
// Engine is safe for concurrent use by multiple goroutines.
type Engine struct {
	root        string
	docsDir     string
	commentsDir string

	locksMu sync.Mutex
	locks   map[string]*sync.Mutex

	// Now is injectable so tests can assert deterministic timestamps.
	Now func() time.Time
}

// New returns an Engine rooted at root and creates docs/ and comments/ beneath
// it when they are missing.
func New(root string) (*Engine, error) {
	if strings.TrimSpace(root) == "" {
		return nil, fmt.Errorf("storage: empty data root")
	}
	abs, err := filepath.Abs(root)
	if err != nil {
		return nil, fmt.Errorf("storage: resolve data root: %w", err)
	}
	e := &Engine{
		root:        abs,
		docsDir:     filepath.Join(abs, docsDirName),
		commentsDir: filepath.Join(abs, commentsDirName),
		locks:       make(map[string]*sync.Mutex),
		Now:         time.Now,
	}
	if err := os.MkdirAll(e.docsDir, 0o755); err != nil {
		return nil, fmt.Errorf("storage: create docs dir: %w", err)
	}
	if err := os.MkdirAll(e.commentsDir, 0o755); err != nil {
		return nil, fmt.Errorf("storage: create comments dir: %w", err)
	}
	return e, nil
}

// Root returns the absolute data directory backing the engine.
func (e *Engine) Root() string { return e.root }

// DocsDir returns the absolute directory holding Markdown documents.
func (e *Engine) DocsDir() string { return e.docsDir }

// CommentsDir returns the absolute directory holding comment JSON files.
func (e *Engine) CommentsDir() string { return e.commentsDir }

func (e *Engine) now() time.Time {
	if e.Now != nil {
		return e.Now()
	}
	return time.Now()
}

// lockFor returns the mutex guarding a single slug. RLock-free by design: the
// critical sections are short, and per-slug granularity is what allows writes to
// different documents to proceed fully in parallel.
func (e *Engine) lockFor(slug string) *sync.Mutex {
	e.locksMu.Lock()
	defer e.locksMu.Unlock()
	m, ok := e.locks[slug]
	if !ok {
		m = &sync.Mutex{}
		e.locks[slug] = m
	}
	return m
}

// ResolveRoot discovers the data directory without any configuration. It honours
// GO_DOC_DATA_DIR first, then walks up from the working directory looking for an
// existing data/ folder, and finally falls back to a relative "data".
func ResolveRoot() string {
	if v := strings.TrimSpace(os.Getenv("GO_DOC_DATA_DIR")); v != "" {
		return v
	}
	dir, err := os.Getwd()
	if err == nil {
		for i := 0; i < 8; i++ {
			candidate := filepath.Join(dir, "data")
			if info, err := os.Stat(filepath.Join(candidate, docsDirName)); err == nil && info.IsDir() {
				return candidate
			}
			parent := filepath.Dir(dir)
			if parent == dir {
				break
			}
			dir = parent
		}
	}
	return "data"
}

var (
	defaultOnce sync.Once
	defaultEng  *Engine
	defaultErr  error
)

// Default returns the process-wide engine rooted at ResolveRoot().
func Default() (*Engine, error) {
	defaultOnce.Do(func() {
		defaultEng, defaultErr = New(ResolveRoot())
	})
	return defaultEng, defaultErr
}

// ResetDefault drops the cached engine so the next Default call re-resolves the
// root. Tests use it between suites that point at different temp directories.
func ResetDefault() {
	defaultOnce = sync.Once{}
	defaultEng = nil
	defaultErr = nil
}

// SetDefault installs root as the process-wide engine.
func SetDefault(root string) (*Engine, error) {
	e, err := New(root)
	if err != nil {
		return nil, err
	}
	ResetDefault()
	defaultOnce.Do(func() {})
	defaultEng, defaultErr = e, nil
	return e, nil
}

// latinFold maps lower-case Latin letters carrying diacritics, plus common
// ligatures, onto their ASCII skeleton. Folding happens after ToLower, so the
// keys are always lower case.
var latinFold = map[rune]rune{
	'á': 'a', 'à': 'a', 'â': 'a', 'ä': 'a', 'ã': 'a', 'å': 'a', 'ā': 'a', 'ă': 'a', 'ą': 'a',
	'ç': 'c', 'ć': 'c', 'č': 'c', 'ĉ': 'c', 'ċ': 'c',
	'ď': 'd', 'đ': 'd',
	'é': 'e', 'è': 'e', 'ê': 'e', 'ë': 'e', 'ē': 'e', 'ė': 'e', 'ę': 'e', 'ě': 'e',
	'ğ': 'g', 'ĝ': 'g', 'ġ': 'g', 'ģ': 'g',
	'í': 'i', 'ì': 'i', 'î': 'i', 'ï': 'i', 'ī': 'i', 'į': 'i', 'ı': 'i',
	'ĵ': 'j',
	'ķ': 'k',
	'ł': 'l', 'ľ': 'l', 'ĺ': 'l',
	'ñ': 'n', 'ń': 'n', 'ň': 'n',
	'ó': 'o', 'ò': 'o', 'ô': 'o', 'ö': 'o', 'õ': 'o', 'ø': 'o', 'ō': 'o', 'ő': 'o',
	'ŕ': 'r', 'ř': 'r',
	'ś': 's', 'š': 's', 'ş': 's', 'ș': 's',
	'ß': 's', 'þ': 't', 'ť': 't',
	'ú': 'u', 'ù': 'u', 'û': 'u', 'ü': 'u', 'ū': 'u', 'ů': 'u', 'ű': 'u',
	'ŵ': 'w', 'ý': 'y', 'ÿ': 'y', 'ŷ': 'y',
	'ź': 'z', 'ż': 'z', 'ž': 'z',
}

// Slugify converts an arbitrary human title into a lower-case, hyphen-separated
// slug: "Core Service API" becomes "core-service-api". Diacritics are folded to
// ASCII, every other separator run collapses into a single hyphen, and the result
// is trimmed and length-capped.
func Slugify(title string) string {
	var b strings.Builder
	b.Grow(len(title))

	pendingSep := true // suppresses a leading hyphen
	for _, r := range strings.ToLower(title) {
		if folded, ok := latinFold[r]; ok {
			r = folded
		}
		switch {
		case (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9'):
			b.WriteRune(r)
			pendingSep = false
		default:
			// Everything else -- punctuation, whitespace, non-ASCII letters that
			// carry no ASCII skeleton, and invalid UTF-8 -- collapses into a single
			// separator so slugs stay safe in URLs and filenames everywhere.
			if !pendingSep {
				b.WriteByte('-')
				pendingSep = true
			}
		}
	}

	slug := strings.Trim(b.String(), "-")
	if utf8.RuneCountInString(slug) > maxSlugLen {
		truncated := slug[:maxSlugLen]
		for len(truncated) > 0 && truncated[len(truncated)-1] == '-' {
			truncated = truncated[:len(truncated)-1]
		}
		slug = truncated
	}
	return slug
}

// NormalizeSlug validates and canonicalises a slug coming from a URL path.
// Path separators, traversal segments and control characters are rejected
// outright with ErrInvalidSlug; anything else is slugified so that
// "Core Service API" and "core-service-api" address the same document.
func NormalizeSlug(slug string) (string, error) {
	trimmed := strings.TrimSpace(slug)
	if trimmed == "" {
		return "", fmt.Errorf("%w: slug is empty", ErrInvalidSlug)
	}
	if len(trimmed) > 2*maxSlugLen {
		return "", fmt.Errorf("%w: slug exceeds %d bytes", ErrInvalidSlug, 2*maxSlugLen)
	}
	if strings.ContainsAny(trimmed, "/\\\x00") || strings.Contains(trimmed, "..") {
		return "", fmt.Errorf("%w: slug %q contains path separators", ErrInvalidSlug, slug)
	}
	for _, r := range trimmed {
		if unicode.IsControl(r) {
			return "", fmt.Errorf("%w: slug %q contains control characters", ErrInvalidSlug, slug)
		}
	}

	normalized := Slugify(trimmed)
	if normalized == "" {
		return "", fmt.Errorf("%w: slug %q has no usable characters", ErrInvalidSlug, slug)
	}
	return normalized, nil
}

var strictSlug = regexp.MustCompile(`^[a-z0-9]+(?:-[a-z0-9]+)*$`)

// IsCanonicalSlug reports whether slug is already in the canonical form produced
// by Slugify. Handlers use it to decide whether a rename touched the filesystem.
func IsCanonicalSlug(slug string) bool { return strictSlug.MatchString(slug) }

func (e *Engine) docPath(slug string) (string, error) {
	normalized, err := NormalizeSlug(slug)
	if err != nil {
		return "", err
	}
	return filepath.Join(e.docsDir, normalized+DocExt), nil
}

// SaveDoc writes content to data/docs/<slug>.md, creating the document when it
// does not exist and replacing it wholesale otherwise. The write is atomic and
// therefore safe to call concurrently with readers.
func (e *Engine) SaveDoc(slug, content string) (string, error) {
	normalized, err := NormalizeSlug(slug)
	if err != nil {
		return "", err
	}
	path := filepath.Join(e.docsDir, normalized+DocExt)

	mu := e.lockFor(normalized)
	mu.Lock()
	defer mu.Unlock()

	normalizedContent := NormalizeMarkdown(content)
	if err := writeFileAtomic(path, []byte(normalizedContent), 0o644); err != nil {
		return "", err
	}
	return normalized, nil
}

// NormalizeMarkdown converts CRLF/CR line endings to LF, guarantees a single
// trailing newline, and leaves the author's content otherwise untouched.
func NormalizeMarkdown(content string) string {
	if content == "" {
		return ""
	}
	normalized := strings.ReplaceAll(content, "\r\n", "\n")
	normalized = strings.ReplaceAll(normalized, "\r", "\n")
	normalized = strings.TrimRight(normalized, "\n") + "\n"
	return normalized
}

// ReadDoc returns the raw Markdown stored at data/docs/<slug>.doc.
// A missing document yields ErrNotFound.
func (e *Engine) ReadDoc(slug string) (string, error) {
	path, err := e.docPath(slug)
	if err != nil {
		return "", err
	}
	data, err := os.ReadFile(path)
	if err != nil {
		if errors.Is(err, fs.ErrNotExist) {
			return "", fmt.Errorf("%w: doc %q", ErrNotFound, slug)
		}
		return "", fmt.Errorf("storage: read doc %q: %w", slug, err)
	}
	return string(data), nil
}

// DeleteDoc permanently removes data/docs/<slug>.md together with the document's
// comment file. Removing a document that does not exist yields ErrNotFound.
func (e *Engine) DeleteDoc(slug string) error {
	normalized, err := NormalizeSlug(slug)
	if err != nil {
		return err
	}
	path := filepath.Join(e.docsDir, normalized+DocExt)

	mu := e.lockFor(normalized)
	mu.Lock()
	defer mu.Unlock()

	if err := os.Remove(path); err != nil {
		if errors.Is(err, fs.ErrNotExist) {
			return fmt.Errorf("%w: doc %q", ErrNotFound, slug)
		}
		return fmt.Errorf("storage: delete doc %q: %w", slug, err)
	}

	// Cascade the comment file so a recreated slug does not inherit stale history.
	commentPath := e.commentPath(normalized)
	if err := os.Remove(commentPath); err != nil && !errors.Is(err, fs.ErrNotExist) {
		return fmt.Errorf("storage: delete comments for %q: %w", slug, err)
	}
	return nil
}

// DocMeta is the index entry describing one stored document.
type DocMeta struct {
	Slug     string    `json:"slug"`
	Title    string    `json:"title"`
	Size     int64     `json:"size"`
	Comments int       `json:"comments"`
	Modified time.Time `json:"modified"`
}

var headingPattern = regexp.MustCompile(`(?m)^#{1,6}[ \t]+(.+?)[ \t]*#*[ \t]*$`)

// ExtractTitle returns the first Markdown heading in content, falling back to a
// title-cased rendering of the slug.
func ExtractTitle(content, slug string) string {
	if len(content) > maxTitleScanBytes {
		content = content[:maxTitleScanBytes]
	}
	if m := headingPattern.FindStringSubmatch(content); m != nil {
		if title := strings.TrimSpace(m[1]); title != "" {
			return title
		}
	}
	return TitleizeSlug(slug)
}

// TitleizeSlug renders a slug as a human title: "core-service-api" becomes
// "Core Service API".
func TitleizeSlug(slug string) string {
	words := strings.FieldsFunc(slug, func(r rune) bool { return r == '-' || r == '_' })
	for i, w := range words {
		if w == "" {
			continue
		}
		runes := []rune(strings.ToLower(w))
		runes[0] = unicode.ToUpper(runes[0])
		words[i] = string(runes)
	}
	if len(words) == 0 {
		return slug
	}
	return strings.Join(words, " ")
}

// DocExists reports whether a document is present on disk.
func (e *Engine) DocExists(slug string) (bool, error) {
	path, err := e.docPath(slug)
	if err != nil {
		return false, err
	}
	info, err := os.Stat(path)
	if err != nil {
		if errors.Is(err, fs.ErrNotExist) {
			return false, nil
		}
		return false, fmt.Errorf("storage: stat doc %q: %w", slug, err)
	}
	return !info.IsDir(), nil
}

// Stat returns index metadata for a single document.
func (e *Engine) Stat(slug string) (DocMeta, error) {
	normalized, err := NormalizeSlug(slug)
	if err != nil {
		return DocMeta{}, err
	}
	path := filepath.Join(e.docsDir, normalized+DocExt)
	info, err := os.Stat(path)
	if err != nil {
		if errors.Is(err, fs.ErrNotExist) {
			return DocMeta{}, fmt.Errorf("%w: doc %q", ErrNotFound, slug)
		}
		return DocMeta{}, fmt.Errorf("storage: stat doc %q: %w", slug, err)
	}

	meta := DocMeta{Slug: normalized, Size: info.Size(), Modified: info.ModTime().UTC()}
	if head, err := os.ReadFile(path); err == nil {
		meta.Title = ExtractTitle(string(head), normalized)
	} else {
		meta.Title = TitleizeSlug(normalized)
	}
	if comments, err := e.ListComments(normalized); err == nil {
		meta.Comments = len(comments)
	}
	return meta, nil
}

// ListDocs indexes every Markdown document on disk, newest modification first.
// It reads only the head of each file to recover the document title, so listing
// hundreds of documents stays cheap; use ListDocsDetailed when comment counts are
// needed as well.
func (e *Engine) ListDocs() ([]DocMeta, error) { return e.listDocs(false) }

// ListDocsDetailed is ListDocs with per-document comment counts filled in.
func (e *Engine) ListDocsDetailed() ([]DocMeta, error) { return e.listDocs(true) }

func (e *Engine) listDocs(withComments bool) ([]DocMeta, error) {
	entries, err := os.ReadDir(e.docsDir)
	if err != nil {
		if errors.Is(err, fs.ErrNotExist) {
			return []DocMeta{}, nil
		}
		return nil, fmt.Errorf("storage: list docs: %w", err)
	}

	metas := make([]DocMeta, 0, len(entries))
	for _, entry := range entries {
		if entry.IsDir() {
			continue
		}
		name := entry.Name()
		if !strings.HasSuffix(name, DocExt) || strings.HasPrefix(name, ".") {
			continue
		}
		slug := strings.TrimSuffix(name, DocExt)
		if !IsCanonicalSlug(slug) {
			continue
		}
		info, err := entry.Info()
		if err != nil {
			continue
		}
		meta := DocMeta{
			Slug:     slug,
			Title:    e.readTitle(filepath.Join(e.docsDir, name), slug),
			Size:     info.Size(),
			Modified: info.ModTime().UTC(),
		}
		if withComments {
			if comments, err := e.ListComments(slug); err == nil {
				meta.Comments = len(comments)
			}
		}
		metas = append(metas, meta)
	}

	sort.Slice(metas, func(i, j int) bool {
		if metas[i].Modified.Equal(metas[j].Modified) {
			return metas[i].Slug < metas[j].Slug
		}
		return metas[i].Modified.After(metas[j].Modified)
	})
	return metas, nil
}

// Search returns documents whose slug or extracted title contains query,
// case-insensitively. An empty query lists everything.
func (e *Engine) Search(query string) ([]DocMeta, error) {
	metas, err := e.ListDocs()
	if err != nil {
		return nil, err
	}
	needle := strings.ToLower(strings.TrimSpace(query))
	if needle == "" {
		return metas, nil
	}
	matches := make([]DocMeta, 0, len(metas))
	for _, meta := range metas {
		if strings.Contains(strings.ToLower(meta.Slug), needle) ||
			strings.Contains(strings.ToLower(meta.Title), needle) {
			matches = append(matches, meta)
		}
	}
	return matches, nil
}

// readTitle recovers a document title from the head of the file, reading at most
// maxTitleScanBytes so a huge document cannot slow the index down. Any failure
// degrades to a title derived from the slug.
func (e *Engine) readTitle(path, slug string) string {
	file, err := os.Open(path)
	if err != nil {
		return TitleizeSlug(slug)
	}
	defer file.Close()

	head := make([]byte, maxTitleScanBytes)
	n, err := file.Read(head)
	if n <= 0 || (err != nil && n == 0) {
		return TitleizeSlug(slug)
	}
	return ExtractTitle(string(head[:n]), slug)
}

// writeFileAtomic writes data to path via a temporary file in the same directory,
// fsyncs it, renames it into place, and fsyncs the parent directory so the
// rename survives a crash. Readers therefore see either the old or the new file.
func writeFileAtomic(path string, data []byte, perm os.FileMode) error {
	dir := filepath.Dir(path)
	tmp, err := os.CreateTemp(dir, ".godoc-tmp-*")
	if err != nil {
		return fmt.Errorf("storage: create temp file: %w", err)
	}
	tmpName := tmp.Name()

	cleanup := func() {
		tmp.Close()
		os.Remove(tmpName)
	}

	if _, err := tmp.Write(data); err != nil {
		cleanup()
		return fmt.Errorf("storage: write temp file: %w", err)
	}
	if err := tmp.Sync(); err != nil {
		cleanup()
		return fmt.Errorf("storage: sync temp file: %w", err)
	}
	if err := tmp.Close(); err != nil {
		os.Remove(tmpName)
		return fmt.Errorf("storage: close temp file: %w", err)
	}
	if err := os.Chmod(tmpName, perm); err != nil {
		os.Remove(tmpName)
		return fmt.Errorf("storage: chmod temp file: %w", err)
	}
	if err := os.Rename(tmpName, path); err != nil {
		os.Remove(tmpName)
		return fmt.Errorf("storage: rename into place: %w", err)
	}
	if err := syncDir(dir); err != nil {
		return fmt.Errorf("storage: sync dir: %w", err)
	}
	return nil
}

// syncDir flushes a directory entry so a completed rename is durable.
func syncDir(dir string) error {
	f, err := os.Open(dir)
	if err != nil {
		return err
	}
	defer f.Close()
	return f.Sync()
}

// SaveDoc writes content to data/docs/<slug>.md through the process-wide engine.
func SaveDoc(slug, content string) (string, error) {
	e, err := Default()
	if err != nil {
		return "", err
	}
	return e.SaveDoc(slug, content)
}

// ReadDoc reads data/docs/<slug>.md through the process-wide engine.
func ReadDoc(slug string) (string, error) {
	e, err := Default()
	if err != nil {
		return "", err
	}
	return e.ReadDoc(slug)
}

// DeleteDoc removes data/docs/<slug>.md through the process-wide engine.
func DeleteDoc(slug string) error {
	e, err := Default()
	if err != nil {
		return err
	}
	return e.DeleteDoc(slug)
}

// ListDocs indexes documents through the process-wide engine.
func ListDocs() ([]DocMeta, error) {
	e, err := Default()
	if err != nil {
		return nil, err
	}
	return e.ListDocs()
}