// Package storage implements the flat-file persistence engine behind go-doc. // // Documents live on disk as raw Markdown files at data/docs/.md and // audience feedback is stored as JSON arrays at data/comments/-comments.json. // // Every mutation is written with a temp-file + fsync + rename sequence so a // concurrent reader can never observe a half-written file, and read-modify-write // cycles on the same document are serialised by a keyed mutex. That combination // is what lets the engine stay correct while the seed script floods it with // thousands of concurrent comments. package storage import ( "errors" "fmt" "io/fs" "os" "path/filepath" "regexp" "sort" "strings" "sync" "time" "unicode" "unicode/utf8" ) const ( // DocExt is the file extension used for every Markdown document. DocExt = ".md" // maxSlugLen caps generated slugs so a pathological title cannot produce an // unusable filename. maxSlugLen = 120 // maxTitleScanBytes limits how much of a document is scanned when extracting // its title from the leading heading. maxTitleScanBytes = 64 << 10 docsDirName = "docs" commentsDirName = "comments" ) // Errors returned by the engine. Callers map these onto HTTP status codes. var ( // ErrNotFound is returned when a document or comment does not exist. ErrNotFound = errors.New("storage: not found") // ErrInvalidSlug is returned for slugs that cannot be safely slugified. ErrInvalidSlug = errors.New("storage: invalid slug") // ErrInvalidRole is returned when a comment carries an unknown role. ErrInvalidRole = errors.New("storage: invalid role") // ErrInvalidComment is returned when a comment payload is unusable. ErrInvalidComment = errors.New("storage: invalid comment") ) // Engine owns a data directory and all flat-file operations against it. // Engine is safe for concurrent use by multiple goroutines. type Engine struct { root string docsDir string commentsDir string locksMu sync.Mutex locks map[string]*sync.Mutex // Now is injectable so tests can assert deterministic timestamps. Now func() time.Time } // New returns an Engine rooted at root and creates docs/ and comments/ beneath // it when they are missing. func New(root string) (*Engine, error) { if strings.TrimSpace(root) == "" { return nil, fmt.Errorf("storage: empty data root") } abs, err := filepath.Abs(root) if err != nil { return nil, fmt.Errorf("storage: resolve data root: %w", err) } e := &Engine{ root: abs, docsDir: filepath.Join(abs, docsDirName), commentsDir: filepath.Join(abs, commentsDirName), locks: make(map[string]*sync.Mutex), Now: time.Now, } if err := os.MkdirAll(e.docsDir, 0o755); err != nil { return nil, fmt.Errorf("storage: create docs dir: %w", err) } if err := os.MkdirAll(e.commentsDir, 0o755); err != nil { return nil, fmt.Errorf("storage: create comments dir: %w", err) } return e, nil } // Root returns the absolute data directory backing the engine. func (e *Engine) Root() string { return e.root } // DocsDir returns the absolute directory holding Markdown documents. func (e *Engine) DocsDir() string { return e.docsDir } // CommentsDir returns the absolute directory holding comment JSON files. func (e *Engine) CommentsDir() string { return e.commentsDir } func (e *Engine) now() time.Time { if e.Now != nil { return e.Now() } return time.Now() } // lockFor returns the mutex guarding a single slug. RLock-free by design: the // critical sections are short, and per-slug granularity is what allows writes to // different documents to proceed fully in parallel. func (e *Engine) lockFor(slug string) *sync.Mutex { e.locksMu.Lock() defer e.locksMu.Unlock() m, ok := e.locks[slug] if !ok { m = &sync.Mutex{} e.locks[slug] = m } return m } // ResolveRoot discovers the data directory without any configuration. It honours // GO_DOC_DATA_DIR first, then walks up from the working directory looking for an // existing data/ folder, and finally falls back to a relative "data". func ResolveRoot() string { if v := strings.TrimSpace(os.Getenv("GO_DOC_DATA_DIR")); v != "" { return v } dir, err := os.Getwd() if err == nil { for i := 0; i < 8; i++ { candidate := filepath.Join(dir, "data") if info, err := os.Stat(filepath.Join(candidate, docsDirName)); err == nil && info.IsDir() { return candidate } parent := filepath.Dir(dir) if parent == dir { break } dir = parent } } return "data" } var ( defaultOnce sync.Once defaultEng *Engine defaultErr error ) // Default returns the process-wide engine rooted at ResolveRoot(). func Default() (*Engine, error) { defaultOnce.Do(func() { defaultEng, defaultErr = New(ResolveRoot()) }) return defaultEng, defaultErr } // ResetDefault drops the cached engine so the next Default call re-resolves the // root. Tests use it between suites that point at different temp directories. func ResetDefault() { defaultOnce = sync.Once{} defaultEng = nil defaultErr = nil } // SetDefault installs root as the process-wide engine. func SetDefault(root string) (*Engine, error) { e, err := New(root) if err != nil { return nil, err } ResetDefault() defaultOnce.Do(func() {}) defaultEng, defaultErr = e, nil return e, nil } // latinFold maps lower-case Latin letters carrying diacritics, plus common // ligatures, onto their ASCII skeleton. Folding happens after ToLower, so the // keys are always lower case. var latinFold = map[rune]rune{ 'á': 'a', 'à': 'a', 'â': 'a', 'ä': 'a', 'ã': 'a', 'å': 'a', 'ā': 'a', 'ă': 'a', 'ą': 'a', 'ç': 'c', 'ć': 'c', 'č': 'c', 'ĉ': 'c', 'ċ': 'c', 'ď': 'd', 'đ': 'd', 'é': 'e', 'è': 'e', 'ê': 'e', 'ë': 'e', 'ē': 'e', 'ė': 'e', 'ę': 'e', 'ě': 'e', 'ğ': 'g', 'ĝ': 'g', 'ġ': 'g', 'ģ': 'g', 'í': 'i', 'ì': 'i', 'î': 'i', 'ï': 'i', 'ī': 'i', 'į': 'i', 'ı': 'i', 'ĵ': 'j', 'ķ': 'k', 'ł': 'l', 'ľ': 'l', 'ĺ': 'l', 'ñ': 'n', 'ń': 'n', 'ň': 'n', 'ó': 'o', 'ò': 'o', 'ô': 'o', 'ö': 'o', 'õ': 'o', 'ø': 'o', 'ō': 'o', 'ő': 'o', 'ŕ': 'r', 'ř': 'r', 'ś': 's', 'š': 's', 'ş': 's', 'ș': 's', 'ß': 's', 'þ': 't', 'ť': 't', 'ú': 'u', 'ù': 'u', 'û': 'u', 'ü': 'u', 'ū': 'u', 'ů': 'u', 'ű': 'u', 'ŵ': 'w', 'ý': 'y', 'ÿ': 'y', 'ŷ': 'y', 'ź': 'z', 'ż': 'z', 'ž': 'z', } // Slugify converts an arbitrary human title into a lower-case, hyphen-separated // slug: "Core Service API" becomes "core-service-api". Diacritics are folded to // ASCII, every other separator run collapses into a single hyphen, and the result // is trimmed and length-capped. func Slugify(title string) string { var b strings.Builder b.Grow(len(title)) pendingSep := true // suppresses a leading hyphen for _, r := range strings.ToLower(title) { if folded, ok := latinFold[r]; ok { r = folded } switch { case (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9'): b.WriteRune(r) pendingSep = false default: // Everything else -- punctuation, whitespace, non-ASCII letters that // carry no ASCII skeleton, and invalid UTF-8 -- collapses into a single // separator so slugs stay safe in URLs and filenames everywhere. if !pendingSep { b.WriteByte('-') pendingSep = true } } } slug := strings.Trim(b.String(), "-") if utf8.RuneCountInString(slug) > maxSlugLen { truncated := slug[:maxSlugLen] for len(truncated) > 0 && truncated[len(truncated)-1] == '-' { truncated = truncated[:len(truncated)-1] } slug = truncated } return slug } // NormalizeSlug validates and canonicalises a slug coming from a URL path. // Path separators, traversal segments and control characters are rejected // outright with ErrInvalidSlug; anything else is slugified so that // "Core Service API" and "core-service-api" address the same document. func NormalizeSlug(slug string) (string, error) { trimmed := strings.TrimSpace(slug) if trimmed == "" { return "", fmt.Errorf("%w: slug is empty", ErrInvalidSlug) } if len(trimmed) > 2*maxSlugLen { return "", fmt.Errorf("%w: slug exceeds %d bytes", ErrInvalidSlug, 2*maxSlugLen) } if strings.ContainsAny(trimmed, "/\\\x00") || strings.Contains(trimmed, "..") { return "", fmt.Errorf("%w: slug %q contains path separators", ErrInvalidSlug, slug) } for _, r := range trimmed { if unicode.IsControl(r) { return "", fmt.Errorf("%w: slug %q contains control characters", ErrInvalidSlug, slug) } } normalized := Slugify(trimmed) if normalized == "" { return "", fmt.Errorf("%w: slug %q has no usable characters", ErrInvalidSlug, slug) } return normalized, nil } var strictSlug = regexp.MustCompile(`^[a-z0-9]+(?:-[a-z0-9]+)*$`) // IsCanonicalSlug reports whether slug is already in the canonical form produced // by Slugify. Handlers use it to decide whether a rename touched the filesystem. func IsCanonicalSlug(slug string) bool { return strictSlug.MatchString(slug) } func (e *Engine) docPath(slug string) (string, error) { normalized, err := NormalizeSlug(slug) if err != nil { return "", err } return filepath.Join(e.docsDir, normalized+DocExt), nil } // SaveDoc writes content to data/docs/.md, creating the document when it // does not exist and replacing it wholesale otherwise. The write is atomic and // therefore safe to call concurrently with readers. func (e *Engine) SaveDoc(slug, content string) (string, error) { normalized, err := NormalizeSlug(slug) if err != nil { return "", err } path := filepath.Join(e.docsDir, normalized+DocExt) mu := e.lockFor(normalized) mu.Lock() defer mu.Unlock() normalizedContent := NormalizeMarkdown(content) if err := writeFileAtomic(path, []byte(normalizedContent), 0o644); err != nil { return "", err } return normalized, nil } // NormalizeMarkdown converts CRLF/CR line endings to LF, guarantees a single // trailing newline, and leaves the author's content otherwise untouched. func NormalizeMarkdown(content string) string { if content == "" { return "" } normalized := strings.ReplaceAll(content, "\r\n", "\n") normalized = strings.ReplaceAll(normalized, "\r", "\n") normalized = strings.TrimRight(normalized, "\n") + "\n" return normalized } // ReadDoc returns the raw Markdown stored at data/docs/.doc. // A missing document yields ErrNotFound. func (e *Engine) ReadDoc(slug string) (string, error) { path, err := e.docPath(slug) if err != nil { return "", err } data, err := os.ReadFile(path) if err != nil { if errors.Is(err, fs.ErrNotExist) { return "", fmt.Errorf("%w: doc %q", ErrNotFound, slug) } return "", fmt.Errorf("storage: read doc %q: %w", slug, err) } return string(data), nil } // DeleteDoc permanently removes data/docs/.md together with the document's // comment file. Removing a document that does not exist yields ErrNotFound. func (e *Engine) DeleteDoc(slug string) error { normalized, err := NormalizeSlug(slug) if err != nil { return err } path := filepath.Join(e.docsDir, normalized+DocExt) mu := e.lockFor(normalized) mu.Lock() defer mu.Unlock() if err := os.Remove(path); err != nil { if errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("%w: doc %q", ErrNotFound, slug) } return fmt.Errorf("storage: delete doc %q: %w", slug, err) } // Cascade the comment file so a recreated slug does not inherit stale history. commentPath := e.commentPath(normalized) if err := os.Remove(commentPath); err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("storage: delete comments for %q: %w", slug, err) } return nil } // DocMeta is the index entry describing one stored document. type DocMeta struct { Slug string `json:"slug"` Title string `json:"title"` Size int64 `json:"size"` Comments int `json:"comments"` Modified time.Time `json:"modified"` } var headingPattern = regexp.MustCompile(`(?m)^#{1,6}[ \t]+(.+?)[ \t]*#*[ \t]*$`) // ExtractTitle returns the first Markdown heading in content, falling back to a // title-cased rendering of the slug. func ExtractTitle(content, slug string) string { if len(content) > maxTitleScanBytes { content = content[:maxTitleScanBytes] } if m := headingPattern.FindStringSubmatch(content); m != nil { if title := strings.TrimSpace(m[1]); title != "" { return title } } return TitleizeSlug(slug) } // TitleizeSlug renders a slug as a human title: "core-service-api" becomes // "Core Service API". func TitleizeSlug(slug string) string { words := strings.FieldsFunc(slug, func(r rune) bool { return r == '-' || r == '_' }) for i, w := range words { if w == "" { continue } runes := []rune(strings.ToLower(w)) runes[0] = unicode.ToUpper(runes[0]) words[i] = string(runes) } if len(words) == 0 { return slug } return strings.Join(words, " ") } // DocExists reports whether a document is present on disk. func (e *Engine) DocExists(slug string) (bool, error) { path, err := e.docPath(slug) if err != nil { return false, err } info, err := os.Stat(path) if err != nil { if errors.Is(err, fs.ErrNotExist) { return false, nil } return false, fmt.Errorf("storage: stat doc %q: %w", slug, err) } return !info.IsDir(), nil } // Stat returns index metadata for a single document. func (e *Engine) Stat(slug string) (DocMeta, error) { normalized, err := NormalizeSlug(slug) if err != nil { return DocMeta{}, err } path := filepath.Join(e.docsDir, normalized+DocExt) info, err := os.Stat(path) if err != nil { if errors.Is(err, fs.ErrNotExist) { return DocMeta{}, fmt.Errorf("%w: doc %q", ErrNotFound, slug) } return DocMeta{}, fmt.Errorf("storage: stat doc %q: %w", slug, err) } meta := DocMeta{Slug: normalized, Size: info.Size(), Modified: info.ModTime().UTC()} if head, err := os.ReadFile(path); err == nil { meta.Title = ExtractTitle(string(head), normalized) } else { meta.Title = TitleizeSlug(normalized) } if comments, err := e.ListComments(normalized); err == nil { meta.Comments = len(comments) } return meta, nil } // ListDocs indexes every Markdown document on disk, newest modification first. // It reads only the head of each file to recover the document title, so listing // hundreds of documents stays cheap; use ListDocsDetailed when comment counts are // needed as well. func (e *Engine) ListDocs() ([]DocMeta, error) { return e.listDocs(false) } // ListDocsDetailed is ListDocs with per-document comment counts filled in. func (e *Engine) ListDocsDetailed() ([]DocMeta, error) { return e.listDocs(true) } func (e *Engine) listDocs(withComments bool) ([]DocMeta, error) { entries, err := os.ReadDir(e.docsDir) if err != nil { if errors.Is(err, fs.ErrNotExist) { return []DocMeta{}, nil } return nil, fmt.Errorf("storage: list docs: %w", err) } metas := make([]DocMeta, 0, len(entries)) for _, entry := range entries { if entry.IsDir() { continue } name := entry.Name() if !strings.HasSuffix(name, DocExt) || strings.HasPrefix(name, ".") { continue } slug := strings.TrimSuffix(name, DocExt) if !IsCanonicalSlug(slug) { continue } info, err := entry.Info() if err != nil { continue } meta := DocMeta{ Slug: slug, Title: e.readTitle(filepath.Join(e.docsDir, name), slug), Size: info.Size(), Modified: info.ModTime().UTC(), } if withComments { if comments, err := e.ListComments(slug); err == nil { meta.Comments = len(comments) } } metas = append(metas, meta) } sort.Slice(metas, func(i, j int) bool { if metas[i].Modified.Equal(metas[j].Modified) { return metas[i].Slug < metas[j].Slug } return metas[i].Modified.After(metas[j].Modified) }) return metas, nil } // Search returns documents whose slug or extracted title contains query, // case-insensitively. An empty query lists everything. func (e *Engine) Search(query string) ([]DocMeta, error) { metas, err := e.ListDocs() if err != nil { return nil, err } needle := strings.ToLower(strings.TrimSpace(query)) if needle == "" { return metas, nil } matches := make([]DocMeta, 0, len(metas)) for _, meta := range metas { if strings.Contains(strings.ToLower(meta.Slug), needle) || strings.Contains(strings.ToLower(meta.Title), needle) { matches = append(matches, meta) } } return matches, nil } // readTitle recovers a document title from the head of the file, reading at most // maxTitleScanBytes so a huge document cannot slow the index down. Any failure // degrades to a title derived from the slug. func (e *Engine) readTitle(path, slug string) string { file, err := os.Open(path) if err != nil { return TitleizeSlug(slug) } defer file.Close() head := make([]byte, maxTitleScanBytes) n, err := file.Read(head) if n <= 0 || (err != nil && n == 0) { return TitleizeSlug(slug) } return ExtractTitle(string(head[:n]), slug) } // writeFileAtomic writes data to path via a temporary file in the same directory, // fsyncs it, renames it into place, and fsyncs the parent directory so the // rename survives a crash. Readers therefore see either the old or the new file. func writeFileAtomic(path string, data []byte, perm os.FileMode) error { dir := filepath.Dir(path) tmp, err := os.CreateTemp(dir, ".godoc-tmp-*") if err != nil { return fmt.Errorf("storage: create temp file: %w", err) } tmpName := tmp.Name() cleanup := func() { tmp.Close() os.Remove(tmpName) } if _, err := tmp.Write(data); err != nil { cleanup() return fmt.Errorf("storage: write temp file: %w", err) } if err := tmp.Sync(); err != nil { cleanup() return fmt.Errorf("storage: sync temp file: %w", err) } if err := tmp.Close(); err != nil { os.Remove(tmpName) return fmt.Errorf("storage: close temp file: %w", err) } if err := os.Chmod(tmpName, perm); err != nil { os.Remove(tmpName) return fmt.Errorf("storage: chmod temp file: %w", err) } if err := os.Rename(tmpName, path); err != nil { os.Remove(tmpName) return fmt.Errorf("storage: rename into place: %w", err) } if err := syncDir(dir); err != nil { return fmt.Errorf("storage: sync dir: %w", err) } return nil } // syncDir flushes a directory entry so a completed rename is durable. func syncDir(dir string) error { f, err := os.Open(dir) if err != nil { return err } defer f.Close() return f.Sync() } // SaveDoc writes content to data/docs/.md through the process-wide engine. func SaveDoc(slug, content string) (string, error) { e, err := Default() if err != nil { return "", err } return e.SaveDoc(slug, content) } // ReadDoc reads data/docs/.md through the process-wide engine. func ReadDoc(slug string) (string, error) { e, err := Default() if err != nil { return "", err } return e.ReadDoc(slug) } // DeleteDoc removes data/docs/.md through the process-wide engine. func DeleteDoc(slug string) error { e, err := Default() if err != nil { return err } return e.DeleteDoc(slug) } // ListDocs indexes documents through the process-wide engine. func ListDocs() ([]DocMeta, error) { e, err := Default() if err != nil { return nil, err } return e.ListDocs() }