418 lines
13 KiB
Go
418 lines
13 KiB
Go
|
package bdd
|
||
|
|
|
||
|
|
import (
|
||
|
|
"encoding/json"
|
||
|
|
"fmt"
|
||
|
|
"strings"
|
||
|
|
"sync"
|
||
|
|
"testing"
|
||
|
|
"time"
|
||
|
|
|
||
|
|
"go-doc/storage"
|
||
|
|
)
|
||
|
|
|
||
|
|
// TestFeatureFileUpdatesUnderLoad covers:
|
||
|
|
//
|
||
|
|
// Given many documents and thousands of comments in flight
|
||
|
|
// When they are written concurrently through the API
|
||
|
|
// Then every update lands on disk with no torn or lost writes
|
||
|
|
func TestFeatureFileUpdatesUnderLoad(t *testing.T) {
|
||
|
|
Run(t, Feature{
|
||
|
|
Name: "File updates under load",
|
||
|
|
Scenarios: []Scenario{
|
||
|
|
{
|
||
|
|
Name: "concurrent comment injection loses no entry",
|
||
|
|
Given: Steps{
|
||
|
|
{Text: "a documentation page with 400 concurrent feedback writers", Do: func(c *Context) error {
|
||
|
|
_, err := c.Engine.SaveDoc("hot-service", "# Hot Service\n\n## For Operators\n\nScaling notes.\n")
|
||
|
|
return err
|
||
|
|
}},
|
||
|
|
},
|
||
|
|
When: Steps{
|
||
|
|
{Text: "400 comments are appended concurrently across all three roles", Do: func(c *Context) error {
|
||
|
|
const writers = 400
|
||
|
|
roles := []storage.Role{storage.RoleUser, storage.RoleOperator, storage.RoleDeveloper}
|
||
|
|
|
||
|
|
var wg sync.WaitGroup
|
||
|
|
errs := make(chan error, writers)
|
||
|
|
start := make(chan struct{})
|
||
|
|
|
||
|
|
for i := 0; i < writers; i++ {
|
||
|
|
wg.Add(1)
|
||
|
|
go func(index int) {
|
||
|
|
defer wg.Done()
|
||
|
|
<-start // release every writer at once
|
||
|
|
role := roles[index%len(roles)]
|
||
|
|
if _, err := appendComment(c, "hot-service", role,
|
||
|
|
fmt.Sprintf("user-%03d", index),
|
||
|
|
fmt.Sprintf("finding number %d", index),
|
||
|
|
); err != nil {
|
||
|
|
errs <- err
|
||
|
|
}
|
||
|
|
}(i)
|
||
|
|
}
|
||
|
|
close(start)
|
||
|
|
wg.Wait()
|
||
|
|
close(errs)
|
||
|
|
|
||
|
|
for err := range errs {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
return nil
|
||
|
|
}},
|
||
|
|
},
|
||
|
|
Then: Steps{
|
||
|
|
{Text: "all 400 comments are present in the JSON array", Do: func(c *Context) error {
|
||
|
|
comments, err := c.RawComments("hot-service")
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
if len(comments) != 400 {
|
||
|
|
return errf("expected 400 persisted comments, got %d", len(comments))
|
||
|
|
}
|
||
|
|
return nil
|
||
|
|
}},
|
||
|
|
{Text: "no comment was lost to a read-modify-write race", Do: func(c *Context) error {
|
||
|
|
comments, err := c.RawComments("hot-service")
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
seen := make(map[string]struct{}, len(comments))
|
||
|
|
for _, comment := range comments {
|
||
|
|
key := fmt.Sprintf("%s|%s", comment.Author, comment.Content)
|
||
|
|
if _, duplicate := seen[key]; duplicate {
|
||
|
|
return errf("duplicate entry survived: %s", key)
|
||
|
|
}
|
||
|
|
seen[key] = struct{}{}
|
||
|
|
}
|
||
|
|
if len(seen) != 400 {
|
||
|
|
return errf("expected 400 distinct entries, got %d", len(seen))
|
||
|
|
}
|
||
|
|
return nil
|
||
|
|
}},
|
||
|
|
{Text: "every comment carries the badge for its own role", Do: func(c *Context) error {
|
||
|
|
comments, err := c.RawComments("hot-service")
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
for _, comment := range comments {
|
||
|
|
if comment.Badge != badgeContract[comment.Role] {
|
||
|
|
return errf("comment %s has badge %q, want %q",
|
||
|
|
comment.ID, comment.Badge, badgeContract[comment.Role])
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return nil
|
||
|
|
}},
|
||
|
|
{Text: "the file on disk is a single valid JSON array, never a torn write", Do: func(c *Context) error {
|
||
|
|
raw, err := c.RawJSON("hot-service")
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
if !strings.HasPrefix(strings.TrimSpace(string(raw)), "[") ||
|
||
|
|
!strings.HasSuffix(strings.TrimSpace(string(raw)), "]") {
|
||
|
|
return errf("feedback file is not a well-formed JSON array")
|
||
|
|
}
|
||
|
|
var decoded []storage.Comment
|
||
|
|
if err := json.Unmarshal(raw, &decoded); err != nil {
|
||
|
|
return errf("feedback file failed to decode: %w", err)
|
||
|
|
}
|
||
|
|
return ExpectEqual(len(decoded), 400, "decoded comment count")
|
||
|
|
}},
|
||
|
|
{Text: "the HTTP read path agrees with the file on disk", Do: func(c *Context) error {
|
||
|
|
response, payload, err := c.Do("GET", "/api/docs/hot-service/comments", "", "")
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
if err := ExpectStatus(response, 200); err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
listed, err := Decode[struct {
|
||
|
|
Count int `json:"count"`
|
||
|
|
}](payload)
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
return ExpectEqual(listed.Count, 400, "API comment count")
|
||
|
|
}},
|
||
|
|
},
|
||
|
|
},
|
||
|
|
{
|
||
|
|
Name: "readers never observe a partially written document while writers stream updates",
|
||
|
|
Given: Steps{
|
||
|
|
{Text: "a document with a long body", Do: func(c *Context) error {
|
||
|
|
body := strings.Repeat("Sentinel paragraph for torn-read detection.\n\n", 400)
|
||
|
|
_, err := c.Engine.SaveDoc("live-doc", "# Live Doc\n\n"+body)
|
||
|
|
return err
|
||
|
|
}},
|
||
|
|
},
|
||
|
|
When: Steps{
|
||
|
|
{Text: "20 writers and 40 readers hammer the API concurrently", Do: func(c *Context) error {
|
||
|
|
const (
|
||
|
|
writers = 20
|
||
|
|
readers = 40
|
||
|
|
rounds = 25
|
||
|
|
)
|
||
|
|
var wg sync.WaitGroup
|
||
|
|
errs := make(chan error, writers+readers)
|
||
|
|
|
||
|
|
for w := 0; w < writers; w++ {
|
||
|
|
wg.Add(1)
|
||
|
|
go func(index int) {
|
||
|
|
defer wg.Done()
|
||
|
|
for round := 0; round < rounds; round++ {
|
||
|
|
body := strings.Repeat(
|
||
|
|
fmt.Sprintf("writer %d round %d payload line.\n\n", index, round), 200)
|
||
|
|
if _, err := c.Engine.SaveDoc("live-doc", "# Live Doc\n\n"+body); err != nil {
|
||
|
|
errs <- err
|
||
|
|
return
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}(w)
|
||
|
|
}
|
||
|
|
|
||
|
|
for r := 0; r < readers; r++ {
|
||
|
|
wg.Add(1)
|
||
|
|
go func() {
|
||
|
|
defer wg.Done()
|
||
|
|
for round := 0; round < rounds*4; round++ {
|
||
|
|
content, err := c.Engine.ReadDoc("live-doc")
|
||
|
|
if err != nil {
|
||
|
|
errs <- err
|
||
|
|
return
|
||
|
|
}
|
||
|
|
if !strings.HasPrefix(content, "# Live Doc\n\n") {
|
||
|
|
errs <- errf("reader saw a truncated document header: %q", truncate(content))
|
||
|
|
return
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}()
|
||
|
|
}
|
||
|
|
|
||
|
|
wg.Wait()
|
||
|
|
close(errs)
|
||
|
|
for err := range errs {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
return nil
|
||
|
|
}},
|
||
|
|
},
|
||
|
|
Then: Steps{
|
||
|
|
{Text: "every read returned a complete document", Do: func(c *Context) error {
|
||
|
|
content, err := c.RawDoc("live-doc")
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
if !strings.HasPrefix(content, "# Live Doc\n\n") {
|
||
|
|
return errf("final document is malformed")
|
||
|
|
}
|
||
|
|
return nil
|
||
|
|
}},
|
||
|
|
{Text: "no temporary file was left behind in data/docs", Do: func(c *Context) error {
|
||
|
|
entries, err := listDir(c.DataDir + "/docs")
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
for _, name := range entries {
|
||
|
|
if strings.HasPrefix(name, ".godoc-tmp-") {
|
||
|
|
return errf("atomic-write temp file leaked: %s", name)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return nil
|
||
|
|
}},
|
||
|
|
},
|
||
|
|
},
|
||
|
|
{
|
||
|
|
Name: "many documents can be seeded in parallel without cross-contamination",
|
||
|
|
Given: Steps{
|
||
|
|
{Text: "120 documents seeded across 8 workers with 25 comments each", Do: func(c *Context) error {
|
||
|
|
const (
|
||
|
|
docs = 120
|
||
|
|
perDoc = 25
|
||
|
|
numWorkers = 8
|
||
|
|
)
|
||
|
|
var wg sync.WaitGroup
|
||
|
|
errs := make(chan error, numWorkers)
|
||
|
|
jobs := make(chan int, docs)
|
||
|
|
|
||
|
|
for i := 0; i < docs; i++ {
|
||
|
|
jobs <- i
|
||
|
|
}
|
||
|
|
close(jobs)
|
||
|
|
|
||
|
|
for w := 0; w < numWorkers; w++ {
|
||
|
|
wg.Add(1)
|
||
|
|
go func(worker int) {
|
||
|
|
defer wg.Done()
|
||
|
|
roles := []storage.Role{storage.RoleUser, storage.RoleOperator, storage.RoleDeveloper}
|
||
|
|
for index := range jobs {
|
||
|
|
title := fmt.Sprintf("Service Suite %03d Overview", index)
|
||
|
|
slug := storage.Slugify(title)
|
||
|
|
body := fmt.Sprintf("# %s\n\n## For Users\n\nUsage.\n\n## For Operators\n\nRunbook.\n\n## For Developers\n\nAPI.\n",
|
||
|
|
title)
|
||
|
|
if _, err := c.Engine.SaveDoc(title, body); err != nil {
|
||
|
|
errs <- err
|
||
|
|
return
|
||
|
|
}
|
||
|
|
for n := 0; n < perDoc; n++ {
|
||
|
|
role := roles[n%len(roles)]
|
||
|
|
if _, err := c.Engine.AppendComment(slug, storage.CommentInput{
|
||
|
|
Author: fmt.Sprintf("w%d-u%d", worker, n),
|
||
|
|
Role: string(role),
|
||
|
|
Content: fmt.Sprintf("note %d for %s", n, slug),
|
||
|
|
}); err != nil {
|
||
|
|
errs <- err
|
||
|
|
return
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}(w)
|
||
|
|
}
|
||
|
|
|
||
|
|
wg.Wait()
|
||
|
|
close(errs)
|
||
|
|
for err := range errs {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
return nil
|
||
|
|
}},
|
||
|
|
},
|
||
|
|
When: Steps{
|
||
|
|
{Text: "the document index is read back", Do: func(c *Context) error {
|
||
|
|
return nil
|
||
|
|
}},
|
||
|
|
},
|
||
|
|
Then: Steps{
|
||
|
|
{Text: "all 120 slugified documents are indexed", Do: func(c *Context) error {
|
||
|
|
metas, err := c.Engine.ListDocs()
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
return ExpectEqual(len(metas), 120, "indexed document count")
|
||
|
|
}},
|
||
|
|
{Text: "each document owns exactly its own 25 comments", Do: func(c *Context) error {
|
||
|
|
for index := 0; index < 120; index++ {
|
||
|
|
slug := storage.Slugify(fmt.Sprintf("Service Suite %03d Overview", index))
|
||
|
|
comments, err := c.RawComments(slug)
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
if len(comments) != 25 {
|
||
|
|
return errf("%s: expected 25 comments, got %d", slug, len(comments))
|
||
|
|
}
|
||
|
|
for _, comment := range comments {
|
||
|
|
if !strings.Contains(comment.Content, slug) {
|
||
|
|
return errf("%s: comment from another document leaked in: %q", slug, comment.Content)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return nil
|
||
|
|
}},
|
||
|
|
{Text: "the aggregate stats total 3000 comments", Do: func(c *Context) error {
|
||
|
|
response, payload, err := c.Do("GET", "/api/stats", "", "")
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
if err := ExpectStatus(response, 200); err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
stats, err := Decode[struct {
|
||
|
|
Docs int `json:"docs"`
|
||
|
|
Comments int `json:"comments"`
|
||
|
|
}](payload)
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
if stats.Docs != 120 {
|
||
|
|
return errf("expected 120 docs, got %d", stats.Docs)
|
||
|
|
}
|
||
|
|
return ExpectEqual(stats.Comments, 3000, "aggregate comment count")
|
||
|
|
}},
|
||
|
|
},
|
||
|
|
},
|
||
|
|
{
|
||
|
|
Name: "a burst of HTTP comment posts is durable",
|
||
|
|
Given: Steps{
|
||
|
|
{Text: "a documentation page served over HTTP", Do: func(c *Context) error {
|
||
|
|
response, _, err := c.Do("POST", "/api/docs/http-burst", "text/markdown", "# HTTP Burst\n")
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
return ExpectStatus(response, 201)
|
||
|
|
}},
|
||
|
|
},
|
||
|
|
When: Steps{
|
||
|
|
{Text: "200 concurrent POSTs hit /api/docs/http-burst/comments", Do: func(c *Context) error {
|
||
|
|
const posts = 200
|
||
|
|
var wg sync.WaitGroup
|
||
|
|
errs := make(chan error, posts)
|
||
|
|
|
||
|
|
for i := 0; i < posts; i++ {
|
||
|
|
wg.Add(1)
|
||
|
|
go func(index int) {
|
||
|
|
defer wg.Done()
|
||
|
|
role := []string{"user", "operator", "developer"}[index%3]
|
||
|
|
body := fmt.Sprintf(`{"author":"burst-%03d","role":"%s","content":"burst comment %d"}`, index, role, index)
|
||
|
|
response, payload, err := c.Do("POST", "/api/docs/http-burst/comments", "application/json", body)
|
||
|
|
if err != nil {
|
||
|
|
errs <- err
|
||
|
|
return
|
||
|
|
}
|
||
|
|
if response.StatusCode != 201 {
|
||
|
|
errs <- errf("POST %d returned %d: %s", index, response.StatusCode, payload)
|
||
|
|
}
|
||
|
|
}(i)
|
||
|
|
}
|
||
|
|
|
||
|
|
wg.Wait()
|
||
|
|
close(errs)
|
||
|
|
for err := range errs {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
return nil
|
||
|
|
}},
|
||
|
|
},
|
||
|
|
Then: Steps{
|
||
|
|
{Text: "the feedback file holds all 200 posts", Do: func(c *Context) error {
|
||
|
|
comments, err := c.RawComments("http-burst")
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
if len(comments) != 200 {
|
||
|
|
return errf("expected 200 persisted comments, got %d", len(comments))
|
||
|
|
}
|
||
|
|
return nil
|
||
|
|
}},
|
||
|
|
{Text: "every comment has a unique ID and a role badge", Do: func(c *Context) error {
|
||
|
|
comments, err := c.RawComments("http-burst")
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
ids := make(map[string]struct{}, len(comments))
|
||
|
|
for _, comment := range comments {
|
||
|
|
if _, duplicate := ids[comment.ID]; duplicate {
|
||
|
|
return errf("duplicate comment ID %s", comment.ID)
|
||
|
|
}
|
||
|
|
ids[comment.ID] = struct{}{}
|
||
|
|
if comment.Badge == "" {
|
||
|
|
return errf("comment %s has no badge", comment.ID)
|
||
|
|
}
|
||
|
|
if comment.Timestamp.IsZero() {
|
||
|
|
return errf("comment %s has no timestamp", comment.ID)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return nil
|
||
|
|
}},
|
||
|
|
{Text: "the write completes well inside a test timeout", Do: func(c *Context) error {
|
||
|
|
// Guards against a regression to unbounded fsync storms: the whole
|
||
|
|
// scenario must finish inside the 10 minute suite budget.
|
||
|
|
if elapsed := time.Since(c.StartedAt); elapsed > 2*time.Minute {
|
||
|
|
return errf("burst scenario took %s", elapsed)
|
||
|
|
}
|
||
|
|
return nil
|
||
|
|
}},
|
||
|
|
},
|
||
|
|
},
|
||
|
|
},
|
||
|
|
})
|
||
|
|
}
|