diff --git a/.gitignore b/.gitignore index 08b0fa7..d0f4085 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ typescript/dist/ typescript/*.tsbuildinfo typescript/.nyc_output/ typescript/coverage/ +typescript/package.json # Rust target/ diff --git a/conformance-tests/runners/golang/go.mod b/conformance-tests/runners/golang/go.mod new file mode 100644 index 0000000..50478df --- /dev/null +++ b/conformance-tests/runners/golang/go.mod @@ -0,0 +1,7 @@ +module conformance-tests/runners/golang + +go 1.21 + +require peoplesgrocers.com/code/oss/lseq/golang v0.0.0 + +replace peoplesgrocers.com/code/oss/lseq/golang => ../../../golang diff --git a/conformance-tests/runners/golang/main_test.go b/conformance-tests/runners/golang/main_test.go new file mode 100644 index 0000000..6276123 --- /dev/null +++ b/conformance-tests/runners/golang/main_test.go @@ -0,0 +1,124 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "sort" + "strings" + "testing" + + "peoplesgrocers.com/code/oss/lseq/golang" +) + +type Operation struct { + Before *int `json:"before,omitempty"` + After *int `json:"after,omitempty"` + Expected string `json:"expected"` +} + +type Scenario struct { + Name string `json:"name"` + Description string `json:"description"` + Seed int `json:"seed"` + Init []string `json:"init"` + RNG []float64 `json:"rng"` + Operations []Operation `json:"operations"` +} + +func createMockRandom(values []float64) func() float64 { + index := 0 + return func() float64 { + if index >= len(values) { + panic("ran out of random values") + } + v := values[index] + index++ + return v + } +} + +func TestConformance(t *testing.T) { + testDataDir := "../../genfiles" + entries, err := os.ReadDir(testDataDir) + if err != nil { + t.Fatalf("Failed to read test data directory: %v", err) + } + + var files []string + for _, entry := range entries { + if strings.HasSuffix(entry.Name(), ".scenario.json") { + files = append(files, entry.Name()) + } + } + sort.Strings(files) + + for _, file := range files { + filePath := filepath.Join(testDataDir, file) + data, err := os.ReadFile(filePath) + if err != nil { + t.Fatalf("Failed to read %s: %v", file, err) + } + + var scenario Scenario + if err := json.Unmarshal(data, &scenario); err != nil { + t.Fatalf("Failed to parse %s: %v", file, err) + } + + t.Run(scenario.Name, func(t *testing.T) { + mockRandom := createMockRandom(scenario.RNG) + l := lseq.NewLSEQ(mockRandom) + state := make([]string, len(scenario.Init)) + copy(state, scenario.Init) + + for i, op := range scenario.Operations { + var beforeKey, afterKey *string + var insertIdx int + + if op.Before != nil { + // Insert before index X + idx := *op.Before + if idx < 0 { + idx = len(state) + idx + } + if idx > 0 { + beforeKey = &state[idx-1] + } + if idx < len(state) { + afterKey = &state[idx] + } + insertIdx = idx + } else if op.After != nil { + // Insert after index X + idx := *op.After + if idx < 0 { + idx = len(state) + idx + } + if idx >= 0 && idx < len(state) { + beforeKey = &state[idx] + } + if idx+1 < len(state) { + afterKey = &state[idx+1] + } + insertIdx = idx + 1 + } else { + // Neither specified - insert at end + if len(state) > 0 { + beforeKey = &state[len(state)-1] + } + insertIdx = len(state) + } + + result := l.Alloc(beforeKey, afterKey) + + if result != op.Expected { + t.Errorf("op %d: alloc(%v, %v) = %q, want %q", + i, beforeKey, afterKey, result, op.Expected) + } + + // Insert result into state + state = append(state[:insertIdx], append([]string{result}, state[insertIdx:]...)...) + } + }) + } +} diff --git a/golang/README.md b/golang/README.md new file mode 100644 index 0000000..f13d9fd --- /dev/null +++ b/golang/README.md @@ -0,0 +1,103 @@ +# lseq + +Go implementation of the L-SEQ algorithm for fractional indexing and list CRDTs. + +The library handles edge cases that break other more naive implementations of +fractional indexing. With this library you can always insert before the first +item or after the last item, indefinitely. + +This is exactly what you need to build realtime collaborative apps with +ordering for lists or trees of items. Users can reorder or insert at arbitrary +positions but in practice people really like moving items to the top or end of +a list. + +## Installation + +```bash +go get peoplesgrocers.com/code/oss/lseq/golang +``` + +## Usage + +```go +package main + +import ( + "fmt" + "math/rand" + "sort" + + "peoplesgrocers.com/code/oss/lseq/golang" +) + +func main() { + l := lseq.NewLSEQ(rand.Float64) + + first := l.Alloc(nil, nil) + second := l.Alloc(&first, nil) + between := l.Alloc(&first, &second) + + // String comparison gives correct order + keys := []string{second, first, between} + sort.Strings(keys) + fmt.Println(keys) // [first, between, second] +} +``` + +## API + +### `NewLSEQ(random func() float64) *LSEQ` + +Create an allocator. Requires a random function returning values in [0, 1). + +### `lseq.Alloc(before, after *string) string` + +Allocate a key between `before` and `after`. + +- `lseq.Alloc(nil, nil)` — first key in an empty list +- `lseq.Alloc(nil, &first)` — insert at head +- `lseq.Alloc(&last, nil)` — insert at tail +- `lseq.Alloc(&a, &b)` — insert between a and b + +### `CompareLSEQ(a, b string) int` + +Compare two keys. Returns -1, 0, or 1. + +### `EvenSpacingIterator` + +Generate evenly-spaced keys for bulk initialization: + +```go +k, iter, err := lseq.NewEvenSpacingIterator(1000) +if err != nil { + panic(err) +} + +var keys []string +for { + pos, ok := iter.Next() + if !ok { + break + } + keys = append(keys, lseq.PositionToKey(k, pos)) +} +``` + +## Design + +Keys are base-64 encoded strings using the alphabet `-0-9A-Z_a-z`. Standard +string comparison produces correct ordering. Keys work in SQL `ORDER BY`, +database indexes, and any language with string sorting. + +## Cross-language support + +Matching implementations exist for Rust, TypeScript, and Python; validated +against a shared conformance test suite. + +## References + +- [LSEQ: an Adaptive Structure for Sequences in Distributed Collaborative Editing](https://hal.science/hal-00921633/document) (Nédelec et al., 2013) + +## License + +AGPL-3.0 diff --git a/golang/go.mod b/golang/go.mod new file mode 100644 index 0000000..e5ee2e5 --- /dev/null +++ b/golang/go.mod @@ -0,0 +1,3 @@ +module peoplesgrocers.com/code/oss/lseq/golang + +go 1.21 diff --git a/golang/lseq.go b/golang/lseq.go new file mode 100644 index 0000000..9576745 --- /dev/null +++ b/golang/lseq.go @@ -0,0 +1,293 @@ +// Package lseq provides an implementation of LSEQ, a CRDT-friendly sequence +// allocation algorithm that generates lexicographically sortable identifiers. +package lseq + +import ( + "errors" + "math" + "strings" +) + +// Alphabet is the 64-character set used for encoding sort keys. +// Characters are ordered so that string comparison matches numeric comparison. +const Alphabet = "-0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz" + +// maxLevelExponent caps level values at 64^8 for JavaScript float compatibility. +const maxLevelExponent = 8 + +// maxValueForLevel returns the maximum value for a given level (0-indexed). +// Level 0: 64^1 - 1 = 63 +// Level 1: 64^2 - 1 = 4095 +// Level 7+: 64^8 - 1 (capped for JS compatibility) +func maxValueForLevel(level int) uint64 { + exp := level + 1 + if exp > maxLevelExponent { + exp = maxLevelExponent + } + return uint64(math.Pow(64, float64(exp))) - 1 +} + +// charsForLevel returns the number of characters needed to encode a value at this level. +func charsForLevel(level int) int { + chars := level + 1 + if chars > maxLevelExponent { + chars = maxLevelExponent + } + return chars +} + +// idToNumbers parses a sort key string into its numeric components. +func idToNumbers(id string) ([]uint64, error) { + if id == "" { + return nil, nil + } + + numbers := make([]uint64, 0) + pos := 0 + level := 0 + + for pos < len(id) { + numChars := charsForLevel(level) + if pos+numChars > len(id) { + return nil, errors.New("invalid sort key length") + } + + var value uint64 + for i := 0; i < numChars; i++ { + char := id[pos+i] + digit := strings.IndexByte(Alphabet, char) + if digit == -1 { + return nil, errors.New("invalid character in sort key") + } + value = value*64 + uint64(digit) + } + + numbers = append(numbers, value) + pos += numChars + level++ + } + + return numbers, nil +} + +// numbersToID encodes numeric components into a sort key string. +func numbersToID(numbers []uint64) string { + var result strings.Builder + for level, value := range numbers { + numChars := charsForLevel(level) + chars := make([]byte, numChars) + v := value + + for i := numChars - 1; i >= 0; i-- { + chars[i] = Alphabet[v%64] + v /= 64 + } + + result.Write(chars) + } + return result.String() +} + +// LSEQ allocates sort keys between existing keys using the LSEQ algorithm. +type LSEQ struct { + strategies []bool + random func() float64 +} + +// NewLSEQ creates a new LSEQ allocator with the given random number generator. +// The random function should return values in [0, 1). +// If random is nil, a default implementation using math/rand is NOT provided; +// the caller must supply their own RNG for deterministic behavior. +func NewLSEQ(random func() float64) *LSEQ { + l := &LSEQ{ + strategies: make([]bool, 1), + random: random, + } + l.strategies[0] = random() < 0.5 + return l +} + +// genRange returns a random integer in [min, max). +func (l *LSEQ) genRange(min, max uint64) uint64 { + return min + uint64(l.random()*float64(max-min)) +} + +// Alloc allocates a new sort key between before and after. +// If before is nil, allocates at the beginning. +// If after is nil, allocates at the end. +func (l *LSEQ) Alloc(before, after *string) string { + var p, q []uint64 + var err error + + if before != nil { + p, err = idToNumbers(*before) + if err != nil { + p = nil + } + } + if after != nil { + q, err = idToNumbers(*after) + if err != nil { + q = nil + } + } + + depth := 0 + result := make([]uint64, 0) + + for { + var pVal uint64 + if depth < len(p) { + pVal = p[depth] + } + + var qUpper *uint64 + if depth < len(q) { + qUpper = &q[depth] + } + + levelMax := maxValueForLevel(depth) + + minAlloc := pVal + 1 + var maxAlloc uint64 + if qUpper != nil { + if *qUpper > 0 { + maxAlloc = *qUpper - 1 + } else { + maxAlloc = 0 + } + } else { + maxAlloc = levelMax + } + + if minAlloc <= maxAlloc { + rangeSize := maxAlloc - minAlloc + 1 + offset := l.genRange(0, rangeSize) + var newValue uint64 + if l.strategies[depth] { + newValue = minAlloc + offset + } else { + newValue = maxAlloc - offset + } + result = append(result, newValue) + return numbersToID(result) + } + + result = append(result, pVal) + + depth++ + if depth >= len(l.strategies) { + l.strategies = append(l.strategies, l.random() < 0.5) + } + } +} + +// CompareLSEQ compares two LSEQ sort keys. +// Returns -1 if a < b, 0 if a == b, 1 if a > b. +func CompareLSEQ(a, b string) int { + return strings.Compare(a, b) +} + +// EvenSpacingIterator generates evenly spaced positions for bulk allocation. +type EvenSpacingIterator struct { + remainingItems int + spaceSize uint64 + nextItem uint64 + stepSizeInteger uint64 + stepSizeError float64 + errorAccumulator float64 +} + +// usableSpace contains (64^k - 1) values for k from 1 to 8. +// Position 0 is reserved (nothing can sort before all-zeros). +var usableSpace = []uint64{ + 64 - 1, // 64^1 - 1 + 4096 - 1, // 64^2 - 1 + 262144 - 1, // 64^3 - 1 + 16777216 - 1, // 64^4 - 1 + 1073741824 - 1, // 64^5 - 1 + 68719476736 - 1, // 64^6 - 1 + 4398046511104 - 1, // 64^7 - 1 + 281474976710656 - 1, // 64^8 - 1 +} + +// ErrTooManyItems is returned when the requested number of items exceeds capacity. +var ErrTooManyItems = errors.New("too many items to allocate") + +// NewEvenSpacingIterator creates an iterator for evenly spacing totalItems positions. +// Returns the level k and the iterator, or an error if totalItems is too large. +func NewEvenSpacingIterator(totalItems int) (k int, iter *EvenSpacingIterator, err error) { + if totalItems == 0 { + return 0, nil, ErrTooManyItems + } + + k = 0 + var spaceSize uint64 + + for i, size := range usableSpace { + if size >= uint64(totalItems) { + k = i + 1 + spaceSize = size + break + } + } + + if k == 0 { + return 0, nil, ErrTooManyItems + } + + stepSize := float64(spaceSize) / float64(totalItems) + stepSizeInteger := uint64(stepSize) + stepSizeError := stepSize - float64(stepSizeInteger) + + iter = &EvenSpacingIterator{ + remainingItems: totalItems, + spaceSize: spaceSize, + nextItem: 1, + stepSizeInteger: stepSizeInteger, + stepSizeError: stepSizeError, + errorAccumulator: 0, + } + + return k, iter, nil +} + +// Next returns the next position and true, or 0 and false if exhausted. +func (e *EvenSpacingIterator) Next() (uint64, bool) { + if e.remainingItems == 0 { + return 0, false + } + + if e.nextItem > e.spaceSize { + return 0, false + } + + currentPosition := e.nextItem + e.remainingItems-- + + e.nextItem += e.stepSizeInteger + + e.errorAccumulator += e.stepSizeError + if e.errorAccumulator >= 1.0 { + e.nextItem++ + e.errorAccumulator -= 1.0 + } + + return currentPosition, true +} + +// PositionToKey converts a position within a level-k space to a sort key string. +// Creates a k-level key where levels 0 through k-2 are 0, and level k-1 contains the position. +func PositionToKey(k int, position uint64) string { + result := make([]uint64, 0, k) + + for i := 0; i < k-1; i++ { + result = append(result, 0) + } + + if k > 0 { + result = append(result, position) + } + + return numbersToID(result) +} diff --git a/golang/lseq_test.go b/golang/lseq_test.go new file mode 100644 index 0000000..22d9c71 --- /dev/null +++ b/golang/lseq_test.go @@ -0,0 +1,44 @@ +package lseq + +import ( + "testing" +) + +func TestStringEncoding(t *testing.T) { + // Basic sanity checks for encoding/decoding + cases := []struct { + numbers []uint64 + want string + }{ + {[]uint64{0}, "-"}, + {[]uint64{63}, "z"}, + {[]uint64{0, 1}, "--0"}, + {[]uint64{0, 4095}, "-zz"}, + } + + for _, tc := range cases { + got := numbersToID(tc.numbers) + if got != tc.want { + t.Errorf("numbersToID(%v) = %q, want %q", tc.numbers, got, tc.want) + } + + parsed, err := idToNumbers(got) + if err != nil { + t.Errorf("idToNumbers(%q) failed: %v", got, err) + continue + } + + if len(parsed) != len(tc.numbers) { + t.Errorf("Roundtrip failed: got %v, want %v", parsed, tc.numbers) + } + } +} + +func TestPositionToKey(t *testing.T) { + if got := PositionToKey(2, 1); got != "--0" { + t.Errorf("PositionToKey(2, 1) = %q, want %q", got, "--0") + } + if got := PositionToKey(1, 1); got != "0" { + t.Errorf("PositionToKey(1, 1) = %q, want %q", got, "0") + } +}