jay/go-heartwood
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Minor improvements for clearer API.

Browse Source
This commit is contained in:
Jay
2026-03-05 09:36:36 -05:00
parent 403f024645
commit 68354a73fc
5 changed files with 180 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

75
README.md
View File

@@ -24,7 +24,80 @@ replace git.wisehodl.dev/jay/go-heartwood => github.com/wisehodl/go-heartwood la
## Usage ## Usage
*Usage examples will be added as the API stabilizes.* *Further usage examples will be added as the API stabilizes.*
### Writing Events
```go
package main
import (
"context"
"encoding/json"
"log"
"git.wisehodl.dev/jay/go-roots/events"
"git.wisehodl.dev/jay/go-roots/keys"
"git.wisehodl.dev/jay/go-heartwood"
"github.com/boltdb/bolt"
)
func main() {
ctx := context.Background()
// Connect to Neo4j
driver, err := heartwood.ConnectNeo4j(ctx, "bolt://localhost:7687", "neo4j", "password")
if err != nil {
log.Fatal(err)
}
defer driver.Close(ctx)
// Ensure the necessary indexes and constraints exist
if err := heartwood.SetNeo4jSchema(ctx, driver); err != nil {
log.Fatal(err)
}
// Open BoltDB
boltdb, err := bolt.Open("events.db", 0600, nil)
if err != nil {
log.Fatal(err)
}
defer boltdb.Close()
// Build events using go-roots
sk, _ := keys.GeneratePrivateKey()
pk, _ := keys.GetPublicKey(sk)
event := events.Event{
PubKey: pk,
CreatedAt: 1000,
Kind: 1,
Content: "hello from heartwood",
Tags: []events.Tag{},
}
event.ID, _ = events.GenerateEventID(event)
event.Sig, _ = events.SignEvent(event, sk)
eventJSON, _ := json.Marshal(event)
// Write events
report, err := heartwood.WriteEvents(
[][]byte{eventJSON},
driver, boltdb,
nil, // default WriteOptions
)
if err != nil {
log.Fatal(err)
}
log.Printf("created: %d, excluded: %d, duration: %s",
report.CreatedEventCount,
len(report.ExcludedEvents),
report.Duration,
)
}
```
## Testing ## Testing

8
boltdb.go
View File

@@ -7,8 +7,8 @@ import (
const BucketName string = "events" const BucketName string = "events"
type EventBlob struct { type EventBlob struct {
ID string ID []byte
JSON string JSON []byte
} }
func SetupBoltDB(boltdb *bolt.DB) error { func SetupBoltDB(boltdb *bolt.DB) error {
@@ -39,9 +39,7 @@ func BatchWriteEvents(boltdb *bolt.DB, events []EventBlob) error {
return boltdb.Update(func(tx *bolt.Tx) error { return boltdb.Update(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(BucketName)) bucket := tx.Bucket([]byte(BucketName))
for _, event := range events { for _, event := range events {
if err := bucket.Put( if err := bucket.Put(event.ID, event.JSON); err != nil {
[]byte(event.ID), []byte(event.JSON),
); err != nil {
return err return err
} }
} }

24
boltdb_test.go
View File

@@ -58,7 +58,7 @@ func TestBatchCheckEventsExist(t *testing.T) {
{ {
name: "some exist", name: "some exist",
setupEvents: []EventBlob{ setupEvents: []EventBlob{
{ID: "id1", JSON: `{"test":1}`}, {ID: []byte("id1"), JSON: []byte(`{"test":1}`)},
}, },
checkIDs: []string{"id1", "id2"}, checkIDs: []string{"id1", "id2"},
expectedMap: map[string]bool{ expectedMap: map[string]bool{
@@ -69,8 +69,8 @@ func TestBatchCheckEventsExist(t *testing.T) {
{ {
name: "all exist", name: "all exist",
setupEvents: []EventBlob{ setupEvents: []EventBlob{
{ID: "id1", JSON: `{"test":1}`}, {ID: []byte("id1"), JSON: []byte(`{"test":1}`)},
{ID: "id2", JSON: `{"test":2}`}, {ID: []byte("id2"), JSON: []byte(`{"test":2}`)},
}, },
checkIDs: []string{"id1", "id2"}, checkIDs: []string{"id1", "id2"},
expectedMap: map[string]bool{ expectedMap: map[string]bool{
@@ -121,7 +121,7 @@ func TestBatchWriteEvents(t *testing.T) {
{ {
name: "single event", name: "single event",
events: []EventBlob{ events: []EventBlob{
{ID: "id1", JSON: `{"test":1}`}, {ID: []byte("id1"), JSON: []byte(`{"test":1}`)},
}, },
verifyID: "id1", verifyID: "id1",
expectedJSON: `{"test":1}`, expectedJSON: `{"test":1}`,
@@ -129,9 +129,9 @@ func TestBatchWriteEvents(t *testing.T) {
{ {
name: "multiple events", name: "multiple events",
events: []EventBlob{ events: []EventBlob{
{ID: "id1", JSON: `{"test":1}`}, {ID: []byte("id1"), JSON: []byte(`{"test":1}`)},
{ID: "id2", JSON: `{"test":2}`}, {ID: []byte("id2"), JSON: []byte(`{"test":2}`)},
{ID: "id3", JSON: `{"test":3}`}, {ID: []byte("id3"), JSON: []byte(`{"test":3}`)},
}, },
verifyID: "id2", verifyID: "id2",
expectedJSON: `{"test":2}`, expectedJSON: `{"test":2}`,
@@ -139,8 +139,8 @@ func TestBatchWriteEvents(t *testing.T) {
{ {
name: "duplicate id overwrites", name: "duplicate id overwrites",
events: []EventBlob{ events: []EventBlob{
{ID: "id1", JSON: `{"test":1}`}, {ID: []byte("id1"), JSON: []byte(`{"test":1}`)},
{ID: "id1", JSON: `{"test":2}`}, {ID: []byte("id1"), JSON: []byte(`{"test":2}`)},
}, },
verifyID: "id1", verifyID: "id1",
expectedJSON: `{"test":2}`, expectedJSON: `{"test":2}`,
@@ -177,9 +177,9 @@ func TestWriteThenCheck(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
events := []EventBlob{ events := []EventBlob{
{ID: "id1", JSON: `{"test":1}`}, {ID: []byte("id1"), JSON: []byte(`{"test":1}`)},
{ID: "id2", JSON: `{"test":2}`}, {ID: []byte("id2"), JSON: []byte(`{"test":2}`)},
{ID: "id3", JSON: `{"test":3}`}, {ID: []byte("id3"), JSON: []byte(`{"test":3}`)},
} }
err = BatchWriteEvents(db, events) err = BatchWriteEvents(db, events)

68
write.go
View File

@@ -18,7 +18,7 @@ type WriteOptions struct {
type EventTraveller struct { type EventTraveller struct {
ID string ID string
JSON string JSON []byte
Event roots.Event Event roots.Event
Subgraph *EventSubgraph Subgraph *EventSubgraph
Error error Error error
@@ -30,8 +30,7 @@ type WriteResult struct {
} }
type WriteReport struct { type WriteReport struct {
InvalidEvents []EventTraveller ExcludedEvents []EventTraveller
SkippedEvents []EventTraveller
CreatedEventCount int CreatedEventCount int
Neo4jResultSummaries []neo4j.ResultSummary Neo4jResultSummaries []neo4j.ResultSummary
Duration time.Duration Duration time.Duration
@@ -39,7 +38,7 @@ type WriteReport struct {
} }
func WriteEvents( func WriteEvents(
events []string, events [][]byte,
driver neo4j.Driver, boltdb *bolt.DB, driver neo4j.Driver, boltdb *bolt.DB,
opts *WriteOptions, opts *WriteOptions,
) (WriteReport, error) { ) (WriteReport, error) {
@@ -59,7 +58,7 @@ func WriteEvents(
var wg sync.WaitGroup var wg sync.WaitGroup
// Create Event Travellers // Create Event Travellers
jsonChan := make(chan string) jsonChan := make(chan []byte)
eventChan := make(chan EventTraveller) eventChan := make(chan EventTraveller)
wg.Add(1) wg.Add(1)
@@ -67,30 +66,30 @@ func WriteEvents(
// Parse Event JSON // Parse Event JSON
parsedChan := make(chan EventTraveller) parsedChan := make(chan EventTraveller)
invalidChan := make(chan EventTraveller) parseExcludedChan := make(chan EventTraveller)
wg.Add(1) wg.Add(1)
go parseEventJSON(&wg, eventChan, parsedChan, invalidChan) go parseEventJSON(&wg, eventChan, parsedChan, parseExcludedChan)
// Collect Invalid Events // Collect Rejected Events
collectedInvalidChan := make(chan []EventTraveller) collectedParseExcludedChan := make(chan []EventTraveller)
wg.Add(1) wg.Add(1)
go collectTravellers(&wg, invalidChan, collectedInvalidChan) go collectTravellers(&wg, parseExcludedChan, collectedParseExcludedChan)
// Enforce Policy Rules // Enforce Policy Rules
queuedChan := make(chan EventTraveller) queuedChan := make(chan EventTraveller)
skippedChan := make(chan EventTraveller) policyExcludedChan := make(chan EventTraveller)
wg.Add(1) wg.Add(1)
go enforcePolicyRules(&wg, driver, boltdb, opts.BoltReadBatchSize, go enforcePolicyRules(&wg, driver, boltdb, opts.BoltReadBatchSize,
parsedChan, queuedChan, skippedChan) parsedChan, queuedChan, policyExcludedChan)
// Collect Skipped Events // Collect Skipped Events
collectedSkippedChan := make(chan []EventTraveller) collectedPolicyExcludedChan := make(chan []EventTraveller)
wg.Add(1) wg.Add(1)
go collectTravellers(&wg, skippedChan, collectedSkippedChan) go collectTravellers(&wg, policyExcludedChan, collectedPolicyExcludedChan)
// Convert Events To Subgraphs // Convert Events To Subgraphs
convertedChan := make(chan EventTraveller) convertedChan := make(chan EventTraveller)
@@ -116,14 +115,15 @@ func WriteEvents(
wg.Wait() wg.Wait()
// Collect results // Collect results
invalid := <-collectedInvalidChan parseExcluded := <-collectedParseExcludedChan
skipped := <-collectedSkippedChan policyExcluded := <-collectedPolicyExcludedChan
writeResult := <-writeResultChan writeResult := <-writeResultChan
excluded := append(parseExcluded, policyExcluded...)
return WriteReport{ return WriteReport{
InvalidEvents: invalid, ExcludedEvents: excluded,
SkippedEvents: skipped, CreatedEventCount: len(events) - len(excluded),
CreatedEventCount: len(events) - len(invalid) - len(skipped),
Neo4jResultSummaries: writeResult.ResultSummaries, Neo4jResultSummaries: writeResult.ResultSummaries,
Duration: time.Since(start), Duration: time.Since(start),
Error: writeResult.Error, Error: writeResult.Error,
@@ -139,7 +139,7 @@ func setDefaultWriteOptions(opts *WriteOptions) {
} }
} }
func createEventTravellers(wg *sync.WaitGroup, jsonChan chan string, eventChan chan EventTraveller) { func createEventTravellers(wg *sync.WaitGroup, jsonChan chan []byte, eventChan chan EventTraveller) {
defer wg.Done() defer wg.Done()
for json := range jsonChan { for json := range jsonChan {
eventChan <- EventTraveller{JSON: json} eventChan <- EventTraveller{JSON: json}
@@ -147,15 +147,22 @@ func createEventTravellers(wg *sync.WaitGroup, jsonChan chan string, eventChan c
close(eventChan) close(eventChan)
} }
func parseEventJSON(wg *sync.WaitGroup, inChan, parsedChan, invalidChan chan EventTraveller) { func parseEventJSON(wg *sync.WaitGroup, inChan, parsedChan, excludedChan chan EventTraveller) {
defer wg.Done() defer wg.Done()
for traveller := range inChan { for traveller := range inChan {
var event roots.Event var event roots.Event
jsonBytes := []byte(traveller.JSON) jsonBytes := traveller.JSON
err := json.Unmarshal(jsonBytes, &event) err := json.Unmarshal(jsonBytes, &event)
if err != nil { if err != nil {
traveller.Error = err traveller.Error = fmt.Errorf("rejected: unrecognized event format: %w", err)
invalidChan <- traveller excludedChan <- traveller
continue
}
err = roots.Validate(event)
if err != nil {
traveller.Error = fmt.Errorf("rejected: invalid event: %w", err)
excludedChan <- traveller
continue continue
} }
@@ -165,14 +172,14 @@ func parseEventJSON(wg *sync.WaitGroup, inChan, parsedChan, invalidChan chan Eve
} }
close(parsedChan) close(parsedChan)
close(invalidChan) close(excludedChan)
} }
func enforcePolicyRules( func enforcePolicyRules(
wg *sync.WaitGroup, wg *sync.WaitGroup,
driver neo4j.Driver, boltdb *bolt.DB, driver neo4j.Driver, boltdb *bolt.DB,
batchSize int, batchSize int,
inChan, queuedChan, skippedChan chan EventTraveller, inChan, queuedChan, excludedChan chan EventTraveller,
) { ) {
defer wg.Done() defer wg.Done()
var batch []EventTraveller var batch []EventTraveller
@@ -181,17 +188,17 @@ func enforcePolicyRules(
batch = append(batch, traveller) batch = append(batch, traveller)
if len(batch) >= batchSize { if len(batch) >= batchSize {
processPolicyRulesBatch(boltdb, batch, queuedChan, skippedChan) processPolicyRulesBatch(boltdb, batch, queuedChan, excludedChan)
batch = []EventTraveller{} batch = []EventTraveller{}
} }
} }
if len(batch) > 0 { if len(batch) > 0 {
processPolicyRulesBatch(boltdb, batch, queuedChan, skippedChan) processPolicyRulesBatch(boltdb, batch, queuedChan, excludedChan)
} }
close(queuedChan) close(queuedChan)
close(skippedChan) close(excludedChan)
} }
func processPolicyRulesBatch( func processPolicyRulesBatch(
@@ -209,6 +216,7 @@ func processPolicyRulesBatch(
for _, traveller := range batch { for _, traveller := range batch {
if existsMap[traveller.ID] { if existsMap[traveller.ID] {
traveller.Error = fmt.Errorf("skipped: event already exists")
skippedChan <- traveller skippedChan <- traveller
} else { } else {
queuedChan <- traveller queuedChan <- traveller
@@ -273,7 +281,7 @@ func writeEventsToBoltDB(
for traveller := range inChan { for traveller := range inChan {
events = append(events, events = append(events,
EventBlob{ID: traveller.ID, JSON: traveller.JSON}) EventBlob{ID: []byte(traveller.ID), JSON: traveller.JSON})
} }
err := BatchWriteEvents(boltdb, events) err := BatchWriteEvents(boltdb, events)

87
write_test.go
View File

@@ -1,7 +1,6 @@
package heartwood package heartwood
import ( import (
"fmt"
roots "git.wisehodl.dev/jay/go-roots/events" roots "git.wisehodl.dev/jay/go-roots/events"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"sync" "sync"
@@ -10,12 +9,16 @@ import (
// Test helpers // Test helpers
func validEventJSON(id, pubkey string) string { func validEventJSON() []byte {
return fmt.Sprintf(`{"id":"%s","pubkey":"%s","created_at":1000,"kind":1,"content":"test","tags":[],"sig":"abc"}`, id, pubkey) return []byte(`{"id":"c7a702e6158744ca03508bbb4c90f9dbb0d6e88fefbfaa511d5ab24b4e3c48ad","pubkey":"cfa87f35acbde29ba1ab3ee42de527b2cad33ac487e80cf2d6405ea0042c8fef","created_at":1760740551,"kind":1,"tags":[],"content":"hello world","sig":"83b71e15649c9e9da362c175f988c36404cabf357a976d869102a74451cfb8af486f6088b5631033b4927bd46cad7a0d90d7f624aefc0ac260364aa65c36071a"}`)
} }
func invalidEventJSON() string { func invalidEventJSON() []byte {
return `{invalid json` return []byte(`{"id":"abc123","pubkey":"xyz789","created_at":1000,"kind":1,"content":"test","tags":[],"sig":"abc"}`)
}
func malformedEventJSON() []byte {
return []byte(`{malformed json`)
} }
// Pipeline stage tests // Pipeline stage tests
@@ -23,28 +26,28 @@ func invalidEventJSON() string {
func TestCreateEventTravellers(t *testing.T) { func TestCreateEventTravellers(t *testing.T) {
cases := []struct { cases := []struct {
name string name string
input []string input [][]byte
expected []EventTraveller expected []EventTraveller
}{ }{
{ {
name: "empty input", name: "empty input",
input: []string{}, input: [][]byte{},
expected: []EventTraveller{}, expected: []EventTraveller{},
}, },
{ {
name: "single json", name: "single json",
input: []string{"test1"}, input: [][]byte{[]byte("test1")},
expected: []EventTraveller{ expected: []EventTraveller{
{JSON: "test1"}, {JSON: []byte("test1")},
}, },
}, },
{ {
name: "multiple jsons", name: "multiple jsons",
input: []string{"test1", "test2", "test3"}, input: [][]byte{[]byte("test1"), []byte("test2"), []byte("test3")},
expected: []EventTraveller{ expected: []EventTraveller{
{JSON: "test1"}, {JSON: []byte("test1")},
{JSON: "test2"}, {JSON: []byte("test2")},
{JSON: "test3"}, {JSON: []byte("test3")},
}, },
}, },
} }
@@ -52,7 +55,7 @@ func TestCreateEventTravellers(t *testing.T) {
for _, tc := range cases { for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
var wg sync.WaitGroup var wg sync.WaitGroup
jsonChan := make(chan string) jsonChan := make(chan []byte)
eventChan := make(chan EventTraveller) eventChan := make(chan EventTraveller)
wg.Add(1) wg.Add(1)
@@ -86,37 +89,48 @@ func TestParseEventJSON(t *testing.T) {
name string name string
input []EventTraveller input []EventTraveller
wantParsed int wantParsed int
wantInvalid int wantRejected int
checkParsedID bool checkParsedID bool
expectedID string expectedID string
wantErrorText string
}{ }{
{ {
name: "valid event", name: "valid event",
input: []EventTraveller{ input: []EventTraveller{
{JSON: validEventJSON("abc123", "pubkey1")}, {JSON: validEventJSON()},
}, },
wantParsed: 1, wantParsed: 1,
wantInvalid: 0, wantRejected: 0,
checkParsedID: true, checkParsedID: true,
expectedID: "abc123", expectedID: "c7a702e6158744ca03508bbb4c90f9dbb0d6e88fefbfaa511d5ab24b4e3c48ad",
}, },
{ {
name: "invalid json", name: "invalid event",
input: []EventTraveller{ input: []EventTraveller{
{JSON: invalidEventJSON()}, {JSON: invalidEventJSON()},
}, },
wantParsed: 0, wantParsed: 0,
wantInvalid: 1, wantRejected: 1,
wantErrorText: "rejected: invalid event",
},
{
name: "malformed json",
input: []EventTraveller{
{JSON: malformedEventJSON()},
},
wantParsed: 0,
wantRejected: 1,
wantErrorText: "rejected: unrecognized event format",
}, },
{ {
name: "mixed batch", name: "mixed batch",
input: []EventTraveller{ input: []EventTraveller{
{JSON: validEventJSON("abc123", "pubkey1")},
{JSON: invalidEventJSON()}, {JSON: invalidEventJSON()},
{JSON: validEventJSON("def456", "pubkey2")}, {JSON: malformedEventJSON()},
{JSON: validEventJSON()},
}, },
wantParsed: 2, wantParsed: 1,
wantInvalid: 1, wantRejected: 2,
}, },
} }
@@ -125,10 +139,10 @@ func TestParseEventJSON(t *testing.T) {
var wg sync.WaitGroup var wg sync.WaitGroup
inChan := make(chan EventTraveller) inChan := make(chan EventTraveller)
parsedChan := make(chan EventTraveller) parsedChan := make(chan EventTraveller)
invalidChan := make(chan EventTraveller) rejectedChan := make(chan EventTraveller)
wg.Add(1) wg.Add(1)
go parseEventJSON(&wg, inChan, parsedChan, invalidChan) go parseEventJSON(&wg, inChan, parsedChan, rejectedChan)
go func() { go func() {
for _, traveller := range tc.input { for _, traveller := range tc.input {
@@ -138,7 +152,7 @@ func TestParseEventJSON(t *testing.T) {
}() }()
var parsed []EventTraveller var parsed []EventTraveller
var invalid []EventTraveller var rejected []EventTraveller
var collectWg sync.WaitGroup var collectWg sync.WaitGroup
collectWg.Add(2) collectWg.Add(2)
@@ -152,8 +166,8 @@ func TestParseEventJSON(t *testing.T) {
go func() { go func() {
defer collectWg.Done() defer collectWg.Done()
for f := range invalidChan { for f := range rejectedChan {
invalid = append(invalid, f) rejected = append(rejected, f)
} }
}() }()
@@ -161,7 +175,7 @@ func TestParseEventJSON(t *testing.T) {
wg.Wait() wg.Wait()
assert.Equal(t, tc.wantParsed, len(parsed)) assert.Equal(t, tc.wantParsed, len(parsed))
assert.Equal(t, tc.wantInvalid, len(invalid)) assert.Equal(t, tc.wantRejected, len(rejected))
// Smoke test first parsed id // Smoke test first parsed id
if tc.checkParsedID && len(parsed) > 0 { if tc.checkParsedID && len(parsed) > 0 {
@@ -169,9 +183,14 @@ func TestParseEventJSON(t *testing.T) {
assert.NotEmpty(t, parsed[0].Event.ID) assert.NotEmpty(t, parsed[0].Event.ID)
} }
for _, inv := range invalid { // Check error text on first rejected event
assert.NotNil(t, inv.Error) if tc.wantErrorText != "" {
assert.Empty(t, inv.Event.ID) assert.ErrorContains(t, rejected[0].Error, tc.wantErrorText)
}
for _, reject := range rejected {
assert.NotNil(t, reject.Error)
assert.Empty(t, reject.Event.ID)
} }
}) })
} }