jay/go-heartwood
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Phase 4: serialize outer pipeline; remove channels and goroutines from WriteEvents; add TestEnforcePolicyRules

Browse Source
This commit is contained in:
Jay
2026-05-22 16:31:52 -04:00
parent bce69a146b
commit 04e6c4fc57
2 changed files with 129 additions and 289 deletions
Download Patch File Download Diff File Expand all files Collapse all files
write.go
+46 -158
View File
@@ -8,7 +8,6 @@ import (
roots "git.wisehodl.dev/jay/go-roots/events"
"github.com/boltdb/bolt"
"github.com/neo4j/neo4j-go-driver/v6/neo4j"
"sync"
"time"
)
@@ -60,69 +59,11 @@ func WriteEvents(
return WriteReport{Error: fmt.Errorf("error setting up bolt db: %w", err)}
}
var wg sync.WaitGroup
// Create Event Travellers
jsonChan := make(chan []byte)
eventChan := make(chan EventTraveller)
wg.Add(1)
go createEventTravellers(&wg, jsonChan, eventChan)
// Parse Event JSON
parsedChan := make(chan EventTraveller)
parseExcludedChan := make(chan EventTraveller)
wg.Add(1)
go parseEventJSON(&wg, eventChan, parsedChan, parseExcludedChan)
// Collect Rejected Events
collectedParseExcludedChan := make(chan []EventTraveller, 1)
wg.Add(1)
go collectTravellers(&wg, parseExcludedChan, collectedParseExcludedChan)
// Enforce Policy Rules
queuedChan := make(chan EventTraveller)
policyExcludedChan := make(chan EventTraveller)
wg.Add(1)
go enforcePolicyRules(&wg, driver, boltdb, opts.BoltReadBatchSize,
parsedChan, queuedChan, policyExcludedChan)
// Collect Skipped Events
collectedPolicyExcludedChan := make(chan []EventTraveller, 1)
wg.Add(1)
go collectTravellers(&wg, policyExcludedChan, collectedPolicyExcludedChan)
// Convert Events To Subgraphs
convertedChan := make(chan EventTraveller)
wg.Add(1)
go convertEventsToSubgraphs(&wg, opts.Expanders, queuedChan, convertedChan)
// Write Events To Databases
writeResultChan := make(chan WriteResult, 1)
wg.Add(1)
go writeEventsToDatabases(&wg, driver, boltdb, convertedChan, writeResultChan)
// Send event jsons into pipeline
go func() {
for _, raw := range events {
jsonChan <- raw
}
close(jsonChan)
}()
// Wait for pipeline to complete
wg.Wait()
// Collect results
parseExcluded := <-collectedParseExcludedChan
policyExcluded := <-collectedPolicyExcludedChan
writeResult := <-writeResultChan
travellers := createEventTravellers(events)
parsed, parseExcluded := parseEventJSON(travellers)
queued, policyExcluded := enforcePolicyRules(parsed, boltdb, opts.BoltReadBatchSize)
converted := convertEventsToSubgraphs(queued, opts.Expanders)
writeResult := writeEventsToDatabases(driver, boltdb, converted)
excluded := append(parseExcluded, policyExcluded...)
@@ -144,129 +85,84 @@ func setDefaultWriteOptions(opts *WriteOptions) {
}
}
func createEventTravellers(wg *sync.WaitGroup, jsonChan chan []byte, eventChan chan EventTraveller) {
defer wg.Done()
for json := range jsonChan {
eventChan <- EventTraveller{JSON: json}
func createEventTravellers(jsons [][]byte) []EventTraveller {
travellers := make([]EventTraveller, 0, len(jsons))
for _, j := range jsons {
travellers = append(travellers, EventTraveller{JSON: j})
}
close(eventChan)
return travellers
}
func parseEventJSON(wg *sync.WaitGroup, inChan, parsedChan, excludedChan chan EventTraveller) {
defer wg.Done()
for traveller := range inChan {
func parseEventJSON(in []EventTraveller) (parsed []EventTraveller, excluded []EventTraveller) {
for _, traveller := range in {
var event roots.Event
jsonBytes := traveller.JSON
err := json.Unmarshal(jsonBytes, &event)
err := json.Unmarshal(traveller.JSON, &event)
if err != nil {
traveller.Error = fmt.Errorf("rejected: %w: %w", ErrMalformedJSON, err)
excludedChan <- traveller
excluded = append(excluded, traveller)
continue
}
err = roots.Validate(event)
if err != nil {
traveller.Error = fmt.Errorf("rejected: %w: %w", ErrInvalidEvent, err)
excludedChan <- traveller
excluded = append(excluded, traveller)
continue
}
traveller.ID = event.ID
traveller.Event = event
parsedChan <- traveller
parsed = append(parsed, traveller)
}
close(parsedChan)
close(excludedChan)
return parsed, excluded
}
func enforcePolicyRules(
wg *sync.WaitGroup,
driver neo4j.Driver, boltdb *bolt.DB,
batchSize int,
inChan, queuedChan, excludedChan chan EventTraveller,
) {
defer wg.Done()
var batch []EventTraveller
func enforcePolicyRules(in []EventTraveller, boltdb *bolt.DB, batchSize int) (queued []EventTraveller, excluded []EventTraveller) {
for i := 0; i < len(in); i += batchSize {
end := i + batchSize
if end > len(in) {
end = len(in)
}
batch := in[i:end]
for traveller := range inChan {
batch = append(batch, traveller)
eventIDs := make([]string, 0, len(batch))
for _, traveller := range batch {
eventIDs = append(eventIDs, traveller.ID)
}
if len(batch) >= batchSize {
processPolicyRulesBatch(boltdb, batch, queuedChan, excludedChan)
batch = []EventTraveller{}
existsMap := BatchCheckEventsExist(boltdb, eventIDs)
for _, traveller := range batch {
if existsMap[traveller.ID] {
traveller.Error = fmt.Errorf("skipped: %w", ErrDuplicate)
excluded = append(excluded, traveller)
} else {
queued = append(queued, traveller)
}
}
}
if len(batch) > 0 {
processPolicyRulesBatch(boltdb, batch, queuedChan, excludedChan)
}
close(queuedChan)
close(excludedChan)
return queued, excluded
}
func processPolicyRulesBatch(
boltdb *bolt.DB,
batch []EventTraveller,
queuedChan, skippedChan chan EventTraveller,
) {
eventIDs := make([]string, 0, len(batch))
for _, traveller := range batch {
eventIDs = append(eventIDs, traveller.ID)
}
existsMap := BatchCheckEventsExist(boltdb, eventIDs)
for _, traveller := range batch {
if existsMap[traveller.ID] {
traveller.Error = fmt.Errorf("skipped: %w", ErrDuplicate)
skippedChan <- traveller
} else {
queuedChan <- traveller
}
}
}
func convertEventsToSubgraphs(
wg *sync.WaitGroup, expanders ExpanderPipeline,
inChan, convertedChan chan EventTraveller,
) {
defer wg.Done()
for traveller := range inChan {
func convertEventsToSubgraphs(in []EventTraveller, expanders ExpanderPipeline) []EventTraveller {
for i, traveller := range in {
// TODO: temporary adapter — removed in Phase 5
validated, _ := roots.NewValidatedEvent(traveller.Event)
subgraph := EventToSubgraph(validated, expanders)
traveller.Subgraph = subgraph
convertedChan <- traveller
in[i].Subgraph = EventToSubgraph(validated, expanders)
}
close(convertedChan)
return in
}
func writeEventsToDatabases(
wg *sync.WaitGroup,
driver neo4j.Driver, boltdb *bolt.DB,
inChan chan EventTraveller,
resultChan chan WriteResult,
) {
defer wg.Done()
var travellers []EventTraveller
for traveller := range inChan {
travellers = append(travellers, traveller)
}
func writeEventsToDatabases(driver neo4j.Driver, boltdb *bolt.DB, travellers []EventTraveller) WriteResult {
boltErr := writeEventsToBoltDB(boltdb, travellers)
if boltErr != nil {
resultChan <- WriteResult{
return WriteResult{
Error: fmt.Errorf("boltdb write failed, aborting graph write: %w", boltErr),
}
return
}
summaries, err := writeEventsToGraphDB(driver, travellers)
resultChan <- WriteResult{
return WriteResult{
ResultSummaries: summaries,
Error: err,
}
@@ -297,12 +193,4 @@ func writeEventsToGraphDB(driver neo4j.Driver, travellers []EventTraveller) ([]n
return MergeSubgraph(context.Background(), driver, batch)
}
func collectTravellers(wg *sync.WaitGroup, inChan chan EventTraveller, resultChan chan []EventTraveller) {
defer wg.Done()
var collected []EventTraveller
for traveller := range inChan {
collected = append(collected, traveller)
}
resultChan <- collected
close(resultChan)
}