From b7289f8690c8b0d14600b35447b4b425fbed8766 Mon Sep 17 00:00:00 2001 From: Taranum Wasu Date: Mon, 29 Jun 2026 22:31:42 +0530 Subject: [PATCH 01/24] sync: add SVS v3 large-group revision (mhash, PARTIAL, announce+pull) Implements the approved large-group SVS revision with backward-compatible defaults (SyncVectorThreshold=0). Adds mhash and PARTIAL/announce+pull paths, 32=sv publish/pull recovery, tests, and a large-sync example. Co-authored-by: Cursor --- std/engine/face/multicast_face.go | 82 ++++++++++ std/examples/svs/large-sync/main.go | 100 ++++++++++++ std/ndn/svs/v3/definitions.go | 13 ++ std/ndn/svs/v3/zz_generated.go | 95 ++++++++++++ std/sync/svs.go | 131 ++++++++++++---- std/sync/svs_announce.go | 146 ++++++++++++++++++ std/sync/svs_announce_test.go | 143 +++++++++++++++++ std/sync/svs_debug.go | 89 +++++++++++ std/sync/svs_encode.go | 231 ++++++++++++++++++++++++++++ std/sync/svs_encode_test.go | 194 +++++++++++++++++++++++ std/sync/svs_mhash.go | 55 +++++++ std/sync/svs_mhash_test.go | 118 ++++++++++++++ std/sync/svs_pull.go | 116 ++++++++++++++ std/sync/svs_test.go | 188 ++++++++++++++++++++++ std/sync/svs_test_mesh.go | 157 +++++++++++++++++++ 15 files changed, 1829 insertions(+), 29 deletions(-) create mode 100644 std/engine/face/multicast_face.go create mode 100644 std/examples/svs/large-sync/main.go create mode 100644 std/sync/svs_announce.go create mode 100644 std/sync/svs_announce_test.go create mode 100644 std/sync/svs_debug.go create mode 100644 std/sync/svs_encode.go create mode 100644 std/sync/svs_encode_test.go create mode 100644 std/sync/svs_mhash.go create mode 100644 std/sync/svs_mhash_test.go create mode 100644 std/sync/svs_pull.go create mode 100644 std/sync/svs_test.go create mode 100644 std/sync/svs_test_mesh.go diff --git a/std/engine/face/multicast_face.go b/std/engine/face/multicast_face.go new file mode 100644 index 00000000..dbbe85f0 --- /dev/null +++ b/std/engine/face/multicast_face.go @@ -0,0 +1,82 @@ +package face + +import ( + "fmt" + "sync" + + enc "github.com/named-data/ndnd/std/encoding" +) + +// MulticastHub connects MulticastFace peers for in-process multicast-style tests. +type MulticastHub struct { + mu sync.Mutex + faces []*MulticastFace +} + +// MulticastFace delivers outgoing packets to every other face in the hub. +type MulticastFace struct { + baseFace + hub *MulticastHub + id int +} + +// NewMulticastHub creates a hub for test faces. +func NewMulticastHub() *MulticastHub { + return &MulticastHub{} +} + +// NewFace registers a new peer face on the hub. +func (h *MulticastHub) NewFace() *MulticastFace { + h.mu.Lock() + defer h.mu.Unlock() + f := &MulticastFace{ + baseFace: newBaseFace(true), + hub: h, + id: len(h.faces), + } + h.faces = append(h.faces, f) + return f +} + +func (f *MulticastFace) String() string { + return fmt.Sprintf("multicast-face-%d", f.id) +} + +func (f *MulticastFace) Open() error { + if f.onError == nil || f.onPkt == nil { + return fmt.Errorf("face callbacks are not set") + } + if f.running.Load() { + return fmt.Errorf("face is already running") + } + f.setStateUp() + return nil +} + +func (f *MulticastFace) Close() error { + f.setStateClosed() + return nil +} + +func (f *MulticastFace) Send(pkt enc.Wire) error { + if !f.running.Load() { + return fmt.Errorf("face is not running") + } + f.hub.broadcast(f.id, pkt.Join()) + return nil +} + +func (h *MulticastHub) broadcast(from int, frame []byte) { + h.mu.Lock() + peers := append([]*MulticastFace(nil), h.faces...) + h.mu.Unlock() + + for _, peer := range peers { + if peer.id == from || !peer.running.Load() || peer.onPkt == nil { + continue + } + frameCopy := make([]byte, len(frame)) + copy(frameCopy, frame) + peer.onPkt(frameCopy) + } +} diff --git a/std/examples/svs/large-sync/main.go b/std/examples/svs/large-sync/main.go new file mode 100644 index 00000000..a09da13f --- /dev/null +++ b/std/examples/svs/large-sync/main.go @@ -0,0 +1,100 @@ +package main + +import ( + "flag" + "fmt" + "os" + "time" + + enc "github.com/named-data/ndnd/std/encoding" + "github.com/named-data/ndnd/std/engine" + "github.com/named-data/ndnd/std/log" + "github.com/named-data/ndnd/std/ndn" + "github.com/named-data/ndnd/std/object" + "github.com/named-data/ndnd/std/object/storage" + "github.com/named-data/ndnd/std/sync" +) + +// Large-group SVS demo: PARTIAL on publication, announce+pull on periodic sync. +// +// Before running, configure NFD multicast on the sync prefix, e.g.: +// +// ndnd fw strategy-set prefix=/ndn/svs/32=svs strategy=/localhost/nfd/strategy/multicast +func main() { + threshold := flag.Int("threshold", 1200, "SyncVectorThreshold in bytes (0 = unlimited inline FULL)") + debug := flag.Bool("debug", false, "log SVS wire modes (PARTIAL, announce+pull, pull recovery)") + flag.Parse() + + if flag.NArg() < 1 { + fmt.Fprintf(os.Stderr, "Usage: %s [flags] \n", os.Args[0]) + os.Exit(1) + } + + if *debug { + log.Default().SetLevel(log.LevelDebug) + } + + name, err := enc.NameFromStr(flag.Arg(0)) + if err != nil { + log.Fatal(nil, "Invalid producer name", "name", flag.Arg(0), "err", err) + } + + app := engine.NewBasicEngine(engine.NewDefaultFace()) + if err := app.Start(); err != nil { + log.Fatal(nil, "Unable to start engine", "err", err) + } + defer app.Stop() + + store := storage.NewMemoryStore() + client := object.NewClient(app, store, nil) + if err := client.Start(); err != nil { + log.Fatal(nil, "Unable to start object client", "err", err) + } + defer client.Stop() + + group := tuName("/ndn/svs") + boot := uint64(time.Now().Unix()) + syncDataName := name. + Append(enc.NewTimestampComponent(boot)). + Append(enc.NewKeywordComponent("svs")) + + svsync := sync.NewSvSync(sync.SvSyncOpts{ + Client: client, + GroupPrefix: group, + SyncDataName: syncDataName, + BootTime: boot, + SyncVectorThreshold: *threshold, + OnUpdate: func(ssu sync.SvSyncUpdate) { + log.Info(nil, "SVS update", "name", ssu.Name, "boot", ssu.Boot, "low", ssu.Low, "high", ssu.High) + }, + }) + + client.AnnouncePrefix(ndn.Announcement{Name: group}) + defer client.WithdrawPrefix(group, nil) + + if err := svsync.Start(); err != nil { + log.Fatal(nil, "Unable to start SvSync", "err", err) + } + defer svsync.Stop() + + log.Info(nil, "Large-group SVS started", + "name", name, + "threshold", *threshold, + "debug", *debug, + "syncData", syncDataName, + "fullVector", syncDataName.Prefix(-1).Append(enc.NewKeywordComponent("sv"))) + + ticker := time.NewTicker(3 * time.Second) + for range ticker.C { + seq := svsync.IncrSeqNo(name) + log.Info(nil, "Published sequence number", "seq", seq) + } +} + +func tuName(s string) enc.Name { + name, err := enc.NameFromStr(s) + if err != nil { + panic(err) + } + return name +} diff --git a/std/ndn/svs/v3/definitions.go b/std/ndn/svs/v3/definitions.go index 3f7777bc..a3fcf415 100644 --- a/std/ndn/svs/v3/definitions.go +++ b/std/ndn/svs/v3/definitions.go @@ -3,9 +3,22 @@ package svs import ( enc "github.com/named-data/ndnd/std/encoding" + "github.com/named-data/ndnd/std/types/optional" +) + +// VectorType values for inline SvsData (TLV 0xCD). +const ( + VectorTypeFull uint64 = 0 + VectorTypePartial uint64 = 1 ) type SvsData struct { + //+field:binary:optional + MemberSetHash []byte `tlv:"0xcb"` + //+field:natural:optional + VectorType optional.Optional[uint64] `tlv:"0xcd"` + //+field:name + SvsDataRef enc.Name `tlv:"0x07"` //+field:struct:StateVector StateVector *StateVector `tlv:"0xc9"` } diff --git a/std/ndn/svs/v3/zz_generated.go b/std/ndn/svs/v3/zz_generated.go index 0fb2e7fc..138dbed3 100644 --- a/std/ndn/svs/v3/zz_generated.go +++ b/std/ndn/svs/v3/zz_generated.go @@ -11,6 +11,7 @@ import ( type SvsDataEncoder struct { Length uint + SvsDataRef_length uint StateVector_encoder StateVectorEncoder } @@ -19,11 +20,32 @@ type SvsDataParsingContext struct { } func (encoder *SvsDataEncoder) Init(value *SvsData) { + + if value.SvsDataRef != nil { + encoder.SvsDataRef_length = 0 + for _, c := range value.SvsDataRef { + encoder.SvsDataRef_length += uint(c.EncodingLength()) + } + } if value.StateVector != nil { encoder.StateVector_encoder.Init(value.StateVector) } l := uint(0) + if value.MemberSetHash != nil { + l += 1 + l += uint(enc.TLNum(len(value.MemberSetHash)).EncodingLength()) + l += uint(len(value.MemberSetHash)) + } + if optval, ok := value.VectorType.Get(); ok { + l += 1 + l += uint(1 + enc.Nat(optval).EncodingLength()) + } + if value.SvsDataRef != nil { + l += 1 + l += uint(enc.TLNum(encoder.SvsDataRef_length).EncodingLength()) + l += encoder.SvsDataRef_length + } if value.StateVector != nil { l += 1 l += uint(enc.TLNum(encoder.StateVector_encoder.Length).EncodingLength()) @@ -34,6 +56,7 @@ func (encoder *SvsDataEncoder) Init(value *SvsData) { } func (context *SvsDataParsingContext) Init() { + context.StateVector_context.Init() } @@ -41,6 +64,29 @@ func (encoder *SvsDataEncoder) EncodeInto(value *SvsData, buf []byte) { pos := uint(0) + if value.MemberSetHash != nil { + buf[pos] = byte(203) + pos += 1 + pos += uint(enc.TLNum(len(value.MemberSetHash)).EncodeInto(buf[pos:])) + copy(buf[pos:], value.MemberSetHash) + pos += uint(len(value.MemberSetHash)) + } + if optval, ok := value.VectorType.Get(); ok { + buf[pos] = byte(205) + pos += 1 + + buf[pos] = byte(enc.Nat(optval).EncodeInto(buf[pos+1:])) + pos += uint(1 + buf[pos]) + + } + if value.SvsDataRef != nil { + buf[pos] = byte(7) + pos += 1 + pos += uint(enc.TLNum(encoder.SvsDataRef_length).EncodeInto(buf[pos:])) + for _, c := range value.SvsDataRef { + pos += uint(c.EncodeInto(buf[pos:])) + } + } if value.StateVector != nil { buf[pos] = byte(201) pos += 1 @@ -64,6 +110,9 @@ func (encoder *SvsDataEncoder) Encode(value *SvsData) enc.Wire { func (context *SvsDataParsingContext) Parse(reader enc.WireView, ignoreCritical bool) (*SvsData, error) { + var handled_MemberSetHash bool = false + var handled_VectorType bool = false + var handled_SvsDataRef bool = false var handled_StateVector bool = false progress := -1 @@ -91,6 +140,43 @@ func (context *SvsDataParsingContext) Parse(reader enc.WireView, ignoreCritical err = nil if handled := false; true { switch typ { + case 203: + if true { + handled = true + handled_MemberSetHash = true + value.MemberSetHash = make([]byte, l) + _, err = reader.ReadFull(value.MemberSetHash) + } + case 205: + if true { + handled = true + handled_VectorType = true + { + optval := uint64(0) + optval = uint64(0) + { + for i := 0; i < int(l); i++ { + x := byte(0) + x, err = reader.ReadByte() + if err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + break + } + optval = uint64(optval<<8) | uint64(x) + } + } + value.VectorType.Set(optval) + } + } + case 7: + if true { + handled = true + handled_SvsDataRef = true + delegate := reader.Delegate(int(l)) + value.SvsDataRef, err = delegate.ReadName() + } case 201: if true { handled = true @@ -115,6 +201,15 @@ func (context *SvsDataParsingContext) Parse(reader enc.WireView, ignoreCritical startPos = reader.Pos() err = nil + if !handled_MemberSetHash && err == nil { + value.MemberSetHash = nil + } + if !handled_VectorType && err == nil { + value.VectorType.Unset() + } + if !handled_SvsDataRef && err == nil { + value.SvsDataRef = nil + } if !handled_StateVector && err == nil { value.StateVector = nil } diff --git a/std/sync/svs.go b/std/sync/svs.go index 60e71015..2b3cc582 100644 --- a/std/sync/svs.go +++ b/std/sync/svs.go @@ -40,6 +40,9 @@ type SvSync struct { // Channel for incoming state vectors recvSv chan svSyncRecvSvArgs + // Prefix for published full State Vector Data (.../32=sv). + fullVectorPrefix enc.Name + // cancellation for face hook faceCancel func() } @@ -73,6 +76,9 @@ type SvSyncOpts struct { UseSignatureTime optional.Optional[bool] // IgnoreValidity ignores validity period in the validation chain IgnoreValidity optional.Optional[bool] + + // SyncVectorThreshold is the max inline SvsData size (bytes). 0 = unlimited (legacy SVS v3). + SyncVectorThreshold int } type SvSyncUpdate struct { @@ -83,8 +89,11 @@ type SvSyncUpdate struct { } type svSyncRecvSvArgs struct { - sv *spec_svs.StateVector - data enc.Wire + sv *spec_svs.StateVector + data enc.Wire + vectorType optional.Optional[uint64] + mhash []byte + svsDataRef enc.Name } // NewSvSync creates a new SV Sync instance. @@ -145,6 +154,8 @@ func NewSvSync(opts SvSyncOpts) *SvSync { recvSv: make(chan svSyncRecvSvArgs, 128), + fullVectorPrefix: deriveFullVectorPrefix(opts.SyncDataName), + faceCancel: func() {}, } } @@ -180,7 +191,7 @@ func (s *SvSync) main() { // Notify everyone when we are back online s.faceCancel = s.o.Client.Engine().Face().OnUp(func() { - time.AfterFunc(100*time.Millisecond, s.sendSyncInterest) + time.AfterFunc(100*time.Millisecond, func() { s.sendSyncInterest(syncSendOther) }) }) defer s.faceCancel() @@ -190,7 +201,7 @@ func (s *SvSync) main() { go s.loadPassiveWires() } else { // Send the initial Sync Interest - go s.sendSyncInterest() + go s.sendSyncInterest(syncSendOther) } for { @@ -242,7 +253,7 @@ func (s *SvSync) SetSeqNo(name enc.Name, seqNo uint64) error { // [Spec] When the node generates a new publication, // immediately emit a Sync Interest s.state.Set(hash, s.o.BootTime, seqNo) - go s.sendSyncInterest() + go s.sendSyncInterest(syncSendPublication, name) return nil } @@ -264,7 +275,7 @@ func (s *SvSync) IncrSeqNo(name enc.Name) uint64 { // [Spec] When the node generates a new publication, // immediately emit a Sync Interest - go s.sendSyncInterest() + go s.sendSyncInterest(syncSendPublication, name) return entry } @@ -372,7 +383,14 @@ func (s *SvSync) onReceiveStateVector(args svSyncRecvSvArgs) { // The above checks each node in the incoming state vector, but // does not check if a node is missing from the incoming state vector. - if !isOutdated && s.state.IsNewerThan(recvSv, func(_, _ uint64) bool { return false }) { + isPartial := false + if vt, ok := args.vectorType.Get(); ok && vt == spec_svs.VectorTypePartial { + isPartial = true + } + if len(args.mhash) > 0 { + s.handleMhashMismatch(args, recvSv) + } + if !isPartial && !isOutdated && s.state.IsNewerThan(recvSv, func(_, _ uint64) bool { return false }) { isOutdated = true canDrop = false } @@ -420,11 +438,11 @@ func (s *SvSync) timerExpired() { // [Spec] On expiration of timer emit a Sync Interest // with the current local state vector. - go s.sendSyncInterest() + go s.sendSyncInterest(syncSendPeriodic) } // (AI GENERATED DESCRIPTION): Sends a sync Interest: if passive mode is enabled, it publishes all buffered state updates without duplicates; otherwise, it encodes the current state vector into a wire and transmits it, provided the sync service is running. -func (s *SvSync) sendSyncInterest() { +func (s *SvSync) sendSyncInterest(reason syncSendReason, pubName ...enc.Name) { if !s.running.Load() { return } @@ -437,8 +455,14 @@ func (s *SvSync) sendSyncInterest() { return } + var sender enc.Name + if reason == syncSendPublication && len(pubName) > 0 { + sender = pubName[0] + } + // Encode and sign the current state vector - wire := s.encodeSyncData() + log.Debug(s, "sync interest", "reason", syncSendReasonString(reason)) + wire := s.encodeSyncData(reason, sender) s.sendSyncInterestWith(wire) } @@ -466,18 +490,35 @@ func (s *SvSync) sendSyncInterestWith(dataWire enc.Wire) { } // (AI GENERATED DESCRIPTION): Builds a signed Data packet containing the current state vector for SVS v3 synchronization. -func (s *SvSync) encodeSyncData() enc.Wire { - // Critical section - sv := func() *spec_svs.StateVector { - s.mutex.Lock() - defer s.mutex.Unlock() - - // [Spec*] Sending always triggers Steady State - s.enterSteadyState() - - return s.state.Encode(func(s uint64) uint64 { return s }) - }() - svWire := (&spec_svs.SvsData{StateVector: sv}).Encode() +func (s *SvSync) encodeSyncData(reason syncSendReason, sender enc.Name) enc.Wire { + s.mutex.Lock() + s.enterSteadyState() + stateSnap := s.state + mtimeSnap := s.mtime + repair, propagation := s.partialTargets() + s.mutex.Unlock() + + var svsData *spec_svs.SvsData + if shouldUseAnnouncePull(reason, s.o.SyncVectorThreshold, stateSnap) { + ref, err := s.publishFullVectorData(stateSnap) + if err != nil { + log.Error(s, "publishFullVectorData failed", "err", err) + return nil + } + svsData = buildAnnounceSvsData(stateSnap, ref) + } else { + svsData = buildSvsDataForSend( + stateSnap, + reason, + s.o.SyncVectorThreshold, + sender, + repair, + propagation, + mtimeSnap, + ) + } + logSyncSend(s, reason, svsData) + svWire := svsData.Encode() // SVS v3 Sync Data name := s.o.SyncDataName.WithVersion(enc.VersionUnixMicro) @@ -536,18 +577,39 @@ func (s *SvSync) onSyncData(dataWire enc.Wire) { return } - // Decode state vector + // Decode SvsData (inline FULL, PARTIAL, announce-only, or legacy v3). svWire := data.Content().Join() params, err := spec_svs.ParseSvsData(enc.NewBufferView(svWire), false) - if err != nil || params.StateVector == nil { - log.Warn(s, "onSyncInterest failed to parse StateVec", "err", err) + if err != nil { + log.Warn(s, "onSyncInterest failed to parse SvsData", "err", err) + return + } + + // Announce-only: retrievable full vector reference. + if params.StateVector == nil && len(params.SvsDataRef) > 0 { + logSyncRecv(s, params, data.Name()) + trustPrefix := pullRefFromSyncDataWire(dataWire) + go s.pullFullVector(params.SvsDataRef, trustPrefix) return } + if params.StateVector == nil { + log.Warn(s, "onSyncInterest SvsData has no StateVector") + return + } + + logSyncRecv(s, params, data.Name()) - s.recvSv <- svSyncRecvSvArgs{ - sv: params.StateVector, - data: dataWire, + args := svSyncRecvSvArgs{ + sv: params.StateVector, + data: dataWire, + mhash: params.MemberSetHash, + svsDataRef: params.SvsDataRef, } + if vt, ok := params.VectorType.Get(); ok { + args.vectorType = optional.Some(vt) + } + + s.recvSv <- args }, }) } @@ -664,5 +726,16 @@ func (s *SvSync) loadPassiveWires() { } // This is hacky but pragmatic - wait for the state to be processed - time.AfterFunc(500*time.Millisecond, s.sendSyncInterest) + time.AfterFunc(500*time.Millisecond, func() { s.sendSyncInterest(syncSendOther) }) +} + +// partialTargets returns repair and propagation name targets from suppression merge state. +func (s *SvSync) partialTargets() (repair, propagation []enc.Name) { + if !s.suppress { + return nil, nil + } + for name := range s.merge.Iter() { + repair = append(repair, name) + } + return repair, nil } diff --git a/std/sync/svs_announce.go b/std/sync/svs_announce.go new file mode 100644 index 00000000..c7cd34ce --- /dev/null +++ b/std/sync/svs_announce.go @@ -0,0 +1,146 @@ +package sync + +import ( + "bytes" + "fmt" + + enc "github.com/named-data/ndnd/std/encoding" + spec "github.com/named-data/ndnd/std/ndn/spec_2022" + spec_svs "github.com/named-data/ndnd/std/ndn/svs/v3" +) + +const ( + syncDataKeyword = "svs" + fullVectorKeyword = "sv" +) + +// deriveFullVectorPrefix maps SyncDataName (.../32=svs) to the published full-vector prefix (.../32=sv). +func deriveFullVectorPrefix(syncDataName enc.Name) enc.Name { + if len(syncDataName) == 0 { + return nil + } + base := syncDataName + if base.At(-1).IsKeyword(syncDataKeyword) { + base = base.Prefix(-1) + } + return base.Append(enc.NewKeywordComponent(fullVectorKeyword)) +} + +// pullRefFromSyncDataWire derives the retrievable full-vector prefix from a signed Sync Data wire. +func pullRefFromSyncDataWire(dataWire enc.Wire) enc.Name { + name, err := syncDataNameFromWire(dataWire) + if err != nil || len(name) == 0 { + return nil + } + if name.At(-1).IsVersion() { + name = name.Prefix(-1) + } + return deriveFullVectorPrefix(name) +} + +func syncDataNameFromWire(dataWire enc.Wire) (enc.Name, error) { + data, _, err := spec.Spec{}.ReadData(enc.NewWireView(dataWire)) + if err != nil { + return nil, err + } + return data.Name(), nil +} + +// buildAnnounceSvsData builds announce-only SvsData (mhash + SvsDataRef). +func buildAnnounceSvsData(state SvMap[uint64], ref enc.Name) *spec_svs.SvsData { + return &spec_svs.SvsData{ + MemberSetHash: ComputeMhash(state), + SvsDataRef: ref, + } +} + +// buildInlineFullFromState builds inline FULL SvsData for the complete local state. +func buildInlineFullFromState(state SvMap[uint64]) *spec_svs.SvsData { + sv := state.Encode(func(seq uint64) uint64 { return seq }) + return buildInlineSvsData(state, spec_svs.VectorTypeFull, sv) +} + +// inlineFullSize returns encoded inline FULL SvsData size for threshold checks. +func inlineFullSize(state SvMap[uint64]) int { + return inlineSvsDataSize(buildInlineFullFromState(state)) +} + +// shouldUseAnnouncePull reports whether the sender should publish at 32=sv and announce-only Sync Data. +func shouldUseAnnouncePull(reason syncSendReason, threshold int, state SvMap[uint64]) bool { + if reason == syncSendRecovery { + return true + } + if reason == syncSendPublication || threshold <= 0 { + return false + } + return exceedsSyncThreshold(threshold, inlineFullSize(state)) +} + +// isTrustedSvsDataRef reports whether ref is under the sender's published full-vector prefix. +func isTrustedSvsDataRef(ref, senderFullVectorPrefix enc.Name) bool { + if len(ref) == 0 || len(senderFullVectorPrefix) == 0 { + return false + } + return senderFullVectorPrefix.IsPrefix(ref) +} + +// membershipTupleCount returns the number of (Name, BootstrapTime) pairs in state. +func membershipTupleCount(m SvMap[uint64]) int { + n := 0 + for _, vals := range m { + n += len(vals) + } + return n +} + +// membershipContains reports whether every membership tuple in inner exists in outer. +func membershipContains(outer, inner SvMap[uint64]) bool { + for hash, vals := range inner { + for _, v := range vals { + found := false + for _, ov := range outer[hash] { + if ov.Boot == v.Boot { + found = true + break + } + } + if !found { + return false + } + } + } + return true +} + +// stateVectorToMap converts a StateVector to a map for membership checks. +func stateVectorToMap(sv *spec_svs.StateVector) SvMap[uint64] { + m := NewSvMap[uint64](len(sv.Entries)) + for _, node := range sv.Entries { + hash := node.Name.TlvStr() + for _, entry := range node.SeqNoEntries { + m.Set(hash, entry.BootstrapTime, entry.SeqNo) + } + } + return m +} + +// parseFullVectorContent parses inline FULL SvsData from published full-vector Data content. +func parseFullVectorContent(content []byte) (*spec_svs.SvsData, error) { + params, err := spec_svs.ParseSvsData(enc.NewBufferView(content), false) + if err != nil { + return nil, err + } + if params.StateVector == nil { + return nil, fmt.Errorf("full vector content has no StateVector") + } + if vt, ok := params.VectorType.Get(); ok && vt != spec_svs.VectorTypeFull { + return nil, fmt.Errorf("full vector VectorType=%d, want FULL", vt) + } + if len(params.MemberSetHash) > 0 { + computed := ComputeMhash(stateVectorToMap(params.StateVector)) + if !bytes.Equal(params.MemberSetHash, computed) { + return nil, fmt.Errorf("full vector mhash mismatch") + } + } + return params, nil +} diff --git a/std/sync/svs_announce_test.go b/std/sync/svs_announce_test.go new file mode 100644 index 00000000..1a175310 --- /dev/null +++ b/std/sync/svs_announce_test.go @@ -0,0 +1,143 @@ +package sync + +import ( + "testing" + "time" + + enc "github.com/named-data/ndnd/std/encoding" + spec_svs "github.com/named-data/ndnd/std/ndn/svs/v3" + tu "github.com/named-data/ndnd/std/utils/testutils" + "github.com/stretchr/testify/require" +) + +func TestDeriveFullVectorPrefix(t *testing.T) { + tu.SetT(t) + + syncData := tu.NoErr(enc.NameFromStr("/ndn/svs/alice/1700000000/32=svs")) + prefix := deriveFullVectorPrefix(syncData) + require.Equal(t, "/ndn/svs/alice/1700000000/32=sv", prefix.String()) +} + +func TestPullRefFromSyncDataWire(t *testing.T) { + tu.SetT(t) + + syncDataName := tu.NoErr(enc.NameFromStr("/ndn/svs/alice/1700000000/32=svs/12345")) + ref := deriveFullVectorPrefix(syncDataName.Prefix(-1)) + require.Equal(t, "/ndn/svs/alice/1700000000/32=sv", ref.String()) +} + +func TestBuildAnnounceSvsData(t *testing.T) { + tu.SetT(t) + + m := testSvMapAliceBob() + ref := tu.NoErr(enc.NameFromStr("/ndn/svs/alice/1700000000/32=sv/999")) + data := buildAnnounceSvsData(m, ref) + + require.Equal(t, ComputeMhash(m), data.MemberSetHash) + require.True(t, ref.Equal(data.SvsDataRef)) + require.Nil(t, data.StateVector) + require.False(t, data.VectorType.IsSet()) + + wire := data.Encode().Join() + parsed, err := spec_svs.ParseSvsData(enc.NewBufferView(wire), false) + require.NoError(t, err) + require.Equal(t, data.MemberSetHash, parsed.MemberSetHash) + require.True(t, ref.Equal(parsed.SvsDataRef)) + require.Nil(t, parsed.StateVector) +} + +func TestShouldUseAnnouncePull(t *testing.T) { + tu.SetT(t) + + m := testSvMapAliceBob() + fullSize := inlineFullSize(m) + + require.False(t, shouldUseAnnouncePull(syncSendPublication, fullSize-1, m)) + require.False(t, shouldUseAnnouncePull(syncSendPeriodic, 0, m)) + require.False(t, shouldUseAnnouncePull(syncSendPeriodic, fullSize+1, m)) + require.True(t, shouldUseAnnouncePull(syncSendPeriodic, fullSize-1, m)) + require.True(t, shouldUseAnnouncePull(syncSendOther, fullSize-1, m)) + require.True(t, shouldUseAnnouncePull(syncSendRecovery, 0, m)) +} + +func TestIsTrustedSvsDataRef(t *testing.T) { + tu.SetT(t) + + trust := tu.NoErr(enc.NameFromStr("/ndn/svs/alice/1/32=sv")) + ref := tu.NoErr(enc.NameFromStr("/ndn/svs/alice/1/32=sv/999")) + bad := tu.NoErr(enc.NameFromStr("/ndn/evil/32=sv/1")) + + require.True(t, isTrustedSvsDataRef(ref, trust)) + require.True(t, isTrustedSvsDataRef(trust, trust)) + require.False(t, isTrustedSvsDataRef(bad, trust)) + require.False(t, isTrustedSvsDataRef(ref, nil)) +} + +func TestParseFullVectorContentRejectsBadMhash(t *testing.T) { + tu.SetT(t) + + m := testSvMapAliceBob() + inline := buildInlineFullFromState(m) + inline.MemberSetHash = []byte("not-a-valid-mhash-padding-000000") + wire := inline.Encode().Join() + + _, err := parseFullVectorContent(wire) + require.Error(t, err) +} + +func TestParseFullVectorContent(t *testing.T) { + tu.SetT(t) + + m := testSvMapAliceBob() + inline := buildInlineFullFromState(m) + wire := inline.Encode().Join() + + parsed, err := parseFullVectorContent(wire) + require.NoError(t, err) + require.Equal(t, inline.MemberSetHash, parsed.MemberSetHash) + require.Len(t, parsed.StateVector.Entries, 2) +} + +func TestOnPulledFullVectorMergesState(t *testing.T) { + tu.SetT(t) + + var updates []SvSyncUpdate + s := &SvSync{ + o: SvSyncOpts{ + OnUpdate: func(u SvSyncUpdate) { updates = append(updates, u) }, + PeriodicTimeout: 30 * time.Second, + }, + state: NewSvMap[uint64](0), + mtime: make(map[string]time.Time), + ticker: time.NewTicker(30 * time.Second), + recvSv: make(chan svSyncRecvSvArgs, 1), + } + + alice := tu.NoErr(enc.NameFromStr("/ndn/alice")) + s.state.Set(alice.TlvStr(), 100, 1) + + remote := testSvMapAliceBob() + content := buildInlineFullFromState(remote).Encode().Join() + + go func() { + s.onPulledFullVector(content) + }() + + s.onReceiveStateVector(<-s.recvSv) + + require.Len(t, updates, 2) + require.EqualValues(t, 3, s.state.Get(tu.NoErr(enc.NameFromStr("/ndn/bob")).TlvStr(), 150)) + require.EqualValues(t, 5, s.state.Get(alice.TlvStr(), 100)) +} + +func TestEncodeSyncDataAnnounceMode(t *testing.T) { + tu.SetT(t) + + m := testSvMapAliceBob() + require.True(t, shouldUseAnnouncePull(syncSendPeriodic, inlineFullSize(m)-1, m)) + + announce := buildAnnounceSvsData(m, tu.NoErr(enc.NameFromStr("/ndn/svs/alice/1/32=sv/2"))) + require.Nil(t, announce.StateVector) + vt, ok := announce.VectorType.Get() + require.False(t, ok || vt == spec_svs.VectorTypePartial) +} diff --git a/std/sync/svs_debug.go b/std/sync/svs_debug.go new file mode 100644 index 00000000..00053df1 --- /dev/null +++ b/std/sync/svs_debug.go @@ -0,0 +1,89 @@ +package sync + +import ( + enc "github.com/named-data/ndnd/std/encoding" + "github.com/named-data/ndnd/std/log" + spec_svs "github.com/named-data/ndnd/std/ndn/svs/v3" +) + +func syncSendReasonString(reason syncSendReason) string { + switch reason { + case syncSendPublication: + return "publication" + case syncSendPeriodic: + return "periodic" + case syncSendRecovery: + return "recovery" + default: + return "other" + } +} + +func logSyncSend(s *SvSync, reason syncSendReason, data *spec_svs.SvsData) { + if data == nil { + return + } + if data.StateVector == nil && len(data.SvsDataRef) > 0 { + log.Debug(s, "sync send announce-only", + "reason", syncSendReasonString(reason), + "ref", data.SvsDataRef, + "bytes", inlineSvsDataSize(data), + ) + return + } + + mode := syncWireModeLabel(data) + entries := 0 + if data.StateVector != nil { + entries = len(data.StateVector.Entries) + } + log.Debug(s, "sync send inline", + "reason", syncSendReasonString(reason), + "mode", mode, + "entries", entries, + "bytes", inlineSvsDataSize(data), + ) +} + +func logSyncRecv(s *SvSync, params *spec_svs.SvsData, syncDataName enc.Name) { + if params == nil { + return + } + if params.StateVector == nil && len(params.SvsDataRef) > 0 { + log.Debug(s, "sync recv announce-only", + "from", syncDataName, + "ref", params.SvsDataRef, + ) + return + } + + mode := syncWireModeLabel(params) + entries := 0 + if params.StateVector != nil { + entries = len(params.StateVector.Entries) + } + log.Debug(s, "sync recv inline", + "from", syncDataName, + "mode", mode, + "entries", entries, + ) +} + +func syncWireModeLabel(data *spec_svs.SvsData) string { + if data.StateVector == nil { + if len(data.SvsDataRef) > 0 { + return "announce-only" + } + return "empty" + } + if vt, ok := data.VectorType.Get(); ok { + if vt == spec_svs.VectorTypePartial { + return "PARTIAL" + } + return "FULL" + } + if len(data.MemberSetHash) == 0 { + return "legacy FULL" + } + return "FULL" +} diff --git a/std/sync/svs_encode.go b/std/sync/svs_encode.go new file mode 100644 index 00000000..1201fe14 --- /dev/null +++ b/std/sync/svs_encode.go @@ -0,0 +1,231 @@ +package sync + +import ( + "cmp" + "math/rand/v2" + "slices" + "time" + + enc "github.com/named-data/ndnd/std/encoding" + spec_svs "github.com/named-data/ndnd/std/ndn/svs/v3" + "github.com/named-data/ndnd/std/types/optional" +) + +// syncSendReason distinguishes why a Sync Interest is being sent. +type syncSendReason int + +const ( + syncSendOther syncSendReason = iota + syncSendPublication + syncSendPeriodic + syncSendRecovery +) + +// PartialEncodeOpts configures subset selection for inline PARTIAL vectors. +type PartialEncodeOpts struct { + Sender enc.Name + Threshold int + Repair []enc.Name + Propagation []enc.Name + Mtime map[string]time.Time +} + +// buildInlineSvsData constructs inline SvsData (mhash + VectorType + StateVector). +func buildInlineSvsData(state SvMap[uint64], vectorType uint64, sv *spec_svs.StateVector) *spec_svs.SvsData { + return &spec_svs.SvsData{ + MemberSetHash: ComputeMhash(state), + VectorType: optional.Some(vectorType), + StateVector: sv, + } +} + +// inlineSvsDataSize returns the encoded byte length of SvsData content. +func inlineSvsDataSize(data *spec_svs.SvsData) int { + return len(data.Encode().Join()) +} + +// exceedsSyncThreshold reports whether size is over the configured inline budget. +// threshold 0 means unlimited (always inline). +func exceedsSyncThreshold(threshold, size int) bool { + return threshold > 0 && size > threshold +} + +// buildSvsDataForSend picks inline FULL or PARTIAL SvsData for an outgoing Sync message. +func buildSvsDataForSend( + state SvMap[uint64], + reason syncSendReason, + threshold int, + sender enc.Name, + repair, propagation []enc.Name, + mtime map[string]time.Time, +) *spec_svs.SvsData { + fullSv := state.Encode(func(seq uint64) uint64 { return seq }) + fullData := buildInlineSvsData(state, spec_svs.VectorTypeFull, fullSv) + + if reason != syncSendPublication || !exceedsSyncThreshold(threshold, inlineSvsDataSize(fullData)) { + return fullData + } + + partialSv := encodePartialStateVector(state, PartialEncodeOpts{ + Sender: sender, + Threshold: threshold, + Repair: repair, + Propagation: propagation, + Mtime: mtime, + }) + return buildInlineSvsData(state, spec_svs.VectorTypePartial, partialSv) +} + +// encodePartialStateVector builds a PARTIAL StateVector per spec Section 4.2. +// Entry [0] is the sender; entries [1..n] are in NDN canonical order. +func encodePartialStateVector(state SvMap[uint64], opts PartialEncodeOpts) *spec_svs.StateVector { + seq := func(v uint64) uint64 { return v } + senderHash := opts.Sender.TlvStr() + + senderEntry := state.encodeNameEntry(opts.Sender, seq) + if senderEntry == nil { + senderEntry = &spec_svs.StateVectorEntry{Name: opts.Sender} + } + + if opts.Threshold <= 0 { + return &spec_svs.StateVector{Entries: []*spec_svs.StateVectorEntry{senderEntry}} + } + + // Sender-only baseline must always fit when possible. + baseline := &spec_svs.StateVector{Entries: []*spec_svs.StateVectorEntry{senderEntry}} + if inlineSvsDataSize(buildInlineSvsData(state, spec_svs.VectorTypePartial, baseline)) > opts.Threshold { + return baseline + } + + candidates := partialCandidateNames(state, senderHash, opts) + included := map[string]bool{senderHash: true} + entries := []*spec_svs.StateVectorEntry{senderEntry} + + for _, name := range candidates { + hash := name.TlvStr() + if included[hash] { + continue + } + entry := state.encodeNameEntry(name, seq) + if entry == nil { + continue + } + + trial := append(slices.Clone(entries), entry) + sortPartialTail(trial) + trialSv := &spec_svs.StateVector{Entries: trial} + if exceedsSyncThreshold(opts.Threshold, inlineSvsDataSize(buildInlineSvsData(state, spec_svs.VectorTypePartial, trialSv))) { + break + } + + entries = trial + included[hash] = true + } + + sortPartialTail(entries) + return &spec_svs.StateVector{Entries: entries} +} + +func partialCandidateNames(state SvMap[uint64], senderHash string, opts PartialEncodeOpts) []enc.Name { + seen := map[string]bool{senderHash: true} + out := make([]enc.Name, 0, len(state)) + + appendUnique := func(names []enc.Name) { + for _, name := range names { + hash := name.TlvStr() + if seen[hash] { + continue + } + if _, ok := state[hash]; !ok { + continue + } + seen[hash] = true + out = append(out, name) + } + } + + appendUnique(opts.Repair) + appendUnique(opts.Propagation) + + inactive := make([]enc.Name, 0) + remaining := make([]enc.Name, 0) + for name, vals := range state.Iter() { + hash := name.TlvStr() + if seen[hash] { + continue + } + if isInactiveProducer(vals) { + inactive = append(inactive, name) + continue + } + remaining = append(remaining, name) + } + + rand.Shuffle(len(inactive), func(i, j int) { + inactive[i], inactive[j] = inactive[j], inactive[i] + }) + appendUnique(inactive) + + slices.SortFunc(remaining, func(a, b enc.Name) int { + return a.Compare(b) + }) + slices.SortFunc(remaining, func(a, b enc.Name) int { + return cmp.Compare(recencyScore(opts.Mtime, b), recencyScore(opts.Mtime, a)) + }) + appendUnique(remaining) + + return out +} + +func isInactiveProducer(vals []SvMapVal[uint64]) bool { + for _, val := range vals { + if val.Value > 0 { + return false + } + } + return true +} + +func recencyScore(mtime map[string]time.Time, name enc.Name) int64 { + if mtime == nil { + return 0 + } + t, ok := mtime[name.TlvStr()] + if !ok { + return 0 + } + return t.UnixNano() +} + +// sortPartialTail keeps entry [0] fixed and sorts [1..n] in canonical name order. +func sortPartialTail(entries []*spec_svs.StateVectorEntry) { + if len(entries) <= 1 { + return + } + slices.SortFunc(entries[1:], func(a, b *spec_svs.StateVectorEntry) int { + return a.Name.Compare(b.Name) + }) +} + +// encodeNameEntry encodes one producer name from the map. +func (m SvMap[V]) encodeNameEntry(name enc.Name, seq func(V) uint64) *spec_svs.StateVectorEntry { + hash := name.TlvStr() + vals, ok := m[hash] + if !ok { + return nil + } + + entry := &spec_svs.StateVectorEntry{ + Name: name, + SeqNoEntries: make([]*spec_svs.SeqNoEntry, 0, len(vals)), + } + for _, val := range vals { + if seqNo := seq(val.Value); seqNo > 0 { + entry.SeqNoEntries = append(entry.SeqNoEntries, &spec_svs.SeqNoEntry{ + BootstrapTime: val.Boot, + SeqNo: seqNo, + }) + } + } + return entry +} diff --git a/std/sync/svs_encode_test.go b/std/sync/svs_encode_test.go new file mode 100644 index 00000000..accf9f3c --- /dev/null +++ b/std/sync/svs_encode_test.go @@ -0,0 +1,194 @@ +package sync + +import ( + "fmt" + "testing" + "time" + + enc "github.com/named-data/ndnd/std/encoding" + spec_svs "github.com/named-data/ndnd/std/ndn/svs/v3" + "github.com/named-data/ndnd/std/types/optional" + tu "github.com/named-data/ndnd/std/utils/testutils" + "github.com/stretchr/testify/require" +) + +func testSvMapAliceBob() SvMap[uint64] { + m := NewSvMap[uint64](0) + m.Set(tu.NoErr(enc.NameFromStr("/ndn/alice")).TlvStr(), 100, 5) + m.Set(tu.NoErr(enc.NameFromStr("/ndn/bob")).TlvStr(), 150, 3) + return m +} + +func TestBuildInlineFullSvsData(t *testing.T) { + tu.SetT(t) + + m := testSvMapAliceBob() + sv := m.Encode(func(s uint64) uint64 { return s }) + data := buildInlineSvsData(m, spec_svs.VectorTypeFull, sv) + + require.Equal(t, ComputeMhash(m), data.MemberSetHash) + vt, ok := data.VectorType.Get() + require.True(t, ok) + require.Equal(t, spec_svs.VectorTypeFull, vt) + require.NotNil(t, data.StateVector) + require.Len(t, data.StateVector.Entries, 2) +} + +func TestExceedsSyncThreshold(t *testing.T) { + tu.SetT(t) + + require.False(t, exceedsSyncThreshold(0, 99999)) + require.False(t, exceedsSyncThreshold(1000, 500)) + require.True(t, exceedsSyncThreshold(1000, 1001)) +} + +func TestOnReceivePartialSkipsMissingNameOutdated(t *testing.T) { + tu.SetT(t) + + s := &SvSync{ + o: SvSyncOpts{ + OnUpdate: func(SvSyncUpdate) {}, + SuppressionPeriod: 200 * time.Millisecond, + PeriodicTimeout: 30 * time.Second, + }, + state: testSvMapAliceBob(), + mtime: make(map[string]time.Time), + ticker: time.NewTicker(30 * time.Second), + suppress: false, + } + + // PARTIAL with only bob; local knows alice — must not enter suppression. + bobOnly := NewSvMap[uint64](0) + bobOnly.Set(tu.NoErr(enc.NameFromStr("/ndn/bob")).TlvStr(), 150, 3) + partialSv := bobOnly.Encode(func(s uint64) uint64 { return s }) + + s.onReceiveStateVector(svSyncRecvSvArgs{ + sv: partialSv, + vectorType: optional.Some(spec_svs.VectorTypePartial), + mhash: ComputeMhash(bobOnly), + }) + + require.False(t, s.suppress) +} + +func TestOnReceiveFullTreatsMissingNameOutdated(t *testing.T) { + tu.SetT(t) + + s := &SvSync{ + o: SvSyncOpts{ + OnUpdate: func(SvSyncUpdate) {}, + SuppressionPeriod: 200 * time.Millisecond, + PeriodicTimeout: 30 * time.Second, + }, + state: testSvMapAliceBob(), + mtime: make(map[string]time.Time), + ticker: time.NewTicker(30 * time.Second), + suppress: false, + } + + bobOnly := NewSvMap[uint64](0) + bobOnly.Set(tu.NoErr(enc.NameFromStr("/ndn/bob")).TlvStr(), 150, 3) + fullSv := bobOnly.Encode(func(s uint64) uint64 { return s }) + + s.onReceiveStateVector(svSyncRecvSvArgs{ + sv: fullSv, + vectorType: optional.Some(spec_svs.VectorTypeFull), + mhash: ComputeMhash(bobOnly), + }) + + require.True(t, s.suppress) +} + +func TestEncodePartialSenderFirst(t *testing.T) { + tu.SetT(t) + + alice := tu.NoErr(enc.NameFromStr("/ndn/alice")) + bob := tu.NoErr(enc.NameFromStr("/ndn/bob")) + carol := tu.NoErr(enc.NameFromStr("/ndn/carol")) + + m := NewSvMap[uint64](0) + m.Set(alice.TlvStr(), 100, 5) + m.Set(bob.TlvStr(), 150, 3) + m.Set(carol.TlvStr(), 150, 7) + + // Threshold large enough for sender + one peer. + full := buildInlineSvsData(m, spec_svs.VectorTypeFull, m.Encode(func(s uint64) uint64 { return s })) + threshold := inlineSvsDataSize(full) - 1 + + partial := encodePartialStateVector(m, PartialEncodeOpts{ + Sender: carol, + Threshold: threshold, + Mtime: map[string]time.Time{ + alice.TlvStr(): time.Unix(10, 0), + bob.TlvStr(): time.Unix(20, 0), + }, + }) + + require.NotEmpty(t, partial.Entries) + require.Equal(t, carol, partial.Entries[0].Name) + if len(partial.Entries) > 2 { + require.Less(t, partial.Entries[1].Name.Compare(partial.Entries[2].Name), 0) + } +} + +func TestBuildSvsDataForSendPublicationPartial(t *testing.T) { + tu.SetT(t) + + alice := tu.NoErr(enc.NameFromStr("/ndn/alice")) + m := NewSvMap[uint64](0) + m.Set(alice.TlvStr(), 100, 5) + for i := range 20 { + name := tu.NoErr(enc.NameFromStr(fmt.Sprintf("/ndn/peer%d", i))) + m.Set(name.TlvStr(), 150, uint64(i+1)) + } + + full := buildInlineSvsData(m, spec_svs.VectorTypeFull, m.Encode(func(s uint64) uint64 { return s })) + threshold := inlineSvsDataSize(full) / 2 + + pub := buildSvsDataForSend(m, syncSendPublication, threshold, alice, nil, nil, nil) + vt, ok := pub.VectorType.Get() + require.True(t, ok) + require.Equal(t, spec_svs.VectorTypePartial, vt) + require.Less(t, len(pub.StateVector.Entries), len(full.StateVector.Entries)) + + periodic := buildSvsDataForSend(m, syncSendPeriodic, threshold, alice, nil, nil, nil) + vt, ok = periodic.VectorType.Get() + require.True(t, ok) + require.Equal(t, spec_svs.VectorTypeFull, vt) +} + +func TestOnReceivePartialMergesPresentEntriesOnly(t *testing.T) { + tu.SetT(t) + + var updates []SvSyncUpdate + s := &SvSync{ + o: SvSyncOpts{ + OnUpdate: func(u SvSyncUpdate) { updates = append(updates, u) }, + PeriodicTimeout: 30 * time.Second, + }, + state: NewSvMap[uint64](0), + mtime: make(map[string]time.Time), + ticker: time.NewTicker(30 * time.Second), + } + + alice := tu.NoErr(enc.NameFromStr("/ndn/alice")) + bob := tu.NoErr(enc.NameFromStr("/ndn/bob")) + s.state.Set(alice.TlvStr(), 100, 1) + s.state.Set(bob.TlvStr(), 150, 1) + + bobOnly := NewSvMap[uint64](0) + bobOnly.Set(bob.TlvStr(), 150, 4) + partialSv := bobOnly.Encode(func(s uint64) uint64 { return s }) + + s.onReceiveStateVector(svSyncRecvSvArgs{ + sv: partialSv, + vectorType: optional.Some(spec_svs.VectorTypePartial), + mhash: ComputeMhash(bobOnly), + }) + + require.Len(t, updates, 1) + require.Equal(t, bob, updates[0].Name) + require.EqualValues(t, 4, updates[0].High) + require.EqualValues(t, 1, s.state.Get(alice.TlvStr(), 100)) + require.EqualValues(t, 4, s.state.Get(bob.TlvStr(), 150)) +} diff --git a/std/sync/svs_mhash.go b/std/sync/svs_mhash.go new file mode 100644 index 00000000..74b9a445 --- /dev/null +++ b/std/sync/svs_mhash.go @@ -0,0 +1,55 @@ +package sync + +import ( + "crypto/sha256" + "slices" + + enc "github.com/named-data/ndnd/std/encoding" +) + +// membershipTuple is one (Name, BootstrapTime) pair in the sync group. +type membershipTuple struct { + name enc.Name + boot uint64 +} + +// ComputeMhash returns the membership hash over all (Name, BootstrapTime) pairs +// in state, per SVS v3 large-group revision spec Section 3.3. +func ComputeMhash(state SvMap[uint64]) []byte { + members := make([]membershipTuple, 0) + for name, vals := range state.Iter() { + for _, val := range vals { + members = append(members, membershipTuple{name: name, boot: val.Boot}) + } + } + + slices.SortFunc(members, func(a, b membershipTuple) int { + if c := a.name.Compare(b.name); c != 0 { + return c + } + if a.boot < b.boot { + return -1 + } + if a.boot > b.boot { + return 1 + } + return 0 + }) + + h := sha256.New() + for _, m := range members { + h.Write(m.name.Bytes()) + h.Write(bootstrapTimeTLV(m.boot)) + } + return h.Sum(nil) +} + +// bootstrapTimeTLV encodes BOOTSTRAP-TIME-TYPE (0xD4) matching SVS v3 SeqNoEntry layout. +func bootstrapTimeTLV(boot uint64) []byte { + valLen := enc.Nat(boot).EncodingLength() + buf := make([]byte, 2+valLen) + buf[0] = 0xD4 + buf[1] = byte(valLen) + enc.Nat(boot).EncodeInto(buf[2:]) + return buf +} diff --git a/std/sync/svs_mhash_test.go b/std/sync/svs_mhash_test.go new file mode 100644 index 00000000..03d42951 --- /dev/null +++ b/std/sync/svs_mhash_test.go @@ -0,0 +1,118 @@ +package sync_test + +import ( + "testing" + + enc "github.com/named-data/ndnd/std/encoding" + ndn_sync "github.com/named-data/ndnd/std/sync" + spec_svs "github.com/named-data/ndnd/std/ndn/svs/v3" + "github.com/named-data/ndnd/std/types/optional" + tu "github.com/named-data/ndnd/std/utils/testutils" + "github.com/stretchr/testify/require" +) + +func TestComputeMhashStable(t *testing.T) { + tu.SetT(t) + + m := ndn_sync.NewSvMap[uint64](0) + m.Set(tu.NoErr(enc.NameFromStr("/ndn/alice")).TlvStr(), 100, 1) + m.Set(tu.NoErr(enc.NameFromStr("/ndn/bob")).TlvStr(), 150, 3) + + h1 := ndn_sync.ComputeMhash(m) + h2 := ndn_sync.ComputeMhash(m) + require.Equal(t, h1, h2) + require.Len(t, h1, 32) +} + +func TestComputeMhashOrderIndependent(t *testing.T) { + tu.SetT(t) + + m1 := ndn_sync.NewSvMap[uint64](0) + m1.Set(tu.NoErr(enc.NameFromStr("/ndn/alice")).TlvStr(), 100, 1) + m1.Set(tu.NoErr(enc.NameFromStr("/ndn/bob")).TlvStr(), 150, 3) + + m2 := ndn_sync.NewSvMap[uint64](0) + m2.Set(tu.NoErr(enc.NameFromStr("/ndn/bob")).TlvStr(), 150, 3) + m2.Set(tu.NoErr(enc.NameFromStr("/ndn/alice")).TlvStr(), 100, 1) + + require.Equal(t, ndn_sync.ComputeMhash(m1), ndn_sync.ComputeMhash(m2)) +} + +func TestComputeMhashChangesOnMembership(t *testing.T) { + tu.SetT(t) + + m := ndn_sync.NewSvMap[uint64](0) + m.Set(tu.NoErr(enc.NameFromStr("/ndn/alice")).TlvStr(), 100, 1) + before := ndn_sync.ComputeMhash(m) + + m.Set(tu.NoErr(enc.NameFromStr("/ndn/bob")).TlvStr(), 150, 3) + after := ndn_sync.ComputeMhash(m) + require.NotEqual(t, before, after) +} + +func TestComputeMhashIgnoresSeqNo(t *testing.T) { + tu.SetT(t) + + m1 := ndn_sync.NewSvMap[uint64](0) + m1.Set(tu.NoErr(enc.NameFromStr("/ndn/alice")).TlvStr(), 100, 1) + + m2 := ndn_sync.NewSvMap[uint64](0) + m2.Set(tu.NoErr(enc.NameFromStr("/ndn/alice")).TlvStr(), 100, 99) + + require.Equal(t, ndn_sync.ComputeMhash(m1), ndn_sync.ComputeMhash(m2)) +} + +func TestSvsDataInlineTLV(t *testing.T) { + tu.SetT(t) + + m := ndn_sync.NewSvMap[uint64](0) + m.Set(tu.NoErr(enc.NameFromStr("/ndn/alice")).TlvStr(), 100, 1) + sv := m.Encode(func(s uint64) uint64 { return s }) + + original := &spec_svs.SvsData{ + MemberSetHash: ndn_sync.ComputeMhash(m), + VectorType: optional.Some(spec_svs.VectorTypeFull), + StateVector: sv, + } + wire := original.Encode().Join() + + parsed, err := spec_svs.ParseSvsData(enc.NewBufferView(wire), false) + require.NoError(t, err) + require.Equal(t, original.MemberSetHash, parsed.MemberSetHash) + require.Equal(t, original.VectorType, parsed.VectorType) + require.Equal(t, original.StateVector.Entries[0].Name.String(), parsed.StateVector.Entries[0].Name.String()) +} + +func TestSvsDataAnnounceTLV(t *testing.T) { + tu.SetT(t) + + ref := tu.NoErr(enc.NameFromStr("/ndn/svs/alice/100/32=sv/1")) + mhash := make([]byte, 32) + + original := &spec_svs.SvsData{ + MemberSetHash: mhash, + SvsDataRef: ref, + } + wire := original.Encode().Join() + + parsed, err := spec_svs.ParseSvsData(enc.NewBufferView(wire), false) + require.NoError(t, err) + require.Equal(t, mhash, parsed.MemberSetHash) + require.Equal(t, ref.String(), parsed.SvsDataRef.String()) + require.Nil(t, parsed.StateVector) + require.False(t, parsed.VectorType.IsSet()) +} + +func TestSvsDataLegacyParse(t *testing.T) { + tu.SetT(t) + + m := ndn_sync.NewSvMap[uint64](0) + m.Set(tu.NoErr(enc.NameFromStr("/ndn/alice")).TlvStr(), 100, 1) + legacy := &spec_svs.SvsData{StateVector: m.Encode(func(s uint64) uint64 { return s })} + wire := legacy.Encode().Join() + + parsed, err := spec_svs.ParseSvsData(enc.NewBufferView(wire), false) + require.NoError(t, err) + require.Nil(t, parsed.MemberSetHash) + require.NotNil(t, parsed.StateVector) +} diff --git a/std/sync/svs_pull.go b/std/sync/svs_pull.go new file mode 100644 index 00000000..8a6906f5 --- /dev/null +++ b/std/sync/svs_pull.go @@ -0,0 +1,116 @@ +package sync + +import ( + "bytes" + "fmt" + + enc "github.com/named-data/ndnd/std/encoding" + "github.com/named-data/ndnd/std/log" + "github.com/named-data/ndnd/std/ndn" + spec_svs "github.com/named-data/ndnd/std/ndn/svs/v3" + "github.com/named-data/ndnd/std/types/optional" +) + +// publishFullVectorData produces retrievable inline FULL SvsData at .../32=sv/. +func (s *SvSync) publishFullVectorData(state SvMap[uint64]) (enc.Name, error) { + if len(s.fullVectorPrefix) == 0 { + return nil, fmt.Errorf("full vector prefix unset") + } + content := buildInlineFullFromState(state).Encode() + name := s.fullVectorPrefix.WithVersion(enc.VersionUnixMicro) + return s.o.Client.Produce(ndn.ProduceArgs{ + Name: name, + Content: content, + }) +} + +// pullFullVector fetches a published full State Vector and merges it on the main loop. +// trustPrefix is the sender's .../32=sv prefix; ref must be equal to or below it. +func (s *SvSync) pullFullVector(ref enc.Name, trustPrefix enc.Name) { + if len(ref) == 0 { + return + } + if !isTrustedSvsDataRef(ref, trustPrefix) { + log.Warn(s, "pullFullVector rejected untrusted SvsDataRef", "ref", ref, "trust", trustPrefix) + return + } + log.Debug(s, "sync pull start", "ref", ref) + + s.o.Client.ConsumeExt(ndn.ConsumeExtArgs{ + Name: ref.Clone(), + TryStore: true, + UseSignatureTime: s.o.UseSignatureTime, + IgnoreValidity: s.o.IgnoreValidity, + Callback: func(st ndn.ConsumeState) { + if st.Error() != nil { + log.Warn(s, "pullFullVector failed", "ref", ref, "err", st.Error()) + return + } + if !st.IsComplete() { + return + } + s.onPulledFullVector(st.Content().Join()) + }, + }) +} + +// onPulledFullVector merges a fetched inline FULL SvsData into local state. +// Segment signatures are validated by client.ConsumeExt during fetch. +func (s *SvSync) onPulledFullVector(content []byte) { + params, err := parseFullVectorContent(content) + if err != nil { + log.Warn(s, "onPulledFullVector parse failed", "err", err) + return + } + entries := 0 + if params.StateVector != nil { + entries = len(params.StateVector.Entries) + } + log.Debug(s, "sync pull merged", "entries", entries, "mode", "FULL") + + s.recvSv <- svSyncRecvSvArgs{ + sv: params.StateVector, + vectorType: optional.Some(spec_svs.VectorTypeFull), + mhash: params.MemberSetHash, + } +} + +// sendRecoveryAnnounce publishes at 32=sv and emits announce-only Sync Data (mhash recovery). +func (s *SvSync) sendRecoveryAnnounce() { + if !s.running.Load() || s.o.Passive { + return + } + log.Debug(s, "sync send recovery announce") + wire := s.encodeSyncData(syncSendRecovery, enc.Name{}) + s.sendSyncInterestWith(wire) +} + +// handleMhashMismatch schedules announce or pull recovery per spec Section 5.6. +func (s *SvSync) handleMhashMismatch(args svSyncRecvSvArgs, recvSv SvMap[uint64]) { + localMhash := ComputeMhash(s.state) + if bytes.Equal(localMhash, args.mhash) { + return + } + + log.Debug(s, "mhash mismatch with sender", "local", localMhash, "remote", args.mhash) + + trustPrefix := pullRefFromSyncDataWire(args.data) + if len(trustPrefix) == 0 { + trustPrefix = s.fullVectorPrefix + } + + if membershipContains(s.state, recvSv) && membershipTupleCount(s.state) > membershipTupleCount(recvSv) { + log.Debug(s, "sync recovery path", "action", "sender announce", "localMembers", membershipTupleCount(s.state), "remoteMembers", membershipTupleCount(recvSv)) + go s.sendRecoveryAnnounce() + return + } + + ref := args.svsDataRef + if len(ref) == 0 { + ref = trustPrefix + } + if len(ref) > 0 { + log.Debug(s, "sync recovery path", "action", "pull", "ref", ref) + go s.pullFullVector(ref, trustPrefix) + } +} diff --git a/std/sync/svs_test.go b/std/sync/svs_test.go new file mode 100644 index 00000000..441b144f --- /dev/null +++ b/std/sync/svs_test.go @@ -0,0 +1,188 @@ +package sync + +import ( + "testing" + "time" + + enc "github.com/named-data/ndnd/std/encoding" + spec_svs "github.com/named-data/ndnd/std/ndn/svs/v3" + "github.com/named-data/ndnd/std/types/optional" + tu "github.com/named-data/ndnd/std/utils/testutils" + "github.com/stretchr/testify/require" +) + +func TestSvSyncSmallGroup(t *testing.T) { + tu.SetT(t) + + mesh := newTestMesh(t, []string{"alice", "bob", "carol"}, nil) + defer mesh.stop() + + seq := mesh.node("alice").publish() + mesh.wait(150 * time.Millisecond) + + mesh.waitUpdates("bob", 1, time.Second) + mesh.waitUpdates("carol", 1, time.Second) + + require.EqualValues(t, seq, mesh.node("bob").svs.GetSeqNo(mesh.node("alice").producer)) + require.EqualValues(t, seq, mesh.node("carol").svs.GetSeqNo(mesh.node("alice").producer)) + + // Default threshold 0 → inline FULL with mhash on the wire. + alice := mesh.node("alice") + alice.svs.mutex.Lock() + full := buildInlineFullFromState(alice.svs.state) + alice.svs.mutex.Unlock() + require.NotEmpty(t, full.MemberSetHash) + vt, ok := full.VectorType.Get() + require.True(t, ok) + require.Equal(t, spec_svs.VectorTypeFull, vt) +} + +func TestSvSyncPartialPublication(t *testing.T) { + tu.SetT(t) + + threshold := 400 + mesh := newTestMesh(t, []string{"alice", "bob"}, func(id string, opts *SvSyncOpts) { + opts.SyncVectorThreshold = threshold + }) + defer mesh.stop() + + seedLargeLocalState(mesh.node("alice").svs, mesh.node("alice").producer, 18) + mesh.node("alice").publish() + mesh.wait(200 * time.Millisecond) + + mesh.waitUpdates("bob", 1, time.Second) + require.EqualValues(t, 2, mesh.node("bob").svs.GetSeqNo(mesh.node("alice").producer)) + + // PARTIAL must not copy the full membership in one shot. + require.Less(t, mesh.node("bob").memberCount(), mesh.node("alice").memberCount()) +} + +func TestSvSyncMhashPull(t *testing.T) { + tu.SetT(t) + + threshold := 400 + mesh := newTestMesh(t, []string{"alice", "bob"}, func(id string, opts *SvSyncOpts) { + opts.SyncVectorThreshold = threshold + }) + defer mesh.stop() + + alice := mesh.node("alice").svs + seedLargeLocalState(alice, mesh.node("alice").producer, 12) + alice.state.Set(mesh.node("alice").producer.TlvStr(), testMeshBoot, 1) + + // Alice publishes full vector and sends announce-only periodic sync. + go alice.sendSyncInterest(syncSendPeriodic) + mesh.wait(300 * time.Millisecond) + + mesh.waitUpdates("bob", 1, 2*time.Second) + require.Greater(t, mesh.node("bob").memberCount(), 1) +} + +func TestSvSyncPeriodicAnnounce(t *testing.T) { + tu.SetT(t) + + threshold := 400 + mesh := newTestMesh(t, []string{"alice", "bob"}, func(id string, opts *SvSyncOpts) { + opts.SyncVectorThreshold = threshold + }) + defer mesh.stop() + + seedLargeLocalState(mesh.node("alice").svs, mesh.node("alice").producer, 12) + mesh.node("alice").svs.state.Set(mesh.node("alice").producer.TlvStr(), testMeshBoot, 2) + + go mesh.node("alice").svs.sendSyncInterest(syncSendPeriodic) + mesh.wait(300 * time.Millisecond) + + mesh.waitUpdates("bob", 1, 2*time.Second) + + // Pulled FULL state should include a seeded peer. + peer0 := tu.NoErr(enc.NameFromStr("/test/svs-mesh/peer0")) + require.Greater(t, mesh.node("bob").svs.GetSeqNo(peer0), uint64(0)) +} + +func TestSvSyncJoin(t *testing.T) { + tu.SetT(t) + + mesh := newTestMesh(t, []string{"alice", "carol"}, nil) + defer mesh.stop() + + // Alice knows about herself and has seq 3. + alice := mesh.node("alice").svs + alice.state.Set(mesh.node("alice").producer.TlvStr(), testMeshBoot, 3) + + // Carol announces join with self-only [carol:0] (simulated multicast to alice). + carolName := mesh.node("carol").producer + carolSv := selfOnlyVector(carolName, testMeshBoot) + carolState := NewSvMap[uint64](0) + carolState.Set(carolName.TlvStr(), testMeshBoot, 0) + mesh.node("alice").svs.onReceiveStateVector(svSyncRecvSvArgs{ + sv: carolSv, + vectorType: optional.Some(spec_svs.VectorTypeFull), + mhash: ComputeMhash(carolState), + }) + + // Alice responds with up-to-date FULL state. + go mesh.node("alice").svs.sendSyncInterest(syncSendPeriodic) + mesh.wait(200 * time.Millisecond) + + mesh.waitUpdates("carol", 1, time.Second) + require.EqualValues(t, 3, mesh.node("carol").svs.GetSeqNo(mesh.node("alice").producer)) +} + +func TestSvSyncJoinLargeGroup(t *testing.T) { + tu.SetT(t) + + threshold := 400 + mesh := newTestMesh(t, []string{"alice", "carol"}, func(id string, opts *SvSyncOpts) { + opts.SyncVectorThreshold = threshold + }) + defer mesh.stop() + + seedLargeLocalState(mesh.node("alice").svs, mesh.node("alice").producer, 12) + mesh.node("alice").svs.state.Set(mesh.node("alice").producer.TlvStr(), testMeshBoot, 3) + + carolName := mesh.node("carol").producer + carolSv := selfOnlyVector(carolName, testMeshBoot) + carolState := NewSvMap[uint64](0) + carolState.Set(carolName.TlvStr(), testMeshBoot, 0) + mesh.node("alice").svs.onReceiveStateVector(svSyncRecvSvArgs{ + sv: carolSv, + vectorType: optional.Some(spec_svs.VectorTypeFull), + mhash: ComputeMhash(carolState), + }) + + go mesh.node("alice").svs.sendSyncInterest(syncSendRecovery) + mesh.wait(400 * time.Millisecond) + + mesh.waitUpdates("carol", 1, 2*time.Second) + require.Greater(t, mesh.node("carol").memberCount(), 1) + require.EqualValues(t, 3, mesh.node("carol").svs.GetSeqNo(mesh.node("alice").producer)) +} + +func TestSvSyncMhashMismatchSenderAnnounce(t *testing.T) { + tu.SetT(t) + + threshold := 400 + mesh := newTestMesh(t, []string{"alice", "bob"}, func(id string, opts *SvSyncOpts) { + opts.SyncVectorThreshold = threshold + }) + defer mesh.stop() + + seedLargeLocalState(mesh.node("alice").svs, mesh.node("alice").producer, 12) + alice := mesh.node("alice").svs + + bobOnly := NewSvMap[uint64](0) + bobOnly.Set(mesh.node("bob").producer.TlvStr(), testMeshBoot, 1) + partialSv := bobOnly.Encode(func(s uint64) uint64 { return s }) + + // Bob sends PARTIAL; Alice has strict superset membership → announce recovery. + alice.onReceiveStateVector(svSyncRecvSvArgs{ + sv: partialSv, + vectorType: optional.Some(spec_svs.VectorTypePartial), + mhash: ComputeMhash(bobOnly), + }) + + mesh.wait(400 * time.Millisecond) + mesh.waitUpdates("bob", 1, 2*time.Second) + require.Greater(t, mesh.node("bob").memberCount(), 1) +} diff --git a/std/sync/svs_test_mesh.go b/std/sync/svs_test_mesh.go new file mode 100644 index 00000000..439ff0da --- /dev/null +++ b/std/sync/svs_test_mesh.go @@ -0,0 +1,157 @@ +package sync + +import ( + "fmt" + "sync" + "testing" + "time" + + enc "github.com/named-data/ndnd/std/encoding" + basic_engine "github.com/named-data/ndnd/std/engine/basic" + "github.com/named-data/ndnd/std/engine/face" + "github.com/named-data/ndnd/std/ndn" + spec_svs "github.com/named-data/ndnd/std/ndn/svs/v3" + "github.com/named-data/ndnd/std/object" + "github.com/named-data/ndnd/std/object/storage" + tu "github.com/named-data/ndnd/std/utils/testutils" + "github.com/stretchr/testify/require" +) + +const testMeshBoot = uint64(1_700_000_000) + +type testMeshNode struct { + id string + producer enc.Name + svs *SvSync + client ndn.Client + engine *basic_engine.Engine + updates []SvSyncUpdate + updatesMu sync.Mutex +} + +type testMesh struct { + t *testing.T + hub *face.MulticastHub + timer *basic_engine.DummyTimer + group enc.Name + nodes map[string]*testMeshNode +} + +func newTestMesh(t *testing.T, ids []string, configure func(id string, opts *SvSyncOpts)) *testMesh { + t.Helper() + tu.SetT(t) + + mesh := &testMesh{ + t: t, + hub: face.NewMulticastHub(), + timer: basic_engine.NewDummyTimer(), + group: tu.NoErr(enc.NameFromStr("/test/svs-mesh/32=svs")), + nodes: make(map[string]*testMeshNode, len(ids)), + } + + for _, id := range ids { + producer := mesh.group.Append(enc.NewGenericComponent(id)) + syncDataName := producer. + Append(enc.NewTimestampComponent(testMeshBoot)). + Append(enc.NewKeywordComponent("svs")) + + f := mesh.hub.NewFace() + engine := basic_engine.NewEngine(f, mesh.timer) + store := storage.NewMemoryStore() + client := object.NewClient(engine, store, nil) + + opts := SvSyncOpts{ + Client: client, + GroupPrefix: mesh.group, + SyncDataName: syncDataName, + BootTime: testMeshBoot, + PeriodicTimeout: 30 * time.Second, + SuppressionPeriod: 50 * time.Millisecond, + } + if configure != nil { + configure(id, &opts) + } + + node := &testMeshNode{ + id: id, + producer: producer, + client: client, + engine: engine, + } + opts.OnUpdate = func(u SvSyncUpdate) { + node.updatesMu.Lock() + node.updates = append(node.updates, u) + node.updatesMu.Unlock() + } + + node.svs = NewSvSync(opts) + require.NoError(t, engine.Start()) + require.NoError(t, client.Start()) + require.NoError(t, node.svs.Start()) + + mesh.nodes[id] = node + } + + mesh.wait(80 * time.Millisecond) + return mesh +} + +func (m *testMesh) node(id string) *testMeshNode { + return m.nodes[id] +} + +func (m *testMesh) wait(d time.Duration) { + time.Sleep(d) +} + +func (m *testMesh) waitUpdates(id string, n int, timeout time.Duration) { + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + m.node(id).updatesMu.Lock() + got := len(m.node(id).updates) + m.node(id).updatesMu.Unlock() + if got >= n { + return + } + time.Sleep(10 * time.Millisecond) + } + m.t.Fatalf("node %s: want %d updates, got %d", id, n, len(m.node(id).updates)) +} + +func (m *testMesh) stop() { + for _, node := range m.nodes { + _ = node.svs.Stop() + _ = node.client.Stop() + _ = node.engine.Stop() + } +} + +func (n *testMeshNode) publish() uint64 { + return n.svs.IncrSeqNo(n.producer) +} + +func (n *testMeshNode) memberCount() int { + return len(n.svs.GetNames()) +} + +func seedLargeLocalState(s *SvSync, self enc.Name, peers int) { + for i := range peers { + name := tu.NoErr(enc.NameFromStr(fmt.Sprintf("/test/svs-mesh/peer%d", i))) + s.state.Set(name.TlvStr(), testMeshBoot, uint64(i+1)) + } + s.state.Set(self.TlvStr(), testMeshBoot, 1) +} + +func selfOnlyVector(name enc.Name, boot uint64) *spec_svs.StateVector { + return &spec_svs.StateVector{ + Entries: []*spec_svs.StateVectorEntry{ + { + Name: name, + SeqNoEntries: []*spec_svs.SeqNoEntry{{ + BootstrapTime: boot, + SeqNo: 0, + }}, + }, + }, + } +} From 85403604267e18ca75f4850782d0dce7226a7944 Mon Sep 17 00:00:00 2001 From: Taranum Wasu Date: Mon, 29 Jun 2026 22:43:22 +0530 Subject: [PATCH 02/24] fix lint: sort imports in svs_mhash_test.go Co-authored-by: Cursor --- std/sync/svs_mhash_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/std/sync/svs_mhash_test.go b/std/sync/svs_mhash_test.go index 03d42951..b37597e3 100644 --- a/std/sync/svs_mhash_test.go +++ b/std/sync/svs_mhash_test.go @@ -4,8 +4,8 @@ import ( "testing" enc "github.com/named-data/ndnd/std/encoding" - ndn_sync "github.com/named-data/ndnd/std/sync" spec_svs "github.com/named-data/ndnd/std/ndn/svs/v3" + ndn_sync "github.com/named-data/ndnd/std/sync" "github.com/named-data/ndnd/std/types/optional" tu "github.com/named-data/ndnd/std/utils/testutils" "github.com/stretchr/testify/require" From 33d13b616866e4fa3af24459426cca069c64a4d2 Mon Sep 17 00:00:00 2001 From: Taranum Wasu Date: Mon, 29 Jun 2026 23:05:05 +0530 Subject: [PATCH 03/24] fix sync: avoid spurious mhash pulls and encode race MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stop blind pulls to unpublished 32=sv prefixes on inline FULL mhash mismatch (spec §5.6: merge inline first). Clone SvMap before encoding outside the mutex to avoid concurrent map mutation panics. Co-authored-by: Cursor --- std/sync/svs.go | 7 +++++-- std/sync/svs_map.go | 9 +++++++++ std/sync/svs_pull.go | 13 ++++++------- std/sync/svs_test.go | 22 ++++++++++++++++++++++ 4 files changed, 42 insertions(+), 9 deletions(-) diff --git a/std/sync/svs.go b/std/sync/svs.go index 2b3cc582..94e36c10 100644 --- a/std/sync/svs.go +++ b/std/sync/svs.go @@ -493,8 +493,11 @@ func (s *SvSync) sendSyncInterestWith(dataWire enc.Wire) { func (s *SvSync) encodeSyncData(reason syncSendReason, sender enc.Name) enc.Wire { s.mutex.Lock() s.enterSteadyState() - stateSnap := s.state - mtimeSnap := s.mtime + stateSnap := cloneSvMap(s.state) + mtimeSnap := make(map[string]time.Time, len(s.mtime)) + for k, v := range s.mtime { + mtimeSnap[k] = v + } repair, propagation := s.partialTargets() s.mutex.Unlock() diff --git a/std/sync/svs_map.go b/std/sync/svs_map.go index be1ffd0c..daf5534b 100644 --- a/std/sync/svs_map.go +++ b/std/sync/svs_map.go @@ -29,6 +29,15 @@ func NewSvMap[V any](size int) SvMap[V] { return make(SvMap[V], size) } +// cloneSvMap returns a shallow copy safe for use without holding SvSync.mutex. +func cloneSvMap[V any](m SvMap[V]) SvMap[V] { + out := NewSvMap[V](len(m)) + for hash, vals := range m { + out[hash] = slices.Clone(vals) + } + return out +} + // Get seq entry for a bootstrap time. func (m SvMap[V]) Get(hash string, boot uint64) (value V) { entry := SvMapVal[V]{boot, value} diff --git a/std/sync/svs_pull.go b/std/sync/svs_pull.go index 8a6906f5..696e56cc 100644 --- a/std/sync/svs_pull.go +++ b/std/sync/svs_pull.go @@ -105,12 +105,11 @@ func (s *SvSync) handleMhashMismatch(args svSyncRecvSvArgs, recvSv SvMap[uint64] return } - ref := args.svsDataRef - if len(ref) == 0 { - ref = trustPrefix - } - if len(ref) > 0 { - log.Debug(s, "sync recovery path", "action", "pull", "ref", ref) - go s.pullFullVector(ref, trustPrefix) + // Inline FULL is already merged in onReceiveStateVector (spec §5.6 step 2). + // Pull only when the sender provided a retrievable SvsDataRef (announce-only sync). + if len(args.svsDataRef) == 0 { + return } + log.Debug(s, "sync recovery path", "action", "pull", "ref", args.svsDataRef) + go s.pullFullVector(args.svsDataRef, trustPrefix) } diff --git a/std/sync/svs_test.go b/std/sync/svs_test.go index 441b144f..f40eb105 100644 --- a/std/sync/svs_test.go +++ b/std/sync/svs_test.go @@ -159,6 +159,28 @@ func TestSvSyncJoinLargeGroup(t *testing.T) { require.EqualValues(t, 3, mesh.node("carol").svs.GetSeqNo(mesh.node("alice").producer)) } +func TestHandleMhashMismatchNoBlindPull(t *testing.T) { + tu.SetT(t) + + mesh := newTestMesh(t, []string{"alice", "bob"}, nil) + defer mesh.stop() + + alice := mesh.node("alice").svs + bobOnly := NewSvMap[uint64](0) + bobOnly.Set(mesh.node("bob").producer.TlvStr(), testMeshBoot, 1) + partialSv := bobOnly.Encode(func(s uint64) uint64 { return s }) + + // Bob's inline FULL with mismatched mhash; alice is not a strict superset. + alice.onReceiveStateVector(svSyncRecvSvArgs{ + sv: partialSv, + vectorType: optional.Some(spec_svs.VectorTypeFull), + mhash: ComputeMhash(bobOnly), + }) + + // Should not hang or panic from blind pull to unpublished 32=sv prefix. + mesh.wait(100 * time.Millisecond) +} + func TestSvSyncMhashMismatchSenderAnnounce(t *testing.T) { tu.SetT(t) From 66757b55a3cbe5ebab91b579bfb5cfae2ce016e7 Mon Sep 17 00:00:00 2001 From: Taranum Wasu Date: Mon, 29 Jun 2026 23:15:30 +0530 Subject: [PATCH 04/24] fix sync: disable announce+pull recovery when threshold is 0 DV prefix-table SVS uses default SyncVectorThreshold=0. Recovery was still forcing announce-only sync on mhash mismatch, breaking step-2 prefix propagation while router reachability looked fine. Gate large-group recovery behind threshold > 0 for true legacy behavior. Co-authored-by: Cursor --- std/sync/svs_announce.go | 6 +++++- std/sync/svs_announce_test.go | 3 ++- std/sync/svs_pull.go | 5 +++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/std/sync/svs_announce.go b/std/sync/svs_announce.go index c7cd34ce..72433de4 100644 --- a/std/sync/svs_announce.go +++ b/std/sync/svs_announce.go @@ -66,11 +66,15 @@ func inlineFullSize(state SvMap[uint64]) int { } // shouldUseAnnouncePull reports whether the sender should publish at 32=sv and announce-only Sync Data. +// Large-group announce+pull is disabled when threshold <= 0 (legacy inline-FULL-only mode). func shouldUseAnnouncePull(reason syncSendReason, threshold int, state SvMap[uint64]) bool { + if threshold <= 0 { + return false + } if reason == syncSendRecovery { return true } - if reason == syncSendPublication || threshold <= 0 { + if reason == syncSendPublication { return false } return exceedsSyncThreshold(threshold, inlineFullSize(state)) diff --git a/std/sync/svs_announce_test.go b/std/sync/svs_announce_test.go index 1a175310..1eae43f3 100644 --- a/std/sync/svs_announce_test.go +++ b/std/sync/svs_announce_test.go @@ -57,7 +57,8 @@ func TestShouldUseAnnouncePull(t *testing.T) { require.False(t, shouldUseAnnouncePull(syncSendPeriodic, fullSize+1, m)) require.True(t, shouldUseAnnouncePull(syncSendPeriodic, fullSize-1, m)) require.True(t, shouldUseAnnouncePull(syncSendOther, fullSize-1, m)) - require.True(t, shouldUseAnnouncePull(syncSendRecovery, 0, m)) + require.False(t, shouldUseAnnouncePull(syncSendRecovery, 0, m)) + require.True(t, shouldUseAnnouncePull(syncSendRecovery, fullSize-1, m)) } func TestIsTrustedSvsDataRef(t *testing.T) { diff --git a/std/sync/svs_pull.go b/std/sync/svs_pull.go index 696e56cc..4734b770 100644 --- a/std/sync/svs_pull.go +++ b/std/sync/svs_pull.go @@ -87,6 +87,11 @@ func (s *SvSync) sendRecoveryAnnounce() { // handleMhashMismatch schedules announce or pull recovery per spec Section 5.6. func (s *SvSync) handleMhashMismatch(args svSyncRecvSvArgs, recvSv SvMap[uint64]) { + // Legacy mode (threshold 0): inline FULL is merged above; large-group recovery is off. + if s.o.SyncVectorThreshold <= 0 { + return + } + localMhash := ComputeMhash(s.state) if bytes.Equal(localMhash, args.mhash) { return From ca6719eebc02bc5a6d51908a15ea758663c07220 Mon Sep 17 00:00:00 2001 From: Taranum Wasu Date: Mon, 29 Jun 2026 23:33:52 +0530 Subject: [PATCH 05/24] fix sync: use legacy SvsData wire when threshold is 0 DV prefix-table sync uses default threshold 0. Emit StateVector-only SvsData (no mhash/VectorType) so step-2 SVS matches pre-revision behavior on the wire while large-group features stay behind threshold > 0. Co-authored-by: Cursor --- std/sync/svs_encode.go | 10 ++++++++++ std/sync/svs_test.go | 10 ++++------ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/std/sync/svs_encode.go b/std/sync/svs_encode.go index 1201fe14..7c8619a9 100644 --- a/std/sync/svs_encode.go +++ b/std/sync/svs_encode.go @@ -30,6 +30,12 @@ type PartialEncodeOpts struct { Mtime map[string]time.Time } +// buildLegacySvsData is the pre-revision SVS v3 wire format used when threshold is 0. +func buildLegacySvsData(state SvMap[uint64]) *spec_svs.SvsData { + sv := state.Encode(func(seq uint64) uint64 { return seq }) + return &spec_svs.SvsData{StateVector: sv} +} + // buildInlineSvsData constructs inline SvsData (mhash + VectorType + StateVector). func buildInlineSvsData(state SvMap[uint64], vectorType uint64, sv *spec_svs.StateVector) *spec_svs.SvsData { return &spec_svs.SvsData{ @@ -59,6 +65,10 @@ func buildSvsDataForSend( repair, propagation []enc.Name, mtime map[string]time.Time, ) *spec_svs.SvsData { + if threshold <= 0 { + return buildLegacySvsData(state) + } + fullSv := state.Encode(func(seq uint64) uint64 { return seq }) fullData := buildInlineSvsData(state, spec_svs.VectorTypeFull, fullSv) diff --git a/std/sync/svs_test.go b/std/sync/svs_test.go index f40eb105..2c833437 100644 --- a/std/sync/svs_test.go +++ b/std/sync/svs_test.go @@ -26,15 +26,13 @@ func TestSvSyncSmallGroup(t *testing.T) { require.EqualValues(t, seq, mesh.node("bob").svs.GetSeqNo(mesh.node("alice").producer)) require.EqualValues(t, seq, mesh.node("carol").svs.GetSeqNo(mesh.node("alice").producer)) - // Default threshold 0 → inline FULL with mhash on the wire. + // Default threshold 0 → legacy wire (StateVector only, no mhash). alice := mesh.node("alice") alice.svs.mutex.Lock() - full := buildInlineFullFromState(alice.svs.state) + legacy := buildLegacySvsData(alice.svs.state) alice.svs.mutex.Unlock() - require.NotEmpty(t, full.MemberSetHash) - vt, ok := full.VectorType.Get() - require.True(t, ok) - require.Equal(t, spec_svs.VectorTypeFull, vt) + require.Empty(t, legacy.MemberSetHash) + require.False(t, legacy.VectorType.IsSet()) } func TestSvSyncPartialPublication(t *testing.T) { From 2b36d342d115a9e7b799af2f1dd617fd538a9eed Mon Sep 17 00:00:00 2001 From: Taranum Wasu Date: Tue, 30 Jun 2026 00:03:03 +0530 Subject: [PATCH 06/24] e2e: add minimal local topology and docker runner script Adds topo.min.conf (6 nodes) and make e2e-local for reliable local validation on Mac Docker/Colima. Full sprint topo remains make e2e for CI. Co-authored-by: Cursor --- Makefile | 4 ++++ e2e/topo.min.conf | 16 ++++++++++++++++ scripts/e2e-local-docker.sh | 21 +++++++++++++++++++++ 3 files changed, 41 insertions(+) create mode 100644 e2e/topo.min.conf create mode 100755 scripts/e2e-local-docker.sh diff --git a/Makefile b/Makefile index 37891c28..b25343eb 100644 --- a/Makefile +++ b/Makefile @@ -29,6 +29,10 @@ e2e: # minindn container sed -i 's/readvertise_nlsr no/readvertise_nlsr yes/g' /usr/local/etc/ndn/nfd.conf.sample python3 e2e/runner.py e2e/topo.sprint.conf +e2e-local: + sed -i 's/readvertise_nlsr no/readvertise_nlsr yes/g' /usr/local/etc/ndn/nfd.conf.sample + python3 e2e/runner.py e2e/topo.min.conf + coverage: go tool cover -html=coverage.out diff --git a/e2e/topo.min.conf b/e2e/topo.min.conf new file mode 100644 index 00000000..c17f31a5 --- /dev/null +++ b/e2e/topo.min.conf @@ -0,0 +1,16 @@ +# Small topology for local e2e (6 nodes, minimal link delays). +# Use: python3 e2e/runner.py e2e/topo.min.conf +[nodes] +alpha: _ network=/minindn router=/alpha/ position=0,0,0 +beta: _ network=/minindn router=/beta/ position=1,0,0 +gamma: _ network=/minindn router=/gamma/ position=2,0,0 +delta: _ network=/minindn router=/delta/ position=3,0,0 +epsilon: _ network=/minindn router=/epsilon/ position=4,0,0 +zeta: _ network=/minindn router=/zeta/ position=5,0,0 +[switches] +[links] +alpha:beta delay=1ms +beta:gamma delay=1ms +gamma:delta delay=1ms +delta:epsilon delay=1ms +epsilon:zeta delay=1ms diff --git a/scripts/e2e-local-docker.sh b/scripts/e2e-local-docker.sh new file mode 100755 index 00000000..dcbea6c9 --- /dev/null +++ b/scripts/e2e-local-docker.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Run local e2e inside the mini-ndn Docker image (Colima/Docker required). +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$ROOT" + +ARCH="$(docker info -f '{{.Architecture}}' 2>/dev/null || uname -m)" +case "$ARCH" in + aarch64|arm64) GOARCH=arm64 ;; + *) GOARCH=amd64 ;; +esac + +CGO_ENABLED=0 GOOS=linux GOARCH="$GOARCH" go build -o "ndnd-linux-${GOARCH}" cmd/ndnd/main.go + +docker run --rm --privileged \ + --sysctl net.ipv6.conf.all.disable_ipv6=0 \ + -v "$ROOT:/workspace/ndnd" \ + -w /workspace/ndnd \ + --entrypoint bash \ + ghcr.io/named-data/mini-ndn:master \ + -c "cp /workspace/ndnd/ndnd-linux-${GOARCH} /usr/bin/ndnd && chmod +x /usr/bin/ndnd && make e2e-local" From 05623e2f99d7de09b0b4bd2c09ee00747f9504ef Mon Sep 17 00:00:00 2001 From: Taranum Wasu Date: Tue, 30 Jun 2026 00:57:57 +0530 Subject: [PATCH 07/24] docs: add SVS v3 large-group revision specification Publish the revision spec alongside the std/sync implementation in PR #190. Co-authored-by: Cursor --- docs/svs-v3-revision.md | 377 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 377 insertions(+) create mode 100644 docs/svs-v3-revision.md diff --git a/docs/svs-v3-revision.md b/docs/svs-v3-revision.md new file mode 100644 index 00000000..f32d480f --- /dev/null +++ b/docs/svs-v3-revision.md @@ -0,0 +1,377 @@ +# State Vector Sync (SVS) v3: Revision Specification + +This document **revises** SVS v3 for large synchronization groups. It is **not** a new protocol. Existing SVS v3 semantics apply whenever the complete State Vector fits within the configured size threshold. + +--- + +## 1. Basic Protocol Design + +### 1.1 Small groups + +For most deployments, the complete State Vector fits in one Sync packet. Nodes exchange **full** State Vectors using existing SVS v3 logic (suppression, steady state, merge, `OnUpdate`). + +### 1.2 Large groups + +When the encoded State Vector exceeds **`SyncVectorThreshold`** (configurable application limit), nodes use three dissemination modes: + +| Mode | When | On the wire | +|------|------|-------------| +| **Inline FULL** | Encoded FULL fits in threshold | `mhash` + `VectorType=FULL` + complete `StateVector` in Sync Data | +| **Inline PARTIAL** | **New publication** and FULL exceeds threshold | `mhash` + `VectorType=PARTIAL` + subset `StateVector` in Sync Data | +| **Announce + pull** | **Periodic sync** (large group), or **`mhash` mismatch** | Produce full vector Data at `32=sv/`; Sync Data carries `mhash` + reference Name only | + +**MemberSetHash (`mhash`)** is always carried inside `SvsData`. It is a **membership hash**, not a full-vector hash. + +**Full state recovery** (fetch complete State Vector from a remote sync member) uses **announce + pull** when: + +1. **`mhash` differs** from the local membership hash, or +2. **Periodic sync** runs while the local FULL encoding exceeds `SyncVectorThreshold`, or +3. An inline **`VectorType = FULL`** State Vector is outdated per Section 6.2. + +Link-level **fragmentation** (NDNLPv2) is not a concern for implementers. Publishers use **existing ndnd object segmentation** APIs when retrievable full-vector Data is large. + +--- + +## 2. Format and Naming + +### 2.1 Sync Interest + +**Sync Interest Name:** + +``` +//v=3 +``` + +Implementations MAY append additional name components after `v=3`. The Interest **nonce** is carried in Interest packet fields, **not** as a name component. + +- Signed **Sync Data** is carried in `ApplicationParameters`. +- Interest Lifetime SHOULD be 1 second. +- Sync Interests are **not** acknowledged (unchanged). + +### 2.2 Sync Data (in ApplicationParameters) + +**Sync Data Name** (signing identity for the Sync message): + +``` +//// +``` + +- **`version`:** microsecond timestamp (default). A hash suffix component is deferred. + +**Sync Data Content:** encoded `SvsData` (Section 3) — either **inline** or **announce-only** form. + +### 2.3 Application publication Data + +``` +////seq= +``` + +Application-level naming may vary. **Sync vector Data MUST NOT share the application publication namespace.** The `32=sv` keyword (Section 2.4) separates sync state from application Data. + +### 2.4 Published full State Vector Data + +Retrievable full State Vector objects use a dedicated sync namespace: + +**Name:** + +``` +////32=sv/ +``` + +**Content:** signed `SvsData` in **inline FULL** form: `mhash` + `VectorType = FULL` + complete `StateVector`. + +**Announce + pull procedure** (periodic sync, `mhash` recovery, join when FULL exceeds threshold): + +1. **Produce** the complete full-vector Data at `////32=sv/` (use ndnd segmentation when large). +2. **Send** a Sync Interest whose AppParam Sync Data contains **announce-only** `SvsData`: `mhash` + `SvsDataRef` pointing at that published name (Section 3.1). +3. Receivers **pull** the referenced Data, validate, and merge. + +Do **not** send an inline PARTIAL vector alongside an announce-only Sync message. + +--- + +## 3. Packet Specification + +### 3.1 `SvsData` + +`SvsData` has two forms. **`mhash` is always present.** It is not a separate protocol message. + +#### 3.1.1 Inline form (FULL or PARTIAL) + +Used when the State Vector (or a publication-time PARTIAL subset) is carried inline in Sync Data, or in published full-vector Data at `32=sv/`. + +``` +SvsData = SVS-DATA-TYPE TLV-LENGTH + MemberSetHash + VectorType + StateVector +``` + +| Field | TLV type | Value | +|-------|----------|-------| +| `MemberSetHash` | `0xCB` | 32-byte SHA-256 digest (`mhash`) | +| `VectorType` | `0xCD` | `0` = FULL, `1` = PARTIAL | +| `StateVector` | `0xC9` | See Section 3.2 | + +#### 3.1.2 Announce-only form + +Used when Sync Data only advertises a retrievable full-vector Data name (periodic sync, `mhash` recovery). **No `VectorType` or `StateVector`.** + +``` +SvsData = SVS-DATA-TYPE TLV-LENGTH + MemberSetHash + SvsDataRef +``` + +| Field | TLV type | Value | +|-------|----------|-------| +| `MemberSetHash` | `0xCB` | 32-byte SHA-256 digest (`mhash`) | +| `SvsDataRef` | `0x07` (Name) | Name of published full-vector Data: `////32=sv/` | + +> **Note:** The inline layout extends ndnd v3 `SvsData` with `MemberSetHash` and `VectorType` before `StateVector`, matching the Python strawman (`mhash` at `0xCB`, vector at `0xC9`/`0xCA`). + +### 3.2 `StateVector` + +``` +StateVector = STATE-VECTOR-TYPE TLV-LENGTH + *StateVectorEntry + +StateVectorEntry = STATE-VECTOR-ENTRY-TYPE TLV-LENGTH + Name + *SeqNoEntry + +SeqNoEntry = SEQ-NO-ENTRY-TYPE TLV-LENGTH + BootstrapTime + SeqNo +``` + +| TLV | Type (decimal) | Type (hex) | +|-----|----------------|------------| +| `STATE-VECTOR-TYPE` | 201 | `0xC9` | +| `STATE-VECTOR-ENTRY-TYPE` | 202 | `0xCA` | +| `SEQ-NO-ENTRY-TYPE` | 210 | `0xD2` | +| `BOOTSTRAP-TIME-TYPE` | 212 | `0xD4` | +| `SEQ-NO-TYPE` | 214 | `0xD6` | + +**Rules (unchanged from SVS v3):** + +- Sequence numbers are 1-indexed. +- Bootstrap time is seconds since Unix epoch. +- If an entry is absent, its sequence number is treated as 0 for comparison. +- If any received `BootstrapTime` is more than 86400s in the future, the entire `StateVector` SHOULD be ignored. + +### 3.3 `MemberSetHash` (`mhash`) + +**`mhash` is a membership hash.** It is **not** a hash of the full State Vector and **not** a hash of sequence numbers. + +**Membership** is the set of participants, each identified by: + +``` +(Producer Name, Bootstrap Time) +``` + +**Computation:** + +``` +members = { (Name, BootstrapTime) | node knows this member in the sync group } +sort by NDN canonical order of Name, then by BootstrapTime ascending +mhash = SHA-256( concatenation of canonical TLV bytes of each (Name, BootstrapTime) pair ) +``` + +Recompute `mhash` whenever membership changes (member added, removed, or new bootstrap time for a name). + +> **Note:** The Python strawman hashes sorted producer **names only**. This revision includes **Bootstrap Time** in each membership tuple, consistent with SVS v3 identity. + +**Membership data and State Vector data are separate concepts.** Membership is carried implicitly in the full State Vector. `mhash` summarizes membership for quick comparison. + +### 3.4 `VectorType` (inline form only) + +| Value | Name | Meaning | +|-------|------|---------| +| `0` | **FULL** | `StateVector` contains the complete advertised state (Section 4.1 ordering). | +| `1` | **PARTIAL** | `StateVector` contains a subset (Section 4.2). Used for **new publication** only when FULL exceeds threshold. | + +`mhash` is present in both inline and announce-only `SvsData` messages. + +--- + +## 4. State Vector Encoding + +### 4.1 FULL State Vector + +- Include all known members and their latest sequence numbers per bootstrap. +- Entries ordered in **NDN canonical order** of `Name` (unchanged SVS v3 rule). +- Set `VectorType = FULL`. + +### 4.2 PARTIAL State Vector + +Used **only on new publication** when `encoded_size(inline FULL SvsData) > SyncVectorThreshold`. + +- Set `VectorType = PARTIAL`. +- **Entry `[0]`** MUST be the **sender's** own `StateVectorEntry`. +- **Entries `[1…n]`** MUST be in **NDN canonical order** among included peers. + +An **implementation** MAY use the following selection priority: + +| Priority | Include | +|----------|---------| +| 1 | Sender (always) | +| 2 | Repair targets | +| 3 | Propagation targets | +| 4 | Random inactive producers | +| 5 | Others by recency | + +Stop adding entries when estimated inline `SvsData` size approaches `SyncVectorThreshold`. + +### 4.3 `SyncVectorThreshold` + +- Configurable implementation parameter (application packet size budget). +- Default value is implementation-defined. +- When `encoded_size(FULL) ≤ SyncVectorThreshold`, nodes use **inline FULL** only (existing SVS v3 behavior). + +--- + +## 5. State Sync + +Sections 5.1–5.4 unchanged in spirit from [SVS v3 Section 4](https://named-data.github.io/StateVectorSync/Specification.html). This revision adds Sections 5.5–5.9. + +### 5.1 Sync Interest timer + +- `PeriodicTimeout` default 30s (±10% jitter). +- `SuppressionPeriod` default 200ms. +- `SuppressionTimeout` exponential decay (unchanged formula). + +### 5.2 Send Sync Interest on new publication + +When the node generates a new publication, it immediately emits a Sync Interest and resets the timer to `PeriodicTimeout`. + +| Condition | Action | +|-----------|--------| +| `encoded_size(inline FULL) ≤ SyncVectorThreshold` | Send **inline FULL** (`mhash` + `VectorType=FULL` + `StateVector`) | +| `encoded_size(inline FULL) > SyncVectorThreshold` | Send **inline PARTIAL** (`mhash` + `VectorType=PARTIAL` + subset `StateVector`) | + +### 5.3 Sync Ack policy + +Do not acknowledge Sync Interests. + +### 5.4 Steady state and suppression (unchanged for inline FULL) + +For incoming Sync Data with inline `VectorType = FULL`, apply existing SVS v3 steady-state and suppression rules. + +### 5.5 PARTIAL State Vector processing + +When `VectorType = PARTIAL`: + +1. Parse `mhash` and `StateVector`. +2. **Do not** treat names **omitted** from the partial `StateVector` as producer removal, outdated sender (by omission alone), or sequence rollback. +3. For each **present** entry, merge newer sequence numbers into local state (unchanged merge rule). +4. If `mhash` differs from local `mhash`, perform **announce + pull** recovery (Section 5.6). + +This is the primary **receive-side change** in ndnd (`svs.go`). + +### 5.6 Full state recovery (announce + pull) + +**Triggers:** + +| # | Condition | Action | +|---|-----------|--------| +| 1 | `mhash` in received `SvsData` ≠ locally computed `mhash` | Announce + pull | +| 2 | Inline `VectorType = FULL` is outdated per Section 6.2 | Merge inline if complete; otherwise announce + pull | +| 3 | Periodic sync while local FULL exceeds `SyncVectorThreshold` | Sender: announce + pull (Section 5.8) | + +**There is no separate membership-only retrieval.** Recovery always fetches the **complete State Vector** from the referenced `32=sv/` Data. + +**Procedure (sender on `mhash` mismatch or periodic large-group sync):** + +1. Produce full-vector Data at `////32=sv/` with inline FULL `SvsData`. +2. Send Sync Interest with announce-only `SvsData` (`mhash` + `SvsDataRef`). + +**Procedure (receiver):** + +1. Identify sender from Sync Data signature and/or PARTIAL entry `[0]`. +2. If Sync Data is **inline FULL** and complete: merge directly. +3. If Sync Data is **announce-only**: read `SvsDataRef`; express Interest for that name; validate; merge; update local `mhash`. +4. Continue application data fetch via SvsALO (`OnUpdate`) as today. + +Use **ndnd segmentation** when fetched Data content is large. + +### 5.7 New node join + +1. Joining node **N** multicasts Sync Interest with **only itself**: `(Name=N, SeqNo=0)` and its current `mhash`. +2. Existing members receive the announcement. +3. **Suppression** limits duplicate responses; typically one member **A** provides recovery state. +4. If FULL fits inline: **A** responds with inline `VectorType = FULL`. +5. If FULL exceeds `SyncVectorThreshold`: **A** uses **announce + pull** (produce at `32=sv/`, then announce-only Sync Data). +6. Normal synchronization proceeds through SvsALO. + +### 5.8 Periodic sync in large groups + +| Local FULL size | Periodic Sync behavior | +|-----------------|------------------------| +| `≤ SyncVectorThreshold` | **Inline FULL** (existing SVS v3) | +| `> SyncVectorThreshold` | **Always announce + pull** (produce full-vector Data, then announce-only Sync Data) | + +Periodic sync does **not** send inline PARTIAL vectors. + +### 5.9 Summary of sync triggers + +| Event | `size ≤ threshold` | `size > threshold` | +|-------|--------------------|--------------------| +| **New publication** | Inline FULL | Inline PARTIAL | +| **Periodic sync** | Inline FULL | Announce + pull | +| **`mhash` mismatch** | Announce + pull (if recovery needed) | Announce + pull | + +--- + +## 6. Comparing and Merging State Vectors + +### 6.1 Merge rule + +For each matching `(Name, BootstrapTime)`, retain the maximum `SeqNo`. + +### 6.2 Outdated vector (inline FULL only) + +State Vector `A` is outdated to `B` if: + +- `A` is missing a name present in `B`, or +- `A` has a strictly smaller `SeqNo` for any entry. + +For `VectorType = PARTIAL`, the missing-name rule **does not** apply to names omitted from the partial message. + +--- + +## 7. Examples + +### 7.1 Small group + +Three nodes `A`, `B`, `C`. Full State Vector fits. `A` publishes; sends inline FULL Sync Interest `[A:11, B:15, C:25]`. Peers merge. + +### 7.2 Large group + +Group exceeds `SyncVectorThreshold`. Producer `P` publishes: + +- `P` sends inline PARTIAL `SvsData { mhash, VectorType=PARTIAL, StateVector=[P:…, A:…, …] }`. +- Receiver merges present entries only. +- If `mhash` differs, `P` (or receiver per policy) triggers announce + pull (Section 5.6). + +### 7.3 Large group + +- `A` produces full vector at `/group/A/boot/32=sv/`. +- `A` sends announce-only Sync Data `{ mhash, SvsDataRef=/group/A/boot/32=sv/ }`. +- Peers pull and merge. + +### 7.4 New node join + +- `N` sends self-only vector `[N:0]` with `mhash`. +- `A` responds with inline FULL or announce + pull. +- `N` merges and synchronizes via SvsALO. + +--- + +## 8. Open Items + +1. **Mixed-version interoperability** — how revised nodes coexist with plain SVS v3 peers in the same sync group, if at all. + +--- + + From 79b79ee033284c514cac7287d4811d9a103c6f97 Mon Sep 17 00:00:00 2001 From: Taranum Wasu Date: Sat, 4 Jul 2026 20:01:52 +0530 Subject: [PATCH 08/24] address review: trim e2e/debug helpers and consolidate sync code Remove local-only e2e topology and docker runner. Drop svs_debug and production multicast_face; keep mesh integration tests with a test-only hub in svs_mesh_test.go. Merge announce helpers into svs_pull, use svsSendInput for buildSvsDataForSend, and remove debug send logging. Co-authored-by: Cursor --- Makefile | 4 - e2e/topo.min.conf | 16 -- scripts/e2e-local-docker.sh | 21 --- std/engine/face/multicast_face.go | 82 ---------- std/sync/svs.go | 25 ++- std/sync/svs_announce.go | 150 ------------------ std/sync/svs_debug.go | 89 ----------- std/sync/svs_encode.go | 61 ++++--- std/sync/svs_encode_test.go | 8 +- .../{svs_test_mesh.go => svs_mesh_test.go} | 112 ++++++++++++- std/sync/svs_pull.go | 124 ++++++++++++++- 11 files changed, 284 insertions(+), 408 deletions(-) delete mode 100644 e2e/topo.min.conf delete mode 100755 scripts/e2e-local-docker.sh delete mode 100644 std/engine/face/multicast_face.go delete mode 100644 std/sync/svs_announce.go delete mode 100644 std/sync/svs_debug.go rename std/sync/{svs_test_mesh.go => svs_mesh_test.go} (57%) diff --git a/Makefile b/Makefile index b25343eb..37891c28 100644 --- a/Makefile +++ b/Makefile @@ -29,10 +29,6 @@ e2e: # minindn container sed -i 's/readvertise_nlsr no/readvertise_nlsr yes/g' /usr/local/etc/ndn/nfd.conf.sample python3 e2e/runner.py e2e/topo.sprint.conf -e2e-local: - sed -i 's/readvertise_nlsr no/readvertise_nlsr yes/g' /usr/local/etc/ndn/nfd.conf.sample - python3 e2e/runner.py e2e/topo.min.conf - coverage: go tool cover -html=coverage.out diff --git a/e2e/topo.min.conf b/e2e/topo.min.conf deleted file mode 100644 index c17f31a5..00000000 --- a/e2e/topo.min.conf +++ /dev/null @@ -1,16 +0,0 @@ -# Small topology for local e2e (6 nodes, minimal link delays). -# Use: python3 e2e/runner.py e2e/topo.min.conf -[nodes] -alpha: _ network=/minindn router=/alpha/ position=0,0,0 -beta: _ network=/minindn router=/beta/ position=1,0,0 -gamma: _ network=/minindn router=/gamma/ position=2,0,0 -delta: _ network=/minindn router=/delta/ position=3,0,0 -epsilon: _ network=/minindn router=/epsilon/ position=4,0,0 -zeta: _ network=/minindn router=/zeta/ position=5,0,0 -[switches] -[links] -alpha:beta delay=1ms -beta:gamma delay=1ms -gamma:delta delay=1ms -delta:epsilon delay=1ms -epsilon:zeta delay=1ms diff --git a/scripts/e2e-local-docker.sh b/scripts/e2e-local-docker.sh deleted file mode 100755 index dcbea6c9..00000000 --- a/scripts/e2e-local-docker.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash -# Run local e2e inside the mini-ndn Docker image (Colima/Docker required). -set -euo pipefail -ROOT="$(cd "$(dirname "$0")/.." && pwd)" -cd "$ROOT" - -ARCH="$(docker info -f '{{.Architecture}}' 2>/dev/null || uname -m)" -case "$ARCH" in - aarch64|arm64) GOARCH=arm64 ;; - *) GOARCH=amd64 ;; -esac - -CGO_ENABLED=0 GOOS=linux GOARCH="$GOARCH" go build -o "ndnd-linux-${GOARCH}" cmd/ndnd/main.go - -docker run --rm --privileged \ - --sysctl net.ipv6.conf.all.disable_ipv6=0 \ - -v "$ROOT:/workspace/ndnd" \ - -w /workspace/ndnd \ - --entrypoint bash \ - ghcr.io/named-data/mini-ndn:master \ - -c "cp /workspace/ndnd/ndnd-linux-${GOARCH} /usr/bin/ndnd && chmod +x /usr/bin/ndnd && make e2e-local" diff --git a/std/engine/face/multicast_face.go b/std/engine/face/multicast_face.go deleted file mode 100644 index dbbe85f0..00000000 --- a/std/engine/face/multicast_face.go +++ /dev/null @@ -1,82 +0,0 @@ -package face - -import ( - "fmt" - "sync" - - enc "github.com/named-data/ndnd/std/encoding" -) - -// MulticastHub connects MulticastFace peers for in-process multicast-style tests. -type MulticastHub struct { - mu sync.Mutex - faces []*MulticastFace -} - -// MulticastFace delivers outgoing packets to every other face in the hub. -type MulticastFace struct { - baseFace - hub *MulticastHub - id int -} - -// NewMulticastHub creates a hub for test faces. -func NewMulticastHub() *MulticastHub { - return &MulticastHub{} -} - -// NewFace registers a new peer face on the hub. -func (h *MulticastHub) NewFace() *MulticastFace { - h.mu.Lock() - defer h.mu.Unlock() - f := &MulticastFace{ - baseFace: newBaseFace(true), - hub: h, - id: len(h.faces), - } - h.faces = append(h.faces, f) - return f -} - -func (f *MulticastFace) String() string { - return fmt.Sprintf("multicast-face-%d", f.id) -} - -func (f *MulticastFace) Open() error { - if f.onError == nil || f.onPkt == nil { - return fmt.Errorf("face callbacks are not set") - } - if f.running.Load() { - return fmt.Errorf("face is already running") - } - f.setStateUp() - return nil -} - -func (f *MulticastFace) Close() error { - f.setStateClosed() - return nil -} - -func (f *MulticastFace) Send(pkt enc.Wire) error { - if !f.running.Load() { - return fmt.Errorf("face is not running") - } - f.hub.broadcast(f.id, pkt.Join()) - return nil -} - -func (h *MulticastHub) broadcast(from int, frame []byte) { - h.mu.Lock() - peers := append([]*MulticastFace(nil), h.faces...) - h.mu.Unlock() - - for _, peer := range peers { - if peer.id == from || !peer.running.Load() || peer.onPkt == nil { - continue - } - frameCopy := make([]byte, len(frame)) - copy(frameCopy, frame) - peer.onPkt(frameCopy) - } -} diff --git a/std/sync/svs.go b/std/sync/svs.go index 94e36c10..6da75e33 100644 --- a/std/sync/svs.go +++ b/std/sync/svs.go @@ -461,7 +461,6 @@ func (s *SvSync) sendSyncInterest(reason syncSendReason, pubName ...enc.Name) { } // Encode and sign the current state vector - log.Debug(s, "sync interest", "reason", syncSendReasonString(reason)) wire := s.encodeSyncData(reason, sender) s.sendSyncInterestWith(wire) } @@ -510,17 +509,16 @@ func (s *SvSync) encodeSyncData(reason syncSendReason, sender enc.Name) enc.Wire } svsData = buildAnnounceSvsData(stateSnap, ref) } else { - svsData = buildSvsDataForSend( - stateSnap, - reason, - s.o.SyncVectorThreshold, - sender, - repair, - propagation, - mtimeSnap, - ) - } - logSyncSend(s, reason, svsData) + svsData = buildSvsDataForSend(svsSendInput{ + State: stateSnap, + Reason: reason, + Threshold: s.o.SyncVectorThreshold, + Sender: sender, + Repair: repair, + Propagation: propagation, + Mtime: mtimeSnap, + }) + } svWire := svsData.Encode() // SVS v3 Sync Data @@ -590,7 +588,6 @@ func (s *SvSync) onSyncData(dataWire enc.Wire) { // Announce-only: retrievable full vector reference. if params.StateVector == nil && len(params.SvsDataRef) > 0 { - logSyncRecv(s, params, data.Name()) trustPrefix := pullRefFromSyncDataWire(dataWire) go s.pullFullVector(params.SvsDataRef, trustPrefix) return @@ -600,8 +597,6 @@ func (s *SvSync) onSyncData(dataWire enc.Wire) { return } - logSyncRecv(s, params, data.Name()) - args := svSyncRecvSvArgs{ sv: params.StateVector, data: dataWire, diff --git a/std/sync/svs_announce.go b/std/sync/svs_announce.go deleted file mode 100644 index 72433de4..00000000 --- a/std/sync/svs_announce.go +++ /dev/null @@ -1,150 +0,0 @@ -package sync - -import ( - "bytes" - "fmt" - - enc "github.com/named-data/ndnd/std/encoding" - spec "github.com/named-data/ndnd/std/ndn/spec_2022" - spec_svs "github.com/named-data/ndnd/std/ndn/svs/v3" -) - -const ( - syncDataKeyword = "svs" - fullVectorKeyword = "sv" -) - -// deriveFullVectorPrefix maps SyncDataName (.../32=svs) to the published full-vector prefix (.../32=sv). -func deriveFullVectorPrefix(syncDataName enc.Name) enc.Name { - if len(syncDataName) == 0 { - return nil - } - base := syncDataName - if base.At(-1).IsKeyword(syncDataKeyword) { - base = base.Prefix(-1) - } - return base.Append(enc.NewKeywordComponent(fullVectorKeyword)) -} - -// pullRefFromSyncDataWire derives the retrievable full-vector prefix from a signed Sync Data wire. -func pullRefFromSyncDataWire(dataWire enc.Wire) enc.Name { - name, err := syncDataNameFromWire(dataWire) - if err != nil || len(name) == 0 { - return nil - } - if name.At(-1).IsVersion() { - name = name.Prefix(-1) - } - return deriveFullVectorPrefix(name) -} - -func syncDataNameFromWire(dataWire enc.Wire) (enc.Name, error) { - data, _, err := spec.Spec{}.ReadData(enc.NewWireView(dataWire)) - if err != nil { - return nil, err - } - return data.Name(), nil -} - -// buildAnnounceSvsData builds announce-only SvsData (mhash + SvsDataRef). -func buildAnnounceSvsData(state SvMap[uint64], ref enc.Name) *spec_svs.SvsData { - return &spec_svs.SvsData{ - MemberSetHash: ComputeMhash(state), - SvsDataRef: ref, - } -} - -// buildInlineFullFromState builds inline FULL SvsData for the complete local state. -func buildInlineFullFromState(state SvMap[uint64]) *spec_svs.SvsData { - sv := state.Encode(func(seq uint64) uint64 { return seq }) - return buildInlineSvsData(state, spec_svs.VectorTypeFull, sv) -} - -// inlineFullSize returns encoded inline FULL SvsData size for threshold checks. -func inlineFullSize(state SvMap[uint64]) int { - return inlineSvsDataSize(buildInlineFullFromState(state)) -} - -// shouldUseAnnouncePull reports whether the sender should publish at 32=sv and announce-only Sync Data. -// Large-group announce+pull is disabled when threshold <= 0 (legacy inline-FULL-only mode). -func shouldUseAnnouncePull(reason syncSendReason, threshold int, state SvMap[uint64]) bool { - if threshold <= 0 { - return false - } - if reason == syncSendRecovery { - return true - } - if reason == syncSendPublication { - return false - } - return exceedsSyncThreshold(threshold, inlineFullSize(state)) -} - -// isTrustedSvsDataRef reports whether ref is under the sender's published full-vector prefix. -func isTrustedSvsDataRef(ref, senderFullVectorPrefix enc.Name) bool { - if len(ref) == 0 || len(senderFullVectorPrefix) == 0 { - return false - } - return senderFullVectorPrefix.IsPrefix(ref) -} - -// membershipTupleCount returns the number of (Name, BootstrapTime) pairs in state. -func membershipTupleCount(m SvMap[uint64]) int { - n := 0 - for _, vals := range m { - n += len(vals) - } - return n -} - -// membershipContains reports whether every membership tuple in inner exists in outer. -func membershipContains(outer, inner SvMap[uint64]) bool { - for hash, vals := range inner { - for _, v := range vals { - found := false - for _, ov := range outer[hash] { - if ov.Boot == v.Boot { - found = true - break - } - } - if !found { - return false - } - } - } - return true -} - -// stateVectorToMap converts a StateVector to a map for membership checks. -func stateVectorToMap(sv *spec_svs.StateVector) SvMap[uint64] { - m := NewSvMap[uint64](len(sv.Entries)) - for _, node := range sv.Entries { - hash := node.Name.TlvStr() - for _, entry := range node.SeqNoEntries { - m.Set(hash, entry.BootstrapTime, entry.SeqNo) - } - } - return m -} - -// parseFullVectorContent parses inline FULL SvsData from published full-vector Data content. -func parseFullVectorContent(content []byte) (*spec_svs.SvsData, error) { - params, err := spec_svs.ParseSvsData(enc.NewBufferView(content), false) - if err != nil { - return nil, err - } - if params.StateVector == nil { - return nil, fmt.Errorf("full vector content has no StateVector") - } - if vt, ok := params.VectorType.Get(); ok && vt != spec_svs.VectorTypeFull { - return nil, fmt.Errorf("full vector VectorType=%d, want FULL", vt) - } - if len(params.MemberSetHash) > 0 { - computed := ComputeMhash(stateVectorToMap(params.StateVector)) - if !bytes.Equal(params.MemberSetHash, computed) { - return nil, fmt.Errorf("full vector mhash mismatch") - } - } - return params, nil -} diff --git a/std/sync/svs_debug.go b/std/sync/svs_debug.go deleted file mode 100644 index 00053df1..00000000 --- a/std/sync/svs_debug.go +++ /dev/null @@ -1,89 +0,0 @@ -package sync - -import ( - enc "github.com/named-data/ndnd/std/encoding" - "github.com/named-data/ndnd/std/log" - spec_svs "github.com/named-data/ndnd/std/ndn/svs/v3" -) - -func syncSendReasonString(reason syncSendReason) string { - switch reason { - case syncSendPublication: - return "publication" - case syncSendPeriodic: - return "periodic" - case syncSendRecovery: - return "recovery" - default: - return "other" - } -} - -func logSyncSend(s *SvSync, reason syncSendReason, data *spec_svs.SvsData) { - if data == nil { - return - } - if data.StateVector == nil && len(data.SvsDataRef) > 0 { - log.Debug(s, "sync send announce-only", - "reason", syncSendReasonString(reason), - "ref", data.SvsDataRef, - "bytes", inlineSvsDataSize(data), - ) - return - } - - mode := syncWireModeLabel(data) - entries := 0 - if data.StateVector != nil { - entries = len(data.StateVector.Entries) - } - log.Debug(s, "sync send inline", - "reason", syncSendReasonString(reason), - "mode", mode, - "entries", entries, - "bytes", inlineSvsDataSize(data), - ) -} - -func logSyncRecv(s *SvSync, params *spec_svs.SvsData, syncDataName enc.Name) { - if params == nil { - return - } - if params.StateVector == nil && len(params.SvsDataRef) > 0 { - log.Debug(s, "sync recv announce-only", - "from", syncDataName, - "ref", params.SvsDataRef, - ) - return - } - - mode := syncWireModeLabel(params) - entries := 0 - if params.StateVector != nil { - entries = len(params.StateVector.Entries) - } - log.Debug(s, "sync recv inline", - "from", syncDataName, - "mode", mode, - "entries", entries, - ) -} - -func syncWireModeLabel(data *spec_svs.SvsData) string { - if data.StateVector == nil { - if len(data.SvsDataRef) > 0 { - return "announce-only" - } - return "empty" - } - if vt, ok := data.VectorType.Get(); ok { - if vt == spec_svs.VectorTypePartial { - return "PARTIAL" - } - return "FULL" - } - if len(data.MemberSetHash) == 0 { - return "legacy FULL" - } - return "FULL" -} diff --git a/std/sync/svs_encode.go b/std/sync/svs_encode.go index 7c8619a9..c2f30004 100644 --- a/std/sync/svs_encode.go +++ b/std/sync/svs_encode.go @@ -56,34 +56,49 @@ func exceedsSyncThreshold(threshold, size int) bool { return threshold > 0 && size > threshold } +// buildInlineFullFromState builds inline FULL SvsData for the complete local state. +func buildInlineFullFromState(state SvMap[uint64]) *spec_svs.SvsData { + sv := state.Encode(func(seq uint64) uint64 { return seq }) + return buildInlineSvsData(state, spec_svs.VectorTypeFull, sv) +} + +// inlineFullSize returns encoded inline FULL SvsData size for threshold checks. +func inlineFullSize(state SvMap[uint64]) int { + return inlineSvsDataSize(buildInlineFullFromState(state)) +} + +// svsSendInput carries everything needed to build inline Sync Data for send. +type svsSendInput struct { + State SvMap[uint64] + Reason syncSendReason + Threshold int + Sender enc.Name + Repair []enc.Name + Propagation []enc.Name + Mtime map[string]time.Time +} + // buildSvsDataForSend picks inline FULL or PARTIAL SvsData for an outgoing Sync message. -func buildSvsDataForSend( - state SvMap[uint64], - reason syncSendReason, - threshold int, - sender enc.Name, - repair, propagation []enc.Name, - mtime map[string]time.Time, -) *spec_svs.SvsData { - if threshold <= 0 { - return buildLegacySvsData(state) - } - - fullSv := state.Encode(func(seq uint64) uint64 { return seq }) - fullData := buildInlineSvsData(state, spec_svs.VectorTypeFull, fullSv) - - if reason != syncSendPublication || !exceedsSyncThreshold(threshold, inlineSvsDataSize(fullData)) { +func buildSvsDataForSend(in svsSendInput) *spec_svs.SvsData { + if in.Threshold <= 0 { + return buildLegacySvsData(in.State) + } + + fullSv := in.State.Encode(func(seq uint64) uint64 { return seq }) + fullData := buildInlineSvsData(in.State, spec_svs.VectorTypeFull, fullSv) + + if in.Reason != syncSendPublication || !exceedsSyncThreshold(in.Threshold, inlineSvsDataSize(fullData)) { return fullData } - partialSv := encodePartialStateVector(state, PartialEncodeOpts{ - Sender: sender, - Threshold: threshold, - Repair: repair, - Propagation: propagation, - Mtime: mtime, + partialSv := encodePartialStateVector(in.State, PartialEncodeOpts{ + Sender: in.Sender, + Threshold: in.Threshold, + Repair: in.Repair, + Propagation: in.Propagation, + Mtime: in.Mtime, }) - return buildInlineSvsData(state, spec_svs.VectorTypePartial, partialSv) + return buildInlineSvsData(in.State, spec_svs.VectorTypePartial, partialSv) } // encodePartialStateVector builds a PARTIAL StateVector per spec Section 4.2. diff --git a/std/sync/svs_encode_test.go b/std/sync/svs_encode_test.go index accf9f3c..e3492c56 100644 --- a/std/sync/svs_encode_test.go +++ b/std/sync/svs_encode_test.go @@ -145,13 +145,17 @@ func TestBuildSvsDataForSendPublicationPartial(t *testing.T) { full := buildInlineSvsData(m, spec_svs.VectorTypeFull, m.Encode(func(s uint64) uint64 { return s })) threshold := inlineSvsDataSize(full) / 2 - pub := buildSvsDataForSend(m, syncSendPublication, threshold, alice, nil, nil, nil) + pub := buildSvsDataForSend(svsSendInput{ + State: m, Reason: syncSendPublication, Threshold: threshold, Sender: alice, + }) vt, ok := pub.VectorType.Get() require.True(t, ok) require.Equal(t, spec_svs.VectorTypePartial, vt) require.Less(t, len(pub.StateVector.Entries), len(full.StateVector.Entries)) - periodic := buildSvsDataForSend(m, syncSendPeriodic, threshold, alice, nil, nil, nil) + periodic := buildSvsDataForSend(svsSendInput{ + State: m, Reason: syncSendPeriodic, Threshold: threshold, Sender: alice, + }) vt, ok = periodic.VectorType.Get() require.True(t, ok) require.Equal(t, spec_svs.VectorTypeFull, vt) diff --git a/std/sync/svs_test_mesh.go b/std/sync/svs_mesh_test.go similarity index 57% rename from std/sync/svs_test_mesh.go rename to std/sync/svs_mesh_test.go index 439ff0da..9d44463d 100644 --- a/std/sync/svs_test_mesh.go +++ b/std/sync/svs_mesh_test.go @@ -3,12 +3,12 @@ package sync import ( "fmt" "sync" + "sync/atomic" "testing" "time" enc "github.com/named-data/ndnd/std/encoding" basic_engine "github.com/named-data/ndnd/std/engine/basic" - "github.com/named-data/ndnd/std/engine/face" "github.com/named-data/ndnd/std/ndn" spec_svs "github.com/named-data/ndnd/std/ndn/svs/v3" "github.com/named-data/ndnd/std/object" @@ -17,8 +17,112 @@ import ( "github.com/stretchr/testify/require" ) +// In-process multi-node SVS mesh for integration tests (svs_test.go). +// +// Each node runs a real ndnd engine + SvSync instance. A test-only multicast hub +// delivers packets between nodes so Sync Interests propagate without NFD/mininet. +// This exercises small-group sync, PARTIAL publication, and announce+pull recovery. + const testMeshBoot = uint64(1_700_000_000) +type meshMulticastHub struct { + mu sync.Mutex + faces []*meshMulticastFace +} + +type meshMulticastFace struct { + hub *meshMulticastHub + id int + running atomic.Bool + local bool + onPkt func(frame []byte) + onError func(err error) + onUp sync.Map + onDown sync.Map + onUpID int + onDownID int +} + +func newMeshMulticastHub() *meshMulticastHub { + return &meshMulticastHub{} +} + +func (h *meshMulticastHub) newFace() *meshMulticastFace { + h.mu.Lock() + defer h.mu.Unlock() + f := &meshMulticastFace{hub: h, id: len(h.faces), local: true} + h.faces = append(h.faces, f) + return f +} + +func (f *meshMulticastFace) String() string { + return fmt.Sprintf("mesh-multicast-face-%d", f.id) +} + +func (f *meshMulticastFace) IsRunning() bool { return f.running.Load() } +func (f *meshMulticastFace) IsLocal() bool { return f.local } +func (f *meshMulticastFace) OnPacket(fn func([]byte)) { f.onPkt = fn } +func (f *meshMulticastFace) OnError(fn func(error)) { f.onError = fn } + +func (f *meshMulticastFace) OnUp(fn func()) (cancel func()) { + id := f.onUpID + f.onUpID++ + f.onUp.Store(id, fn) + return func() { f.onUp.Delete(id) } +} + +func (f *meshMulticastFace) OnDown(fn func()) (cancel func()) { + id := f.onDownID + f.onDownID++ + f.onDown.Store(id, fn) + return func() { f.onDown.Delete(id) } +} + +func (f *meshMulticastFace) Open() error { + if f.onError == nil || f.onPkt == nil { + return fmt.Errorf("face callbacks are not set") + } + if f.running.Load() { + return fmt.Errorf("face is already running") + } + f.running.Store(true) + f.onUp.Range(func(_, cb any) bool { + cb.(func())() + return true + }) + return nil +} + +func (f *meshMulticastFace) Close() error { + if !f.running.Swap(false) { + return fmt.Errorf("face is not running") + } + return nil +} + +func (f *meshMulticastFace) Send(pkt enc.Wire) error { + if !f.running.Load() { + return fmt.Errorf("face is not running") + } + f.hub.broadcast(f.id, pkt.Join()) + return nil +} + +func (h *meshMulticastHub) broadcast(from int, frame []byte) { + h.mu.Lock() + peers := append([]*meshMulticastFace(nil), h.faces...) + h.mu.Unlock() + + for _, peer := range peers { + if peer.id == from || !peer.running.Load() || peer.onPkt == nil { + continue + } + frameCopy := make([]byte, len(frame)) + copy(frameCopy, frame) + peer.onPkt(frameCopy) + } +} + type testMeshNode struct { id string producer enc.Name @@ -31,7 +135,7 @@ type testMeshNode struct { type testMesh struct { t *testing.T - hub *face.MulticastHub + hub *meshMulticastHub timer *basic_engine.DummyTimer group enc.Name nodes map[string]*testMeshNode @@ -43,7 +147,7 @@ func newTestMesh(t *testing.T, ids []string, configure func(id string, opts *SvS mesh := &testMesh{ t: t, - hub: face.NewMulticastHub(), + hub: newMeshMulticastHub(), timer: basic_engine.NewDummyTimer(), group: tu.NoErr(enc.NameFromStr("/test/svs-mesh/32=svs")), nodes: make(map[string]*testMeshNode, len(ids)), @@ -55,7 +159,7 @@ func newTestMesh(t *testing.T, ids []string, configure func(id string, opts *SvS Append(enc.NewTimestampComponent(testMeshBoot)). Append(enc.NewKeywordComponent("svs")) - f := mesh.hub.NewFace() + f := mesh.hub.newFace() engine := basic_engine.NewEngine(f, mesh.timer) store := storage.NewMemoryStore() client := object.NewClient(engine, store, nil) diff --git a/std/sync/svs_pull.go b/std/sync/svs_pull.go index 4734b770..a7d664db 100644 --- a/std/sync/svs_pull.go +++ b/std/sync/svs_pull.go @@ -7,10 +7,65 @@ import ( enc "github.com/named-data/ndnd/std/encoding" "github.com/named-data/ndnd/std/log" "github.com/named-data/ndnd/std/ndn" + spec "github.com/named-data/ndnd/std/ndn/spec_2022" spec_svs "github.com/named-data/ndnd/std/ndn/svs/v3" "github.com/named-data/ndnd/std/types/optional" ) +const ( + syncDataKeyword = "svs" + fullVectorKeyword = "sv" +) + +// deriveFullVectorPrefix maps SyncDataName (.../32=svs) to the published full-vector prefix (.../32=sv). +func deriveFullVectorPrefix(syncDataName enc.Name) enc.Name { + if len(syncDataName) == 0 { + return nil + } + base := syncDataName + if base.At(-1).IsKeyword(syncDataKeyword) { + base = base.Prefix(-1) + } + return base.Append(enc.NewKeywordComponent(fullVectorKeyword)) +} + +func pullRefFromSyncDataWire(dataWire enc.Wire) enc.Name { + data, _, err := spec.Spec{}.ReadData(enc.NewWireView(dataWire)) + if err != nil { + return nil + } + name := data.Name() + if len(name) == 0 { + return nil + } + if name.At(-1).IsVersion() { + name = name.Prefix(-1) + } + return deriveFullVectorPrefix(name) +} + +func buildAnnounceSvsData(state SvMap[uint64], ref enc.Name) *spec_svs.SvsData { + return &spec_svs.SvsData{ + MemberSetHash: ComputeMhash(state), + SvsDataRef: ref, + } +} + +// shouldUseAnnouncePull reports whether the sender should publish at 32=sv and announce-only Sync Data. +// Large-group announce+pull is disabled when threshold <= 0 (legacy inline-FULL-only mode). +func shouldUseAnnouncePull(reason syncSendReason, threshold int, state SvMap[uint64]) bool { + if threshold <= 0 { + return false + } + if reason == syncSendRecovery { + return true + } + if reason == syncSendPublication { + return false + } + return exceedsSyncThreshold(threshold, inlineFullSize(state)) +} + // publishFullVectorData produces retrievable inline FULL SvsData at .../32=sv/. func (s *SvSync) publishFullVectorData(state SvMap[uint64]) (enc.Name, error) { if len(s.fullVectorPrefix) == 0 { @@ -75,6 +130,37 @@ func (s *SvSync) onPulledFullVector(content []byte) { } } +func parseFullVectorContent(content []byte) (*spec_svs.SvsData, error) { + params, err := spec_svs.ParseSvsData(enc.NewBufferView(content), false) + if err != nil { + return nil, err + } + if params.StateVector == nil { + return nil, fmt.Errorf("full vector content has no StateVector") + } + if vt, ok := params.VectorType.Get(); ok && vt != spec_svs.VectorTypeFull { + return nil, fmt.Errorf("full vector VectorType=%d, want FULL", vt) + } + if len(params.MemberSetHash) > 0 { + computed := ComputeMhash(stateVectorToMap(params.StateVector)) + if !bytes.Equal(params.MemberSetHash, computed) { + return nil, fmt.Errorf("full vector mhash mismatch") + } + } + return params, nil +} + +func stateVectorToMap(sv *spec_svs.StateVector) SvMap[uint64] { + m := NewSvMap[uint64](len(sv.Entries)) + for _, node := range sv.Entries { + hash := node.Name.TlvStr() + for _, entry := range node.SeqNoEntries { + m.Set(hash, entry.BootstrapTime, entry.SeqNo) + } + } + return m +} + // sendRecoveryAnnounce publishes at 32=sv and emits announce-only Sync Data (mhash recovery). func (s *SvSync) sendRecoveryAnnounce() { if !s.running.Load() || s.o.Passive { @@ -104,8 +190,9 @@ func (s *SvSync) handleMhashMismatch(args svSyncRecvSvArgs, recvSv SvMap[uint64] trustPrefix = s.fullVectorPrefix } - if membershipContains(s.state, recvSv) && membershipTupleCount(s.state) > membershipTupleCount(recvSv) { - log.Debug(s, "sync recovery path", "action", "sender announce", "localMembers", membershipTupleCount(s.state), "remoteMembers", membershipTupleCount(recvSv)) + localTuples, remoteTuples := membershipTupleCount(s.state), membershipTupleCount(recvSv) + if localTuples > remoteTuples && membershipContains(s.state, recvSv) { + log.Debug(s, "sync recovery path", "action", "sender announce", "localMembers", localTuples, "remoteMembers", remoteTuples) go s.sendRecoveryAnnounce() return } @@ -118,3 +205,36 @@ func (s *SvSync) handleMhashMismatch(args svSyncRecvSvArgs, recvSv SvMap[uint64] log.Debug(s, "sync recovery path", "action", "pull", "ref", args.svsDataRef) go s.pullFullVector(args.svsDataRef, trustPrefix) } + +func membershipContains(outer, inner SvMap[uint64]) bool { + for hash, vals := range inner { + for _, v := range vals { + found := false + for _, ov := range outer[hash] { + if ov.Boot == v.Boot { + found = true + break + } + } + if !found { + return false + } + } + } + return true +} + +func membershipTupleCount(m SvMap[uint64]) int { + n := 0 + for _, vals := range m { + n += len(vals) + } + return n +} + +func isTrustedSvsDataRef(ref, senderFullVectorPrefix enc.Name) bool { + if len(ref) == 0 || len(senderFullVectorPrefix) == 0 { + return false + } + return senderFullVectorPrefix.IsPrefix(ref) +} From 60474b33fb8bba3e1d166493e9f36471bdb0f612 Mon Sep 17 00:00:00 2001 From: Taranum Wasu Date: Sat, 4 Jul 2026 22:39:07 +0530 Subject: [PATCH 09/24] style: align SVS polish with ndnd conventions Remove dead large-sync debug flag and tuName helper, clarify threshold-0 comments, trim pull-path debug logging, rename announce tests to svs_pull_test, and test pullRefFromSyncDataWire with a signed Sync Data wire. Co-authored-by: Cursor --- std/examples/svs/large-sync/main.go | 50 ++++++++----------- std/sync/svs.go | 3 +- std/sync/svs_encode.go | 4 +- std/sync/svs_mhash.go | 3 +- std/sync/svs_pull.go | 19 ++----- ...{svs_announce_test.go => svs_pull_test.go} | 17 ++++++- std/sync/svs_test.go | 4 +- 7 files changed, 46 insertions(+), 54 deletions(-) rename std/sync/{svs_announce_test.go => svs_pull_test.go} (89%) diff --git a/std/examples/svs/large-sync/main.go b/std/examples/svs/large-sync/main.go index a09da13f..8ab15d9e 100644 --- a/std/examples/svs/large-sync/main.go +++ b/std/examples/svs/large-sync/main.go @@ -15,44 +15,43 @@ import ( "github.com/named-data/ndnd/std/sync" ) -// Large-group SVS demo: PARTIAL on publication, announce+pull on periodic sync. -// -// Before running, configure NFD multicast on the sync prefix, e.g.: -// -// ndnd fw strategy-set prefix=/ndn/svs/32=svs strategy=/localhost/nfd/strategy/multicast +// (AI GENERATED DESCRIPTION): Launches an SVSync client with configurable SyncVectorThreshold for large-group PARTIAL and announce+pull behavior. func main() { - threshold := flag.Int("threshold", 1200, "SyncVectorThreshold in bytes (0 = unlimited inline FULL)") - debug := flag.Bool("debug", false, "log SVS wire modes (PARTIAL, announce+pull, pull recovery)") + // Before running this example, make sure the strategy is correctly setup + // to multicast for the sync prefix. For example, using the following: + // + // ndnd fw strategy-set prefix=/ndn/svs/32=svs strategy=/localhost/nfd/strategy/multicast + // + + threshold := flag.Int("threshold", 1200, "SyncVectorThreshold in bytes (0 = legacy inline FULL only)") flag.Parse() if flag.NArg() < 1 { - fmt.Fprintf(os.Stderr, "Usage: %s [flags] \n", os.Args[0]) + fmt.Fprintf(os.Stderr, "Usage: %s [-threshold N] \n", os.Args[0]) os.Exit(1) } - if *debug { - log.Default().SetLevel(log.LevelDebug) - } - name, err := enc.NameFromStr(flag.Arg(0)) if err != nil { - log.Fatal(nil, "Invalid producer name", "name", flag.Arg(0), "err", err) + log.Fatal(nil, "Invalid node ID", "name", flag.Arg(0), "err", err) } app := engine.NewBasicEngine(engine.NewDefaultFace()) - if err := app.Start(); err != nil { + err = app.Start() + if err != nil { log.Fatal(nil, "Unable to start engine", "err", err) } defer app.Stop() store := storage.NewMemoryStore() client := object.NewClient(app, store, nil) - if err := client.Start(); err != nil { + err = client.Start() + if err != nil { log.Fatal(nil, "Unable to start object client", "err", err) } defer client.Stop() - group := tuName("/ndn/svs") + group, _ := enc.NameFromStr("/ndn/svs") boot := uint64(time.Now().Unix()) syncDataName := name. Append(enc.NewTimestampComponent(boot)). @@ -65,14 +64,15 @@ func main() { BootTime: boot, SyncVectorThreshold: *threshold, OnUpdate: func(ssu sync.SvSyncUpdate) { - log.Info(nil, "SVS update", "name", ssu.Name, "boot", ssu.Boot, "low", ssu.Low, "high", ssu.High) + log.Info(nil, "Received update", "update", ssu) }, }) client.AnnouncePrefix(ndn.Announcement{Name: group}) defer client.WithdrawPrefix(group, nil) - if err := svsync.Start(); err != nil { + err = svsync.Start() + if err != nil { log.Fatal(nil, "Unable to start SvSync", "err", err) } defer svsync.Stop() @@ -80,21 +80,11 @@ func main() { log.Info(nil, "Large-group SVS started", "name", name, "threshold", *threshold, - "debug", *debug, - "syncData", syncDataName, - "fullVector", syncDataName.Prefix(-1).Append(enc.NewKeywordComponent("sv"))) + "syncData", syncDataName) ticker := time.NewTicker(3 * time.Second) for range ticker.C { seq := svsync.IncrSeqNo(name) - log.Info(nil, "Published sequence number", "seq", seq) - } -} - -func tuName(s string) enc.Name { - name, err := enc.NameFromStr(s) - if err != nil { - panic(err) + log.Info(nil, "Published new sequence number", "seq", seq) } - return name } diff --git a/std/sync/svs.go b/std/sync/svs.go index 6da75e33..3dc7caf5 100644 --- a/std/sync/svs.go +++ b/std/sync/svs.go @@ -77,7 +77,8 @@ type SvSyncOpts struct { // IgnoreValidity ignores validity period in the validation chain IgnoreValidity optional.Optional[bool] - // SyncVectorThreshold is the max inline SvsData size (bytes). 0 = unlimited (legacy SVS v3). + // SyncVectorThreshold is the max inline SvsData size (bytes). + // 0 = legacy mode (StateVector-only wire, no mhash or announce+pull recovery). SyncVectorThreshold int } diff --git a/std/sync/svs_encode.go b/std/sync/svs_encode.go index c2f30004..c3a1f343 100644 --- a/std/sync/svs_encode.go +++ b/std/sync/svs_encode.go @@ -51,7 +51,7 @@ func inlineSvsDataSize(data *spec_svs.SvsData) int { } // exceedsSyncThreshold reports whether size is over the configured inline budget. -// threshold 0 means unlimited (always inline). +// threshold 0 disables the large-group size limit (legacy mode). func exceedsSyncThreshold(threshold, size int) bool { return threshold > 0 && size > threshold } @@ -101,7 +101,7 @@ func buildSvsDataForSend(in svsSendInput) *spec_svs.SvsData { return buildInlineSvsData(in.State, spec_svs.VectorTypePartial, partialSv) } -// encodePartialStateVector builds a PARTIAL StateVector per spec Section 4.2. +// encodePartialStateVector builds a PARTIAL StateVector for new publication. // Entry [0] is the sender; entries [1..n] are in NDN canonical order. func encodePartialStateVector(state SvMap[uint64], opts PartialEncodeOpts) *spec_svs.StateVector { seq := func(v uint64) uint64 { return v } diff --git a/std/sync/svs_mhash.go b/std/sync/svs_mhash.go index 74b9a445..510c16cc 100644 --- a/std/sync/svs_mhash.go +++ b/std/sync/svs_mhash.go @@ -13,8 +13,7 @@ type membershipTuple struct { boot uint64 } -// ComputeMhash returns the membership hash over all (Name, BootstrapTime) pairs -// in state, per SVS v3 large-group revision spec Section 3.3. +// ComputeMhash returns the membership hash over all (Name, BootstrapTime) pairs in state. func ComputeMhash(state SvMap[uint64]) []byte { members := make([]membershipTuple, 0) for name, vals := range state.Iter() { diff --git a/std/sync/svs_pull.go b/std/sync/svs_pull.go index a7d664db..77f9b163 100644 --- a/std/sync/svs_pull.go +++ b/std/sync/svs_pull.go @@ -52,7 +52,7 @@ func buildAnnounceSvsData(state SvMap[uint64], ref enc.Name) *spec_svs.SvsData { } // shouldUseAnnouncePull reports whether the sender should publish at 32=sv and announce-only Sync Data. -// Large-group announce+pull is disabled when threshold <= 0 (legacy inline-FULL-only mode). +// Large-group announce+pull is disabled when threshold <= 0 (legacy mode). func shouldUseAnnouncePull(reason syncSendReason, threshold int, state SvMap[uint64]) bool { if threshold <= 0 { return false @@ -89,7 +89,6 @@ func (s *SvSync) pullFullVector(ref enc.Name, trustPrefix enc.Name) { log.Warn(s, "pullFullVector rejected untrusted SvsDataRef", "ref", ref, "trust", trustPrefix) return } - log.Debug(s, "sync pull start", "ref", ref) s.o.Client.ConsumeExt(ndn.ConsumeExtArgs{ Name: ref.Clone(), @@ -117,11 +116,6 @@ func (s *SvSync) onPulledFullVector(content []byte) { log.Warn(s, "onPulledFullVector parse failed", "err", err) return } - entries := 0 - if params.StateVector != nil { - entries = len(params.StateVector.Entries) - } - log.Debug(s, "sync pull merged", "entries", entries, "mode", "FULL") s.recvSv <- svSyncRecvSvArgs{ sv: params.StateVector, @@ -166,14 +160,13 @@ func (s *SvSync) sendRecoveryAnnounce() { if !s.running.Load() || s.o.Passive { return } - log.Debug(s, "sync send recovery announce") wire := s.encodeSyncData(syncSendRecovery, enc.Name{}) s.sendSyncInterestWith(wire) } -// handleMhashMismatch schedules announce or pull recovery per spec Section 5.6. +// handleMhashMismatch schedules announce or pull recovery on membership mismatch. func (s *SvSync) handleMhashMismatch(args svSyncRecvSvArgs, recvSv SvMap[uint64]) { - // Legacy mode (threshold 0): inline FULL is merged above; large-group recovery is off. + // [Spec] Legacy mode (threshold 0): inline FULL is merged above; large-group recovery is off. if s.o.SyncVectorThreshold <= 0 { return } @@ -183,8 +176,6 @@ func (s *SvSync) handleMhashMismatch(args svSyncRecvSvArgs, recvSv SvMap[uint64] return } - log.Debug(s, "mhash mismatch with sender", "local", localMhash, "remote", args.mhash) - trustPrefix := pullRefFromSyncDataWire(args.data) if len(trustPrefix) == 0 { trustPrefix = s.fullVectorPrefix @@ -192,17 +183,15 @@ func (s *SvSync) handleMhashMismatch(args svSyncRecvSvArgs, recvSv SvMap[uint64] localTuples, remoteTuples := membershipTupleCount(s.state), membershipTupleCount(recvSv) if localTuples > remoteTuples && membershipContains(s.state, recvSv) { - log.Debug(s, "sync recovery path", "action", "sender announce", "localMembers", localTuples, "remoteMembers", remoteTuples) go s.sendRecoveryAnnounce() return } - // Inline FULL is already merged in onReceiveStateVector (spec §5.6 step 2). + // [Spec] Inline FULL is already merged in onReceiveStateVector. // Pull only when the sender provided a retrievable SvsDataRef (announce-only sync). if len(args.svsDataRef) == 0 { return } - log.Debug(s, "sync recovery path", "action", "pull", "ref", args.svsDataRef) go s.pullFullVector(args.svsDataRef, trustPrefix) } diff --git a/std/sync/svs_announce_test.go b/std/sync/svs_pull_test.go similarity index 89% rename from std/sync/svs_announce_test.go rename to std/sync/svs_pull_test.go index 1eae43f3..56e9d4ed 100644 --- a/std/sync/svs_announce_test.go +++ b/std/sync/svs_pull_test.go @@ -5,7 +5,11 @@ import ( "time" enc "github.com/named-data/ndnd/std/encoding" + "github.com/named-data/ndnd/std/ndn" + spec "github.com/named-data/ndnd/std/ndn/spec_2022" spec_svs "github.com/named-data/ndnd/std/ndn/svs/v3" + sig "github.com/named-data/ndnd/std/security/signer" + "github.com/named-data/ndnd/std/types/optional" tu "github.com/named-data/ndnd/std/utils/testutils" "github.com/stretchr/testify/require" ) @@ -21,8 +25,17 @@ func TestDeriveFullVectorPrefix(t *testing.T) { func TestPullRefFromSyncDataWire(t *testing.T) { tu.SetT(t) - syncDataName := tu.NoErr(enc.NameFromStr("/ndn/svs/alice/1700000000/32=svs/12345")) - ref := deriveFullVectorPrefix(syncDataName.Prefix(-1)) + syncDataName := tu.NoErr(enc.NameFromStr("/ndn/svs/alice/1700000000/32=svs")). + Append(enc.NewVersionComponent(12345)) + dataWire, err := spec.Spec{}.MakeData( + syncDataName, + &ndn.DataConfig{ContentType: optional.Some(ndn.ContentTypeBlob)}, + enc.Wire{enc.Buffer{0x01}}, + sig.NewSha256Signer(), + ) + require.NoError(t, err) + + ref := pullRefFromSyncDataWire(dataWire.Wire) require.Equal(t, "/ndn/svs/alice/1700000000/32=sv", ref.String()) } diff --git a/std/sync/svs_test.go b/std/sync/svs_test.go index 2c833437..a85de1c0 100644 --- a/std/sync/svs_test.go +++ b/std/sync/svs_test.go @@ -26,7 +26,7 @@ func TestSvSyncSmallGroup(t *testing.T) { require.EqualValues(t, seq, mesh.node("bob").svs.GetSeqNo(mesh.node("alice").producer)) require.EqualValues(t, seq, mesh.node("carol").svs.GetSeqNo(mesh.node("alice").producer)) - // Default threshold 0 → legacy wire (StateVector only, no mhash). + // Default threshold 0: legacy wire (StateVector only, no mhash). alice := mesh.node("alice") alice.svs.mutex.Lock() legacy := buildLegacySvsData(alice.svs.state) @@ -195,7 +195,7 @@ func TestSvSyncMhashMismatchSenderAnnounce(t *testing.T) { bobOnly.Set(mesh.node("bob").producer.TlvStr(), testMeshBoot, 1) partialSv := bobOnly.Encode(func(s uint64) uint64 { return s }) - // Bob sends PARTIAL; Alice has strict superset membership → announce recovery. + // Bob sends PARTIAL; Alice has strict superset membership and announces recovery. alice.onReceiveStateVector(svSyncRecvSvArgs{ sv: partialSv, vectorType: optional.Some(spec_svs.VectorTypePartial), From 7323078cfc7c8faf5d6d6841c6fa8615d2e0242e Mon Sep 17 00:00:00 2001 From: Taranum Wasu Date: Sat, 4 Jul 2026 22:53:51 +0530 Subject: [PATCH 10/24] style: remove AI-generated comment from large-sync example Co-authored-by: Cursor --- std/examples/svs/large-sync/main.go | 1 - 1 file changed, 1 deletion(-) diff --git a/std/examples/svs/large-sync/main.go b/std/examples/svs/large-sync/main.go index 8ab15d9e..3437e04c 100644 --- a/std/examples/svs/large-sync/main.go +++ b/std/examples/svs/large-sync/main.go @@ -15,7 +15,6 @@ import ( "github.com/named-data/ndnd/std/sync" ) -// (AI GENERATED DESCRIPTION): Launches an SVSync client with configurable SyncVectorThreshold for large-group PARTIAL and announce+pull behavior. func main() { // Before running this example, make sure the strategy is correctly setup // to multicast for the sync prefix. For example, using the following: From ffcc89050171869080708dd026b43abf69b73f34 Mon Sep 17 00:00:00 2001 From: Taranum Wasu Date: Mon, 6 Jul 2026 13:22:13 +0530 Subject: [PATCH 11/24] fix lint: align meshMulticastFace field columns Co-authored-by: Cursor --- std/sync/svs_mesh_test.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/std/sync/svs_mesh_test.go b/std/sync/svs_mesh_test.go index 9d44463d..8ae68df6 100644 --- a/std/sync/svs_mesh_test.go +++ b/std/sync/svs_mesh_test.go @@ -31,15 +31,15 @@ type meshMulticastHub struct { } type meshMulticastFace struct { - hub *meshMulticastHub - id int - running atomic.Bool - local bool - onPkt func(frame []byte) - onError func(err error) - onUp sync.Map - onDown sync.Map - onUpID int + hub *meshMulticastHub + id int + running atomic.Bool + local bool + onPkt func(frame []byte) + onError func(err error) + onUp sync.Map + onDown sync.Map + onUpID int onDownID int } @@ -59,10 +59,10 @@ func (f *meshMulticastFace) String() string { return fmt.Sprintf("mesh-multicast-face-%d", f.id) } -func (f *meshMulticastFace) IsRunning() bool { return f.running.Load() } -func (f *meshMulticastFace) IsLocal() bool { return f.local } +func (f *meshMulticastFace) IsRunning() bool { return f.running.Load() } +func (f *meshMulticastFace) IsLocal() bool { return f.local } func (f *meshMulticastFace) OnPacket(fn func([]byte)) { f.onPkt = fn } -func (f *meshMulticastFace) OnError(fn func(error)) { f.onError = fn } +func (f *meshMulticastFace) OnError(fn func(error)) { f.onError = fn } func (f *meshMulticastFace) OnUp(fn func()) (cancel func()) { id := f.onUpID From bdaecba61887eb6f021d930832c19d0a7a57173b Mon Sep 17 00:00:00 2001 From: Taranum Wasu Date: Thu, 9 Jul 2026 12:52:25 +0530 Subject: [PATCH 12/24] examples: drop large-sync example Sync API for SvSync is unchanged; pure-sync suffices for svs examples. --- std/examples/svs/large-sync/main.go | 89 ----------------------------- 1 file changed, 89 deletions(-) delete mode 100644 std/examples/svs/large-sync/main.go diff --git a/std/examples/svs/large-sync/main.go b/std/examples/svs/large-sync/main.go deleted file mode 100644 index 3437e04c..00000000 --- a/std/examples/svs/large-sync/main.go +++ /dev/null @@ -1,89 +0,0 @@ -package main - -import ( - "flag" - "fmt" - "os" - "time" - - enc "github.com/named-data/ndnd/std/encoding" - "github.com/named-data/ndnd/std/engine" - "github.com/named-data/ndnd/std/log" - "github.com/named-data/ndnd/std/ndn" - "github.com/named-data/ndnd/std/object" - "github.com/named-data/ndnd/std/object/storage" - "github.com/named-data/ndnd/std/sync" -) - -func main() { - // Before running this example, make sure the strategy is correctly setup - // to multicast for the sync prefix. For example, using the following: - // - // ndnd fw strategy-set prefix=/ndn/svs/32=svs strategy=/localhost/nfd/strategy/multicast - // - - threshold := flag.Int("threshold", 1200, "SyncVectorThreshold in bytes (0 = legacy inline FULL only)") - flag.Parse() - - if flag.NArg() < 1 { - fmt.Fprintf(os.Stderr, "Usage: %s [-threshold N] \n", os.Args[0]) - os.Exit(1) - } - - name, err := enc.NameFromStr(flag.Arg(0)) - if err != nil { - log.Fatal(nil, "Invalid node ID", "name", flag.Arg(0), "err", err) - } - - app := engine.NewBasicEngine(engine.NewDefaultFace()) - err = app.Start() - if err != nil { - log.Fatal(nil, "Unable to start engine", "err", err) - } - defer app.Stop() - - store := storage.NewMemoryStore() - client := object.NewClient(app, store, nil) - err = client.Start() - if err != nil { - log.Fatal(nil, "Unable to start object client", "err", err) - } - defer client.Stop() - - group, _ := enc.NameFromStr("/ndn/svs") - boot := uint64(time.Now().Unix()) - syncDataName := name. - Append(enc.NewTimestampComponent(boot)). - Append(enc.NewKeywordComponent("svs")) - - svsync := sync.NewSvSync(sync.SvSyncOpts{ - Client: client, - GroupPrefix: group, - SyncDataName: syncDataName, - BootTime: boot, - SyncVectorThreshold: *threshold, - OnUpdate: func(ssu sync.SvSyncUpdate) { - log.Info(nil, "Received update", "update", ssu) - }, - }) - - client.AnnouncePrefix(ndn.Announcement{Name: group}) - defer client.WithdrawPrefix(group, nil) - - err = svsync.Start() - if err != nil { - log.Fatal(nil, "Unable to start SvSync", "err", err) - } - defer svsync.Stop() - - log.Info(nil, "Large-group SVS started", - "name", name, - "threshold", *threshold, - "syncData", syncDataName) - - ticker := time.NewTicker(3 * time.Second) - for range ticker.C { - seq := svsync.IncrSeqNo(name) - log.Info(nil, "Published new sequence number", "seq", seq) - } -} From 4c8926bbab8f81160cde722595a74a4cc23958f3 Mon Sep 17 00:00:00 2001 From: Taranum Wasu Date: Thu, 9 Jul 2026 13:00:38 +0530 Subject: [PATCH 13/24] test: consolidate svs tests into svs_test.go Merge svs_encode_test.go, svs_mhash_test.go, svs_pull_test.go into the single svs_test.go. svsmap tests remain separate (testing the generic SvMap type, not the svs protocol). svsmat tests are dropped next. --- std/sync/svs_encode_test.go | 198 ---------------- std/sync/svs_mhash_test.go | 118 ---------- std/sync/svs_pull_test.go | 157 ------------- std/sync/svs_test.go | 445 ++++++++++++++++++++++++++++++++++++ 4 files changed, 445 insertions(+), 473 deletions(-) delete mode 100644 std/sync/svs_encode_test.go delete mode 100644 std/sync/svs_mhash_test.go delete mode 100644 std/sync/svs_pull_test.go diff --git a/std/sync/svs_encode_test.go b/std/sync/svs_encode_test.go deleted file mode 100644 index e3492c56..00000000 --- a/std/sync/svs_encode_test.go +++ /dev/null @@ -1,198 +0,0 @@ -package sync - -import ( - "fmt" - "testing" - "time" - - enc "github.com/named-data/ndnd/std/encoding" - spec_svs "github.com/named-data/ndnd/std/ndn/svs/v3" - "github.com/named-data/ndnd/std/types/optional" - tu "github.com/named-data/ndnd/std/utils/testutils" - "github.com/stretchr/testify/require" -) - -func testSvMapAliceBob() SvMap[uint64] { - m := NewSvMap[uint64](0) - m.Set(tu.NoErr(enc.NameFromStr("/ndn/alice")).TlvStr(), 100, 5) - m.Set(tu.NoErr(enc.NameFromStr("/ndn/bob")).TlvStr(), 150, 3) - return m -} - -func TestBuildInlineFullSvsData(t *testing.T) { - tu.SetT(t) - - m := testSvMapAliceBob() - sv := m.Encode(func(s uint64) uint64 { return s }) - data := buildInlineSvsData(m, spec_svs.VectorTypeFull, sv) - - require.Equal(t, ComputeMhash(m), data.MemberSetHash) - vt, ok := data.VectorType.Get() - require.True(t, ok) - require.Equal(t, spec_svs.VectorTypeFull, vt) - require.NotNil(t, data.StateVector) - require.Len(t, data.StateVector.Entries, 2) -} - -func TestExceedsSyncThreshold(t *testing.T) { - tu.SetT(t) - - require.False(t, exceedsSyncThreshold(0, 99999)) - require.False(t, exceedsSyncThreshold(1000, 500)) - require.True(t, exceedsSyncThreshold(1000, 1001)) -} - -func TestOnReceivePartialSkipsMissingNameOutdated(t *testing.T) { - tu.SetT(t) - - s := &SvSync{ - o: SvSyncOpts{ - OnUpdate: func(SvSyncUpdate) {}, - SuppressionPeriod: 200 * time.Millisecond, - PeriodicTimeout: 30 * time.Second, - }, - state: testSvMapAliceBob(), - mtime: make(map[string]time.Time), - ticker: time.NewTicker(30 * time.Second), - suppress: false, - } - - // PARTIAL with only bob; local knows alice — must not enter suppression. - bobOnly := NewSvMap[uint64](0) - bobOnly.Set(tu.NoErr(enc.NameFromStr("/ndn/bob")).TlvStr(), 150, 3) - partialSv := bobOnly.Encode(func(s uint64) uint64 { return s }) - - s.onReceiveStateVector(svSyncRecvSvArgs{ - sv: partialSv, - vectorType: optional.Some(spec_svs.VectorTypePartial), - mhash: ComputeMhash(bobOnly), - }) - - require.False(t, s.suppress) -} - -func TestOnReceiveFullTreatsMissingNameOutdated(t *testing.T) { - tu.SetT(t) - - s := &SvSync{ - o: SvSyncOpts{ - OnUpdate: func(SvSyncUpdate) {}, - SuppressionPeriod: 200 * time.Millisecond, - PeriodicTimeout: 30 * time.Second, - }, - state: testSvMapAliceBob(), - mtime: make(map[string]time.Time), - ticker: time.NewTicker(30 * time.Second), - suppress: false, - } - - bobOnly := NewSvMap[uint64](0) - bobOnly.Set(tu.NoErr(enc.NameFromStr("/ndn/bob")).TlvStr(), 150, 3) - fullSv := bobOnly.Encode(func(s uint64) uint64 { return s }) - - s.onReceiveStateVector(svSyncRecvSvArgs{ - sv: fullSv, - vectorType: optional.Some(spec_svs.VectorTypeFull), - mhash: ComputeMhash(bobOnly), - }) - - require.True(t, s.suppress) -} - -func TestEncodePartialSenderFirst(t *testing.T) { - tu.SetT(t) - - alice := tu.NoErr(enc.NameFromStr("/ndn/alice")) - bob := tu.NoErr(enc.NameFromStr("/ndn/bob")) - carol := tu.NoErr(enc.NameFromStr("/ndn/carol")) - - m := NewSvMap[uint64](0) - m.Set(alice.TlvStr(), 100, 5) - m.Set(bob.TlvStr(), 150, 3) - m.Set(carol.TlvStr(), 150, 7) - - // Threshold large enough for sender + one peer. - full := buildInlineSvsData(m, spec_svs.VectorTypeFull, m.Encode(func(s uint64) uint64 { return s })) - threshold := inlineSvsDataSize(full) - 1 - - partial := encodePartialStateVector(m, PartialEncodeOpts{ - Sender: carol, - Threshold: threshold, - Mtime: map[string]time.Time{ - alice.TlvStr(): time.Unix(10, 0), - bob.TlvStr(): time.Unix(20, 0), - }, - }) - - require.NotEmpty(t, partial.Entries) - require.Equal(t, carol, partial.Entries[0].Name) - if len(partial.Entries) > 2 { - require.Less(t, partial.Entries[1].Name.Compare(partial.Entries[2].Name), 0) - } -} - -func TestBuildSvsDataForSendPublicationPartial(t *testing.T) { - tu.SetT(t) - - alice := tu.NoErr(enc.NameFromStr("/ndn/alice")) - m := NewSvMap[uint64](0) - m.Set(alice.TlvStr(), 100, 5) - for i := range 20 { - name := tu.NoErr(enc.NameFromStr(fmt.Sprintf("/ndn/peer%d", i))) - m.Set(name.TlvStr(), 150, uint64(i+1)) - } - - full := buildInlineSvsData(m, spec_svs.VectorTypeFull, m.Encode(func(s uint64) uint64 { return s })) - threshold := inlineSvsDataSize(full) / 2 - - pub := buildSvsDataForSend(svsSendInput{ - State: m, Reason: syncSendPublication, Threshold: threshold, Sender: alice, - }) - vt, ok := pub.VectorType.Get() - require.True(t, ok) - require.Equal(t, spec_svs.VectorTypePartial, vt) - require.Less(t, len(pub.StateVector.Entries), len(full.StateVector.Entries)) - - periodic := buildSvsDataForSend(svsSendInput{ - State: m, Reason: syncSendPeriodic, Threshold: threshold, Sender: alice, - }) - vt, ok = periodic.VectorType.Get() - require.True(t, ok) - require.Equal(t, spec_svs.VectorTypeFull, vt) -} - -func TestOnReceivePartialMergesPresentEntriesOnly(t *testing.T) { - tu.SetT(t) - - var updates []SvSyncUpdate - s := &SvSync{ - o: SvSyncOpts{ - OnUpdate: func(u SvSyncUpdate) { updates = append(updates, u) }, - PeriodicTimeout: 30 * time.Second, - }, - state: NewSvMap[uint64](0), - mtime: make(map[string]time.Time), - ticker: time.NewTicker(30 * time.Second), - } - - alice := tu.NoErr(enc.NameFromStr("/ndn/alice")) - bob := tu.NoErr(enc.NameFromStr("/ndn/bob")) - s.state.Set(alice.TlvStr(), 100, 1) - s.state.Set(bob.TlvStr(), 150, 1) - - bobOnly := NewSvMap[uint64](0) - bobOnly.Set(bob.TlvStr(), 150, 4) - partialSv := bobOnly.Encode(func(s uint64) uint64 { return s }) - - s.onReceiveStateVector(svSyncRecvSvArgs{ - sv: partialSv, - vectorType: optional.Some(spec_svs.VectorTypePartial), - mhash: ComputeMhash(bobOnly), - }) - - require.Len(t, updates, 1) - require.Equal(t, bob, updates[0].Name) - require.EqualValues(t, 4, updates[0].High) - require.EqualValues(t, 1, s.state.Get(alice.TlvStr(), 100)) - require.EqualValues(t, 4, s.state.Get(bob.TlvStr(), 150)) -} diff --git a/std/sync/svs_mhash_test.go b/std/sync/svs_mhash_test.go deleted file mode 100644 index b37597e3..00000000 --- a/std/sync/svs_mhash_test.go +++ /dev/null @@ -1,118 +0,0 @@ -package sync_test - -import ( - "testing" - - enc "github.com/named-data/ndnd/std/encoding" - spec_svs "github.com/named-data/ndnd/std/ndn/svs/v3" - ndn_sync "github.com/named-data/ndnd/std/sync" - "github.com/named-data/ndnd/std/types/optional" - tu "github.com/named-data/ndnd/std/utils/testutils" - "github.com/stretchr/testify/require" -) - -func TestComputeMhashStable(t *testing.T) { - tu.SetT(t) - - m := ndn_sync.NewSvMap[uint64](0) - m.Set(tu.NoErr(enc.NameFromStr("/ndn/alice")).TlvStr(), 100, 1) - m.Set(tu.NoErr(enc.NameFromStr("/ndn/bob")).TlvStr(), 150, 3) - - h1 := ndn_sync.ComputeMhash(m) - h2 := ndn_sync.ComputeMhash(m) - require.Equal(t, h1, h2) - require.Len(t, h1, 32) -} - -func TestComputeMhashOrderIndependent(t *testing.T) { - tu.SetT(t) - - m1 := ndn_sync.NewSvMap[uint64](0) - m1.Set(tu.NoErr(enc.NameFromStr("/ndn/alice")).TlvStr(), 100, 1) - m1.Set(tu.NoErr(enc.NameFromStr("/ndn/bob")).TlvStr(), 150, 3) - - m2 := ndn_sync.NewSvMap[uint64](0) - m2.Set(tu.NoErr(enc.NameFromStr("/ndn/bob")).TlvStr(), 150, 3) - m2.Set(tu.NoErr(enc.NameFromStr("/ndn/alice")).TlvStr(), 100, 1) - - require.Equal(t, ndn_sync.ComputeMhash(m1), ndn_sync.ComputeMhash(m2)) -} - -func TestComputeMhashChangesOnMembership(t *testing.T) { - tu.SetT(t) - - m := ndn_sync.NewSvMap[uint64](0) - m.Set(tu.NoErr(enc.NameFromStr("/ndn/alice")).TlvStr(), 100, 1) - before := ndn_sync.ComputeMhash(m) - - m.Set(tu.NoErr(enc.NameFromStr("/ndn/bob")).TlvStr(), 150, 3) - after := ndn_sync.ComputeMhash(m) - require.NotEqual(t, before, after) -} - -func TestComputeMhashIgnoresSeqNo(t *testing.T) { - tu.SetT(t) - - m1 := ndn_sync.NewSvMap[uint64](0) - m1.Set(tu.NoErr(enc.NameFromStr("/ndn/alice")).TlvStr(), 100, 1) - - m2 := ndn_sync.NewSvMap[uint64](0) - m2.Set(tu.NoErr(enc.NameFromStr("/ndn/alice")).TlvStr(), 100, 99) - - require.Equal(t, ndn_sync.ComputeMhash(m1), ndn_sync.ComputeMhash(m2)) -} - -func TestSvsDataInlineTLV(t *testing.T) { - tu.SetT(t) - - m := ndn_sync.NewSvMap[uint64](0) - m.Set(tu.NoErr(enc.NameFromStr("/ndn/alice")).TlvStr(), 100, 1) - sv := m.Encode(func(s uint64) uint64 { return s }) - - original := &spec_svs.SvsData{ - MemberSetHash: ndn_sync.ComputeMhash(m), - VectorType: optional.Some(spec_svs.VectorTypeFull), - StateVector: sv, - } - wire := original.Encode().Join() - - parsed, err := spec_svs.ParseSvsData(enc.NewBufferView(wire), false) - require.NoError(t, err) - require.Equal(t, original.MemberSetHash, parsed.MemberSetHash) - require.Equal(t, original.VectorType, parsed.VectorType) - require.Equal(t, original.StateVector.Entries[0].Name.String(), parsed.StateVector.Entries[0].Name.String()) -} - -func TestSvsDataAnnounceTLV(t *testing.T) { - tu.SetT(t) - - ref := tu.NoErr(enc.NameFromStr("/ndn/svs/alice/100/32=sv/1")) - mhash := make([]byte, 32) - - original := &spec_svs.SvsData{ - MemberSetHash: mhash, - SvsDataRef: ref, - } - wire := original.Encode().Join() - - parsed, err := spec_svs.ParseSvsData(enc.NewBufferView(wire), false) - require.NoError(t, err) - require.Equal(t, mhash, parsed.MemberSetHash) - require.Equal(t, ref.String(), parsed.SvsDataRef.String()) - require.Nil(t, parsed.StateVector) - require.False(t, parsed.VectorType.IsSet()) -} - -func TestSvsDataLegacyParse(t *testing.T) { - tu.SetT(t) - - m := ndn_sync.NewSvMap[uint64](0) - m.Set(tu.NoErr(enc.NameFromStr("/ndn/alice")).TlvStr(), 100, 1) - legacy := &spec_svs.SvsData{StateVector: m.Encode(func(s uint64) uint64 { return s })} - wire := legacy.Encode().Join() - - parsed, err := spec_svs.ParseSvsData(enc.NewBufferView(wire), false) - require.NoError(t, err) - require.Nil(t, parsed.MemberSetHash) - require.NotNil(t, parsed.StateVector) -} diff --git a/std/sync/svs_pull_test.go b/std/sync/svs_pull_test.go deleted file mode 100644 index 56e9d4ed..00000000 --- a/std/sync/svs_pull_test.go +++ /dev/null @@ -1,157 +0,0 @@ -package sync - -import ( - "testing" - "time" - - enc "github.com/named-data/ndnd/std/encoding" - "github.com/named-data/ndnd/std/ndn" - spec "github.com/named-data/ndnd/std/ndn/spec_2022" - spec_svs "github.com/named-data/ndnd/std/ndn/svs/v3" - sig "github.com/named-data/ndnd/std/security/signer" - "github.com/named-data/ndnd/std/types/optional" - tu "github.com/named-data/ndnd/std/utils/testutils" - "github.com/stretchr/testify/require" -) - -func TestDeriveFullVectorPrefix(t *testing.T) { - tu.SetT(t) - - syncData := tu.NoErr(enc.NameFromStr("/ndn/svs/alice/1700000000/32=svs")) - prefix := deriveFullVectorPrefix(syncData) - require.Equal(t, "/ndn/svs/alice/1700000000/32=sv", prefix.String()) -} - -func TestPullRefFromSyncDataWire(t *testing.T) { - tu.SetT(t) - - syncDataName := tu.NoErr(enc.NameFromStr("/ndn/svs/alice/1700000000/32=svs")). - Append(enc.NewVersionComponent(12345)) - dataWire, err := spec.Spec{}.MakeData( - syncDataName, - &ndn.DataConfig{ContentType: optional.Some(ndn.ContentTypeBlob)}, - enc.Wire{enc.Buffer{0x01}}, - sig.NewSha256Signer(), - ) - require.NoError(t, err) - - ref := pullRefFromSyncDataWire(dataWire.Wire) - require.Equal(t, "/ndn/svs/alice/1700000000/32=sv", ref.String()) -} - -func TestBuildAnnounceSvsData(t *testing.T) { - tu.SetT(t) - - m := testSvMapAliceBob() - ref := tu.NoErr(enc.NameFromStr("/ndn/svs/alice/1700000000/32=sv/999")) - data := buildAnnounceSvsData(m, ref) - - require.Equal(t, ComputeMhash(m), data.MemberSetHash) - require.True(t, ref.Equal(data.SvsDataRef)) - require.Nil(t, data.StateVector) - require.False(t, data.VectorType.IsSet()) - - wire := data.Encode().Join() - parsed, err := spec_svs.ParseSvsData(enc.NewBufferView(wire), false) - require.NoError(t, err) - require.Equal(t, data.MemberSetHash, parsed.MemberSetHash) - require.True(t, ref.Equal(parsed.SvsDataRef)) - require.Nil(t, parsed.StateVector) -} - -func TestShouldUseAnnouncePull(t *testing.T) { - tu.SetT(t) - - m := testSvMapAliceBob() - fullSize := inlineFullSize(m) - - require.False(t, shouldUseAnnouncePull(syncSendPublication, fullSize-1, m)) - require.False(t, shouldUseAnnouncePull(syncSendPeriodic, 0, m)) - require.False(t, shouldUseAnnouncePull(syncSendPeriodic, fullSize+1, m)) - require.True(t, shouldUseAnnouncePull(syncSendPeriodic, fullSize-1, m)) - require.True(t, shouldUseAnnouncePull(syncSendOther, fullSize-1, m)) - require.False(t, shouldUseAnnouncePull(syncSendRecovery, 0, m)) - require.True(t, shouldUseAnnouncePull(syncSendRecovery, fullSize-1, m)) -} - -func TestIsTrustedSvsDataRef(t *testing.T) { - tu.SetT(t) - - trust := tu.NoErr(enc.NameFromStr("/ndn/svs/alice/1/32=sv")) - ref := tu.NoErr(enc.NameFromStr("/ndn/svs/alice/1/32=sv/999")) - bad := tu.NoErr(enc.NameFromStr("/ndn/evil/32=sv/1")) - - require.True(t, isTrustedSvsDataRef(ref, trust)) - require.True(t, isTrustedSvsDataRef(trust, trust)) - require.False(t, isTrustedSvsDataRef(bad, trust)) - require.False(t, isTrustedSvsDataRef(ref, nil)) -} - -func TestParseFullVectorContentRejectsBadMhash(t *testing.T) { - tu.SetT(t) - - m := testSvMapAliceBob() - inline := buildInlineFullFromState(m) - inline.MemberSetHash = []byte("not-a-valid-mhash-padding-000000") - wire := inline.Encode().Join() - - _, err := parseFullVectorContent(wire) - require.Error(t, err) -} - -func TestParseFullVectorContent(t *testing.T) { - tu.SetT(t) - - m := testSvMapAliceBob() - inline := buildInlineFullFromState(m) - wire := inline.Encode().Join() - - parsed, err := parseFullVectorContent(wire) - require.NoError(t, err) - require.Equal(t, inline.MemberSetHash, parsed.MemberSetHash) - require.Len(t, parsed.StateVector.Entries, 2) -} - -func TestOnPulledFullVectorMergesState(t *testing.T) { - tu.SetT(t) - - var updates []SvSyncUpdate - s := &SvSync{ - o: SvSyncOpts{ - OnUpdate: func(u SvSyncUpdate) { updates = append(updates, u) }, - PeriodicTimeout: 30 * time.Second, - }, - state: NewSvMap[uint64](0), - mtime: make(map[string]time.Time), - ticker: time.NewTicker(30 * time.Second), - recvSv: make(chan svSyncRecvSvArgs, 1), - } - - alice := tu.NoErr(enc.NameFromStr("/ndn/alice")) - s.state.Set(alice.TlvStr(), 100, 1) - - remote := testSvMapAliceBob() - content := buildInlineFullFromState(remote).Encode().Join() - - go func() { - s.onPulledFullVector(content) - }() - - s.onReceiveStateVector(<-s.recvSv) - - require.Len(t, updates, 2) - require.EqualValues(t, 3, s.state.Get(tu.NoErr(enc.NameFromStr("/ndn/bob")).TlvStr(), 150)) - require.EqualValues(t, 5, s.state.Get(alice.TlvStr(), 100)) -} - -func TestEncodeSyncDataAnnounceMode(t *testing.T) { - tu.SetT(t) - - m := testSvMapAliceBob() - require.True(t, shouldUseAnnouncePull(syncSendPeriodic, inlineFullSize(m)-1, m)) - - announce := buildAnnounceSvsData(m, tu.NoErr(enc.NameFromStr("/ndn/svs/alice/1/32=sv/2"))) - require.Nil(t, announce.StateVector) - vt, ok := announce.VectorType.Get() - require.False(t, ok || vt == spec_svs.VectorTypePartial) -} diff --git a/std/sync/svs_test.go b/std/sync/svs_test.go index a85de1c0..3b45a22a 100644 --- a/std/sync/svs_test.go +++ b/std/sync/svs_test.go @@ -1,16 +1,22 @@ package sync import ( + "fmt" "testing" "time" enc "github.com/named-data/ndnd/std/encoding" + "github.com/named-data/ndnd/std/ndn" + spec "github.com/named-data/ndnd/std/ndn/spec_2022" spec_svs "github.com/named-data/ndnd/std/ndn/svs/v3" + sig "github.com/named-data/ndnd/std/security/signer" "github.com/named-data/ndnd/std/types/optional" tu "github.com/named-data/ndnd/std/utils/testutils" "github.com/stretchr/testify/require" ) +// --- SvSync end-to-end (mesh) tests --- + func TestSvSyncSmallGroup(t *testing.T) { tu.SetT(t) @@ -206,3 +212,442 @@ func TestSvSyncMhashMismatchSenderAnnounce(t *testing.T) { mesh.waitUpdates("bob", 1, 2*time.Second) require.Greater(t, mesh.node("bob").memberCount(), 1) } + +// --- shared test helpers and encode tests --- + +func testSvMapAliceBob() SvMap[uint64] { + m := NewSvMap[uint64](0) + m.Set(tu.NoErr(enc.NameFromStr("/ndn/alice")).TlvStr(), 100, 5) + m.Set(tu.NoErr(enc.NameFromStr("/ndn/bob")).TlvStr(), 150, 3) + return m +} + +func TestBuildInlineFullSvsData(t *testing.T) { + tu.SetT(t) + + m := testSvMapAliceBob() + sv := m.Encode(func(s uint64) uint64 { return s }) + data := buildInlineSvsData(m, spec_svs.VectorTypeFull, sv) + + require.Equal(t, ComputeMhash(m), data.MemberSetHash) + vt, ok := data.VectorType.Get() + require.True(t, ok) + require.Equal(t, spec_svs.VectorTypeFull, vt) + require.NotNil(t, data.StateVector) + require.Len(t, data.StateVector.Entries, 2) +} + +func TestExceedsSyncThreshold(t *testing.T) { + tu.SetT(t) + + require.False(t, exceedsSyncThreshold(0, 99999)) + require.False(t, exceedsSyncThreshold(1000, 500)) + require.True(t, exceedsSyncThreshold(1000, 1001)) +} + +func TestOnReceivePartialSkipsMissingNameOutdated(t *testing.T) { + tu.SetT(t) + + s := &SvSync{ + o: SvSyncOpts{ + OnUpdate: func(SvSyncUpdate) {}, + SuppressionPeriod: 200 * time.Millisecond, + PeriodicTimeout: 30 * time.Second, + }, + state: testSvMapAliceBob(), + mtime: make(map[string]time.Time), + ticker: time.NewTicker(30 * time.Second), + suppress: false, + } + + // PARTIAL with only bob; local knows alice — must not enter suppression. + bobOnly := NewSvMap[uint64](0) + bobOnly.Set(tu.NoErr(enc.NameFromStr("/ndn/bob")).TlvStr(), 150, 3) + partialSv := bobOnly.Encode(func(s uint64) uint64 { return s }) + + s.onReceiveStateVector(svSyncRecvSvArgs{ + sv: partialSv, + vectorType: optional.Some(spec_svs.VectorTypePartial), + mhash: ComputeMhash(bobOnly), + }) + + require.False(t, s.suppress) +} + +func TestOnReceiveFullTreatsMissingNameOutdated(t *testing.T) { + tu.SetT(t) + + s := &SvSync{ + o: SvSyncOpts{ + OnUpdate: func(SvSyncUpdate) {}, + SuppressionPeriod: 200 * time.Millisecond, + PeriodicTimeout: 30 * time.Second, + }, + state: testSvMapAliceBob(), + mtime: make(map[string]time.Time), + ticker: time.NewTicker(30 * time.Second), + suppress: false, + } + + bobOnly := NewSvMap[uint64](0) + bobOnly.Set(tu.NoErr(enc.NameFromStr("/ndn/bob")).TlvStr(), 150, 3) + fullSv := bobOnly.Encode(func(s uint64) uint64 { return s }) + + s.onReceiveStateVector(svSyncRecvSvArgs{ + sv: fullSv, + vectorType: optional.Some(spec_svs.VectorTypeFull), + mhash: ComputeMhash(bobOnly), + }) + + require.True(t, s.suppress) +} + +func TestEncodePartialSenderFirst(t *testing.T) { + tu.SetT(t) + + alice := tu.NoErr(enc.NameFromStr("/ndn/alice")) + bob := tu.NoErr(enc.NameFromStr("/ndn/bob")) + carol := tu.NoErr(enc.NameFromStr("/ndn/carol")) + + m := NewSvMap[uint64](0) + m.Set(alice.TlvStr(), 100, 5) + m.Set(bob.TlvStr(), 150, 3) + m.Set(carol.TlvStr(), 150, 7) + + // Threshold large enough for sender + one peer. + full := buildInlineSvsData(m, spec_svs.VectorTypeFull, m.Encode(func(s uint64) uint64 { return s })) + threshold := inlineSvsDataSize(full) - 1 + + partial := encodePartialStateVector(m, PartialEncodeOpts{ + Sender: carol, + Threshold: threshold, + Mtime: map[string]time.Time{ + alice.TlvStr(): time.Unix(10, 0), + bob.TlvStr(): time.Unix(20, 0), + }, + }) + + require.NotEmpty(t, partial.Entries) + require.Equal(t, carol, partial.Entries[0].Name) + if len(partial.Entries) > 2 { + require.Less(t, partial.Entries[1].Name.Compare(partial.Entries[2].Name), 0) + } +} + +func TestBuildSvsDataForSendPublicationPartial(t *testing.T) { + tu.SetT(t) + + alice := tu.NoErr(enc.NameFromStr("/ndn/alice")) + m := NewSvMap[uint64](0) + m.Set(alice.TlvStr(), 100, 5) + for i := range 20 { + name := tu.NoErr(enc.NameFromStr(fmt.Sprintf("/ndn/peer%d", i))) + m.Set(name.TlvStr(), 150, uint64(i+1)) + } + + full := buildInlineSvsData(m, spec_svs.VectorTypeFull, m.Encode(func(s uint64) uint64 { return s })) + threshold := inlineSvsDataSize(full) / 2 + + pub := buildSvsDataForSend(svsSendInput{ + State: m, Reason: syncSendPublication, Threshold: threshold, Sender: alice, + }) + vt, ok := pub.VectorType.Get() + require.True(t, ok) + require.Equal(t, spec_svs.VectorTypePartial, vt) + require.Less(t, len(pub.StateVector.Entries), len(full.StateVector.Entries)) + + periodic := buildSvsDataForSend(svsSendInput{ + State: m, Reason: syncSendPeriodic, Threshold: threshold, Sender: alice, + }) + vt, ok = periodic.VectorType.Get() + require.True(t, ok) + require.Equal(t, spec_svs.VectorTypeFull, vt) +} + +func TestOnReceivePartialMergesPresentEntriesOnly(t *testing.T) { + tu.SetT(t) + + var updates []SvSyncUpdate + s := &SvSync{ + o: SvSyncOpts{ + OnUpdate: func(u SvSyncUpdate) { updates = append(updates, u) }, + PeriodicTimeout: 30 * time.Second, + }, + state: NewSvMap[uint64](0), + mtime: make(map[string]time.Time), + ticker: time.NewTicker(30 * time.Second), + } + + alice := tu.NoErr(enc.NameFromStr("/ndn/alice")) + bob := tu.NoErr(enc.NameFromStr("/ndn/bob")) + s.state.Set(alice.TlvStr(), 100, 1) + s.state.Set(bob.TlvStr(), 150, 1) + + bobOnly := NewSvMap[uint64](0) + bobOnly.Set(bob.TlvStr(), 150, 4) + partialSv := bobOnly.Encode(func(s uint64) uint64 { return s }) + + s.onReceiveStateVector(svSyncRecvSvArgs{ + sv: partialSv, + vectorType: optional.Some(spec_svs.VectorTypePartial), + mhash: ComputeMhash(bobOnly), + }) + + require.Len(t, updates, 1) + require.Equal(t, bob, updates[0].Name) + require.EqualValues(t, 4, updates[0].High) + require.EqualValues(t, 1, s.state.Get(alice.TlvStr(), 100)) + require.EqualValues(t, 4, s.state.Get(bob.TlvStr(), 150)) +} + +// --- mhash tests --- + +func TestComputeMhashStable(t *testing.T) { + tu.SetT(t) + + m := NewSvMap[uint64](0) + m.Set(tu.NoErr(enc.NameFromStr("/ndn/alice")).TlvStr(), 100, 1) + m.Set(tu.NoErr(enc.NameFromStr("/ndn/bob")).TlvStr(), 150, 3) + + h1 := ComputeMhash(m) + h2 := ComputeMhash(m) + require.Equal(t, h1, h2) + require.Len(t, h1, 32) +} + +func TestComputeMhashOrderIndependent(t *testing.T) { + tu.SetT(t) + + m1 := NewSvMap[uint64](0) + m1.Set(tu.NoErr(enc.NameFromStr("/ndn/alice")).TlvStr(), 100, 1) + m1.Set(tu.NoErr(enc.NameFromStr("/ndn/bob")).TlvStr(), 150, 3) + + m2 := NewSvMap[uint64](0) + m2.Set(tu.NoErr(enc.NameFromStr("/ndn/bob")).TlvStr(), 150, 3) + m2.Set(tu.NoErr(enc.NameFromStr("/ndn/alice")).TlvStr(), 100, 1) + + require.Equal(t, ComputeMhash(m1), ComputeMhash(m2)) +} + +func TestComputeMhashChangesOnMembership(t *testing.T) { + tu.SetT(t) + + m := NewSvMap[uint64](0) + m.Set(tu.NoErr(enc.NameFromStr("/ndn/alice")).TlvStr(), 100, 1) + before := ComputeMhash(m) + + m.Set(tu.NoErr(enc.NameFromStr("/ndn/bob")).TlvStr(), 150, 3) + after := ComputeMhash(m) + require.NotEqual(t, before, after) +} + +func TestComputeMhashIgnoresSeqNo(t *testing.T) { + tu.SetT(t) + + m1 := NewSvMap[uint64](0) + m1.Set(tu.NoErr(enc.NameFromStr("/ndn/alice")).TlvStr(), 100, 1) + + m2 := NewSvMap[uint64](0) + m2.Set(tu.NoErr(enc.NameFromStr("/ndn/alice")).TlvStr(), 100, 99) + + require.Equal(t, ComputeMhash(m1), ComputeMhash(m2)) +} + +func TestSvsDataInlineTLV(t *testing.T) { + tu.SetT(t) + + m := NewSvMap[uint64](0) + m.Set(tu.NoErr(enc.NameFromStr("/ndn/alice")).TlvStr(), 100, 1) + sv := m.Encode(func(s uint64) uint64 { return s }) + + original := &spec_svs.SvsData{ + MemberSetHash: ComputeMhash(m), + VectorType: optional.Some(spec_svs.VectorTypeFull), + StateVector: sv, + } + wire := original.Encode().Join() + + parsed, err := spec_svs.ParseSvsData(enc.NewBufferView(wire), false) + require.NoError(t, err) + require.Equal(t, original.MemberSetHash, parsed.MemberSetHash) + require.Equal(t, original.VectorType, parsed.VectorType) + require.Equal(t, original.StateVector.Entries[0].Name.String(), parsed.StateVector.Entries[0].Name.String()) +} + +func TestSvsDataAnnounceTLV(t *testing.T) { + tu.SetT(t) + + ref := tu.NoErr(enc.NameFromStr("/ndn/svs/alice/100/32=sv/1")) + mhash := make([]byte, 32) + + original := &spec_svs.SvsData{ + MemberSetHash: mhash, + SvsDataRef: ref, + } + wire := original.Encode().Join() + + parsed, err := spec_svs.ParseSvsData(enc.NewBufferView(wire), false) + require.NoError(t, err) + require.Equal(t, mhash, parsed.MemberSetHash) + require.Equal(t, ref.String(), parsed.SvsDataRef.String()) + require.Nil(t, parsed.StateVector) + require.False(t, parsed.VectorType.IsSet()) +} + +func TestSvsDataLegacyParse(t *testing.T) { + tu.SetT(t) + + m := NewSvMap[uint64](0) + m.Set(tu.NoErr(enc.NameFromStr("/ndn/alice")).TlvStr(), 100, 1) + legacy := &spec_svs.SvsData{StateVector: m.Encode(func(s uint64) uint64 { return s })} + wire := legacy.Encode().Join() + + parsed, err := spec_svs.ParseSvsData(enc.NewBufferView(wire), false) + require.NoError(t, err) + require.Nil(t, parsed.MemberSetHash) + require.NotNil(t, parsed.StateVector) +} + +// --- pull / recovery tests --- + +func TestDeriveFullVectorPrefix(t *testing.T) { + tu.SetT(t) + + syncData := tu.NoErr(enc.NameFromStr("/ndn/svs/alice/1700000000/32=svs")) + prefix := deriveFullVectorPrefix(syncData) + require.Equal(t, "/ndn/svs/alice/1700000000/32=sv", prefix.String()) +} + +func TestPullRefFromSyncDataWire(t *testing.T) { + tu.SetT(t) + + syncDataName := tu.NoErr(enc.NameFromStr("/ndn/svs/alice/1700000000/32=svs")). + Append(enc.NewVersionComponent(12345)) + dataWire, err := spec.Spec{}.MakeData( + syncDataName, + &ndn.DataConfig{ContentType: optional.Some(ndn.ContentTypeBlob)}, + enc.Wire{enc.Buffer{0x01}}, + sig.NewSha256Signer(), + ) + require.NoError(t, err) + + ref := pullRefFromSyncDataWire(dataWire.Wire) + require.Equal(t, "/ndn/svs/alice/1700000000/32=sv", ref.String()) +} + +func TestBuildAnnounceSvsData(t *testing.T) { + tu.SetT(t) + + m := testSvMapAliceBob() + ref := tu.NoErr(enc.NameFromStr("/ndn/svs/alice/1700000000/32=sv/999")) + data := buildAnnounceSvsData(m, ref) + + require.Equal(t, ComputeMhash(m), data.MemberSetHash) + require.True(t, ref.Equal(data.SvsDataRef)) + require.Nil(t, data.StateVector) + require.False(t, data.VectorType.IsSet()) + + wire := data.Encode().Join() + parsed, err := spec_svs.ParseSvsData(enc.NewBufferView(wire), false) + require.NoError(t, err) + require.Equal(t, data.MemberSetHash, parsed.MemberSetHash) + require.True(t, ref.Equal(parsed.SvsDataRef)) + require.Nil(t, parsed.StateVector) +} + +func TestShouldUseAnnouncePull(t *testing.T) { + tu.SetT(t) + + m := testSvMapAliceBob() + fullSize := inlineFullSize(m) + + require.False(t, shouldUseAnnouncePull(syncSendPublication, fullSize-1, m)) + require.False(t, shouldUseAnnouncePull(syncSendPeriodic, 0, m)) + require.False(t, shouldUseAnnouncePull(syncSendPeriodic, fullSize+1, m)) + require.True(t, shouldUseAnnouncePull(syncSendPeriodic, fullSize-1, m)) + require.True(t, shouldUseAnnouncePull(syncSendOther, fullSize-1, m)) + require.False(t, shouldUseAnnouncePull(syncSendRecovery, 0, m)) + require.True(t, shouldUseAnnouncePull(syncSendRecovery, fullSize-1, m)) +} + +func TestIsTrustedSvsDataRef(t *testing.T) { + tu.SetT(t) + + trust := tu.NoErr(enc.NameFromStr("/ndn/svs/alice/1/32=sv")) + ref := tu.NoErr(enc.NameFromStr("/ndn/svs/alice/1/32=sv/999")) + bad := tu.NoErr(enc.NameFromStr("/ndn/evil/32=sv/1")) + + require.True(t, isTrustedSvsDataRef(ref, trust)) + require.True(t, isTrustedSvsDataRef(trust, trust)) + require.False(t, isTrustedSvsDataRef(bad, trust)) + require.False(t, isTrustedSvsDataRef(ref, nil)) +} + +func TestParseFullVectorContentRejectsBadMhash(t *testing.T) { + tu.SetT(t) + + m := testSvMapAliceBob() + inline := buildInlineFullFromState(m) + inline.MemberSetHash = []byte("not-a-valid-mhash-padding-000000") + wire := inline.Encode().Join() + + _, err := parseFullVectorContent(wire) + require.Error(t, err) +} + +func TestParseFullVectorContent(t *testing.T) { + tu.SetT(t) + + m := testSvMapAliceBob() + inline := buildInlineFullFromState(m) + wire := inline.Encode().Join() + + parsed, err := parseFullVectorContent(wire) + require.NoError(t, err) + require.Equal(t, inline.MemberSetHash, parsed.MemberSetHash) + require.Len(t, parsed.StateVector.Entries, 2) +} + +func TestOnPulledFullVectorMergesState(t *testing.T) { + tu.SetT(t) + + var updates []SvSyncUpdate + s := &SvSync{ + o: SvSyncOpts{ + OnUpdate: func(u SvSyncUpdate) { updates = append(updates, u) }, + PeriodicTimeout: 30 * time.Second, + }, + state: NewSvMap[uint64](0), + mtime: make(map[string]time.Time), + ticker: time.NewTicker(30 * time.Second), + recvSv: make(chan svSyncRecvSvArgs, 1), + } + + alice := tu.NoErr(enc.NameFromStr("/ndn/alice")) + s.state.Set(alice.TlvStr(), 100, 1) + + remote := testSvMapAliceBob() + content := buildInlineFullFromState(remote).Encode().Join() + + go func() { + s.onPulledFullVector(content) + }() + + s.onReceiveStateVector(<-s.recvSv) + + require.Len(t, updates, 2) + require.EqualValues(t, 3, s.state.Get(tu.NoErr(enc.NameFromStr("/ndn/bob")).TlvStr(), 150)) + require.EqualValues(t, 5, s.state.Get(alice.TlvStr(), 100)) +} + +func TestEncodeSyncDataAnnounceMode(t *testing.T) { + tu.SetT(t) + + m := testSvMapAliceBob() + require.True(t, shouldUseAnnouncePull(syncSendPeriodic, inlineFullSize(m)-1, m)) + + announce := buildAnnounceSvsData(m, tu.NoErr(enc.NameFromStr("/ndn/svs/alice/1/32=sv/2"))) + require.Nil(t, announce.StateVector) + vt, ok := announce.VectorType.Get() + require.False(t, ok || vt == spec_svs.VectorTypePartial) +} From 220ecd30e616e2b2862670d87c4b2605016ad5b8 Mon Sep 17 00:00:00 2001 From: Taranum Wasu Date: Thu, 9 Jul 2026 13:01:27 +0530 Subject: [PATCH 14/24] test: remove in-process mesh tests for svs The mesh-based end-to-end tests in svs_mesh_test.go are redundant with the GitHub e2e tests (sprint topology) and add an in-process hub plus test-only face type to the package. Remove them and the helper functions in svs_test.go that depended on the mesh. --- std/sync/svs_mesh_test.go | 261 -------------------------------------- std/sync/svs_test.go | 198 ----------------------------- 2 files changed, 459 deletions(-) delete mode 100644 std/sync/svs_mesh_test.go diff --git a/std/sync/svs_mesh_test.go b/std/sync/svs_mesh_test.go deleted file mode 100644 index 8ae68df6..00000000 --- a/std/sync/svs_mesh_test.go +++ /dev/null @@ -1,261 +0,0 @@ -package sync - -import ( - "fmt" - "sync" - "sync/atomic" - "testing" - "time" - - enc "github.com/named-data/ndnd/std/encoding" - basic_engine "github.com/named-data/ndnd/std/engine/basic" - "github.com/named-data/ndnd/std/ndn" - spec_svs "github.com/named-data/ndnd/std/ndn/svs/v3" - "github.com/named-data/ndnd/std/object" - "github.com/named-data/ndnd/std/object/storage" - tu "github.com/named-data/ndnd/std/utils/testutils" - "github.com/stretchr/testify/require" -) - -// In-process multi-node SVS mesh for integration tests (svs_test.go). -// -// Each node runs a real ndnd engine + SvSync instance. A test-only multicast hub -// delivers packets between nodes so Sync Interests propagate without NFD/mininet. -// This exercises small-group sync, PARTIAL publication, and announce+pull recovery. - -const testMeshBoot = uint64(1_700_000_000) - -type meshMulticastHub struct { - mu sync.Mutex - faces []*meshMulticastFace -} - -type meshMulticastFace struct { - hub *meshMulticastHub - id int - running atomic.Bool - local bool - onPkt func(frame []byte) - onError func(err error) - onUp sync.Map - onDown sync.Map - onUpID int - onDownID int -} - -func newMeshMulticastHub() *meshMulticastHub { - return &meshMulticastHub{} -} - -func (h *meshMulticastHub) newFace() *meshMulticastFace { - h.mu.Lock() - defer h.mu.Unlock() - f := &meshMulticastFace{hub: h, id: len(h.faces), local: true} - h.faces = append(h.faces, f) - return f -} - -func (f *meshMulticastFace) String() string { - return fmt.Sprintf("mesh-multicast-face-%d", f.id) -} - -func (f *meshMulticastFace) IsRunning() bool { return f.running.Load() } -func (f *meshMulticastFace) IsLocal() bool { return f.local } -func (f *meshMulticastFace) OnPacket(fn func([]byte)) { f.onPkt = fn } -func (f *meshMulticastFace) OnError(fn func(error)) { f.onError = fn } - -func (f *meshMulticastFace) OnUp(fn func()) (cancel func()) { - id := f.onUpID - f.onUpID++ - f.onUp.Store(id, fn) - return func() { f.onUp.Delete(id) } -} - -func (f *meshMulticastFace) OnDown(fn func()) (cancel func()) { - id := f.onDownID - f.onDownID++ - f.onDown.Store(id, fn) - return func() { f.onDown.Delete(id) } -} - -func (f *meshMulticastFace) Open() error { - if f.onError == nil || f.onPkt == nil { - return fmt.Errorf("face callbacks are not set") - } - if f.running.Load() { - return fmt.Errorf("face is already running") - } - f.running.Store(true) - f.onUp.Range(func(_, cb any) bool { - cb.(func())() - return true - }) - return nil -} - -func (f *meshMulticastFace) Close() error { - if !f.running.Swap(false) { - return fmt.Errorf("face is not running") - } - return nil -} - -func (f *meshMulticastFace) Send(pkt enc.Wire) error { - if !f.running.Load() { - return fmt.Errorf("face is not running") - } - f.hub.broadcast(f.id, pkt.Join()) - return nil -} - -func (h *meshMulticastHub) broadcast(from int, frame []byte) { - h.mu.Lock() - peers := append([]*meshMulticastFace(nil), h.faces...) - h.mu.Unlock() - - for _, peer := range peers { - if peer.id == from || !peer.running.Load() || peer.onPkt == nil { - continue - } - frameCopy := make([]byte, len(frame)) - copy(frameCopy, frame) - peer.onPkt(frameCopy) - } -} - -type testMeshNode struct { - id string - producer enc.Name - svs *SvSync - client ndn.Client - engine *basic_engine.Engine - updates []SvSyncUpdate - updatesMu sync.Mutex -} - -type testMesh struct { - t *testing.T - hub *meshMulticastHub - timer *basic_engine.DummyTimer - group enc.Name - nodes map[string]*testMeshNode -} - -func newTestMesh(t *testing.T, ids []string, configure func(id string, opts *SvSyncOpts)) *testMesh { - t.Helper() - tu.SetT(t) - - mesh := &testMesh{ - t: t, - hub: newMeshMulticastHub(), - timer: basic_engine.NewDummyTimer(), - group: tu.NoErr(enc.NameFromStr("/test/svs-mesh/32=svs")), - nodes: make(map[string]*testMeshNode, len(ids)), - } - - for _, id := range ids { - producer := mesh.group.Append(enc.NewGenericComponent(id)) - syncDataName := producer. - Append(enc.NewTimestampComponent(testMeshBoot)). - Append(enc.NewKeywordComponent("svs")) - - f := mesh.hub.newFace() - engine := basic_engine.NewEngine(f, mesh.timer) - store := storage.NewMemoryStore() - client := object.NewClient(engine, store, nil) - - opts := SvSyncOpts{ - Client: client, - GroupPrefix: mesh.group, - SyncDataName: syncDataName, - BootTime: testMeshBoot, - PeriodicTimeout: 30 * time.Second, - SuppressionPeriod: 50 * time.Millisecond, - } - if configure != nil { - configure(id, &opts) - } - - node := &testMeshNode{ - id: id, - producer: producer, - client: client, - engine: engine, - } - opts.OnUpdate = func(u SvSyncUpdate) { - node.updatesMu.Lock() - node.updates = append(node.updates, u) - node.updatesMu.Unlock() - } - - node.svs = NewSvSync(opts) - require.NoError(t, engine.Start()) - require.NoError(t, client.Start()) - require.NoError(t, node.svs.Start()) - - mesh.nodes[id] = node - } - - mesh.wait(80 * time.Millisecond) - return mesh -} - -func (m *testMesh) node(id string) *testMeshNode { - return m.nodes[id] -} - -func (m *testMesh) wait(d time.Duration) { - time.Sleep(d) -} - -func (m *testMesh) waitUpdates(id string, n int, timeout time.Duration) { - deadline := time.Now().Add(timeout) - for time.Now().Before(deadline) { - m.node(id).updatesMu.Lock() - got := len(m.node(id).updates) - m.node(id).updatesMu.Unlock() - if got >= n { - return - } - time.Sleep(10 * time.Millisecond) - } - m.t.Fatalf("node %s: want %d updates, got %d", id, n, len(m.node(id).updates)) -} - -func (m *testMesh) stop() { - for _, node := range m.nodes { - _ = node.svs.Stop() - _ = node.client.Stop() - _ = node.engine.Stop() - } -} - -func (n *testMeshNode) publish() uint64 { - return n.svs.IncrSeqNo(n.producer) -} - -func (n *testMeshNode) memberCount() int { - return len(n.svs.GetNames()) -} - -func seedLargeLocalState(s *SvSync, self enc.Name, peers int) { - for i := range peers { - name := tu.NoErr(enc.NameFromStr(fmt.Sprintf("/test/svs-mesh/peer%d", i))) - s.state.Set(name.TlvStr(), testMeshBoot, uint64(i+1)) - } - s.state.Set(self.TlvStr(), testMeshBoot, 1) -} - -func selfOnlyVector(name enc.Name, boot uint64) *spec_svs.StateVector { - return &spec_svs.StateVector{ - Entries: []*spec_svs.StateVectorEntry{ - { - Name: name, - SeqNoEntries: []*spec_svs.SeqNoEntry{{ - BootstrapTime: boot, - SeqNo: 0, - }}, - }, - }, - } -} diff --git a/std/sync/svs_test.go b/std/sync/svs_test.go index 3b45a22a..d62ed687 100644 --- a/std/sync/svs_test.go +++ b/std/sync/svs_test.go @@ -15,204 +15,6 @@ import ( "github.com/stretchr/testify/require" ) -// --- SvSync end-to-end (mesh) tests --- - -func TestSvSyncSmallGroup(t *testing.T) { - tu.SetT(t) - - mesh := newTestMesh(t, []string{"alice", "bob", "carol"}, nil) - defer mesh.stop() - - seq := mesh.node("alice").publish() - mesh.wait(150 * time.Millisecond) - - mesh.waitUpdates("bob", 1, time.Second) - mesh.waitUpdates("carol", 1, time.Second) - - require.EqualValues(t, seq, mesh.node("bob").svs.GetSeqNo(mesh.node("alice").producer)) - require.EqualValues(t, seq, mesh.node("carol").svs.GetSeqNo(mesh.node("alice").producer)) - - // Default threshold 0: legacy wire (StateVector only, no mhash). - alice := mesh.node("alice") - alice.svs.mutex.Lock() - legacy := buildLegacySvsData(alice.svs.state) - alice.svs.mutex.Unlock() - require.Empty(t, legacy.MemberSetHash) - require.False(t, legacy.VectorType.IsSet()) -} - -func TestSvSyncPartialPublication(t *testing.T) { - tu.SetT(t) - - threshold := 400 - mesh := newTestMesh(t, []string{"alice", "bob"}, func(id string, opts *SvSyncOpts) { - opts.SyncVectorThreshold = threshold - }) - defer mesh.stop() - - seedLargeLocalState(mesh.node("alice").svs, mesh.node("alice").producer, 18) - mesh.node("alice").publish() - mesh.wait(200 * time.Millisecond) - - mesh.waitUpdates("bob", 1, time.Second) - require.EqualValues(t, 2, mesh.node("bob").svs.GetSeqNo(mesh.node("alice").producer)) - - // PARTIAL must not copy the full membership in one shot. - require.Less(t, mesh.node("bob").memberCount(), mesh.node("alice").memberCount()) -} - -func TestSvSyncMhashPull(t *testing.T) { - tu.SetT(t) - - threshold := 400 - mesh := newTestMesh(t, []string{"alice", "bob"}, func(id string, opts *SvSyncOpts) { - opts.SyncVectorThreshold = threshold - }) - defer mesh.stop() - - alice := mesh.node("alice").svs - seedLargeLocalState(alice, mesh.node("alice").producer, 12) - alice.state.Set(mesh.node("alice").producer.TlvStr(), testMeshBoot, 1) - - // Alice publishes full vector and sends announce-only periodic sync. - go alice.sendSyncInterest(syncSendPeriodic) - mesh.wait(300 * time.Millisecond) - - mesh.waitUpdates("bob", 1, 2*time.Second) - require.Greater(t, mesh.node("bob").memberCount(), 1) -} - -func TestSvSyncPeriodicAnnounce(t *testing.T) { - tu.SetT(t) - - threshold := 400 - mesh := newTestMesh(t, []string{"alice", "bob"}, func(id string, opts *SvSyncOpts) { - opts.SyncVectorThreshold = threshold - }) - defer mesh.stop() - - seedLargeLocalState(mesh.node("alice").svs, mesh.node("alice").producer, 12) - mesh.node("alice").svs.state.Set(mesh.node("alice").producer.TlvStr(), testMeshBoot, 2) - - go mesh.node("alice").svs.sendSyncInterest(syncSendPeriodic) - mesh.wait(300 * time.Millisecond) - - mesh.waitUpdates("bob", 1, 2*time.Second) - - // Pulled FULL state should include a seeded peer. - peer0 := tu.NoErr(enc.NameFromStr("/test/svs-mesh/peer0")) - require.Greater(t, mesh.node("bob").svs.GetSeqNo(peer0), uint64(0)) -} - -func TestSvSyncJoin(t *testing.T) { - tu.SetT(t) - - mesh := newTestMesh(t, []string{"alice", "carol"}, nil) - defer mesh.stop() - - // Alice knows about herself and has seq 3. - alice := mesh.node("alice").svs - alice.state.Set(mesh.node("alice").producer.TlvStr(), testMeshBoot, 3) - - // Carol announces join with self-only [carol:0] (simulated multicast to alice). - carolName := mesh.node("carol").producer - carolSv := selfOnlyVector(carolName, testMeshBoot) - carolState := NewSvMap[uint64](0) - carolState.Set(carolName.TlvStr(), testMeshBoot, 0) - mesh.node("alice").svs.onReceiveStateVector(svSyncRecvSvArgs{ - sv: carolSv, - vectorType: optional.Some(spec_svs.VectorTypeFull), - mhash: ComputeMhash(carolState), - }) - - // Alice responds with up-to-date FULL state. - go mesh.node("alice").svs.sendSyncInterest(syncSendPeriodic) - mesh.wait(200 * time.Millisecond) - - mesh.waitUpdates("carol", 1, time.Second) - require.EqualValues(t, 3, mesh.node("carol").svs.GetSeqNo(mesh.node("alice").producer)) -} - -func TestSvSyncJoinLargeGroup(t *testing.T) { - tu.SetT(t) - - threshold := 400 - mesh := newTestMesh(t, []string{"alice", "carol"}, func(id string, opts *SvSyncOpts) { - opts.SyncVectorThreshold = threshold - }) - defer mesh.stop() - - seedLargeLocalState(mesh.node("alice").svs, mesh.node("alice").producer, 12) - mesh.node("alice").svs.state.Set(mesh.node("alice").producer.TlvStr(), testMeshBoot, 3) - - carolName := mesh.node("carol").producer - carolSv := selfOnlyVector(carolName, testMeshBoot) - carolState := NewSvMap[uint64](0) - carolState.Set(carolName.TlvStr(), testMeshBoot, 0) - mesh.node("alice").svs.onReceiveStateVector(svSyncRecvSvArgs{ - sv: carolSv, - vectorType: optional.Some(spec_svs.VectorTypeFull), - mhash: ComputeMhash(carolState), - }) - - go mesh.node("alice").svs.sendSyncInterest(syncSendRecovery) - mesh.wait(400 * time.Millisecond) - - mesh.waitUpdates("carol", 1, 2*time.Second) - require.Greater(t, mesh.node("carol").memberCount(), 1) - require.EqualValues(t, 3, mesh.node("carol").svs.GetSeqNo(mesh.node("alice").producer)) -} - -func TestHandleMhashMismatchNoBlindPull(t *testing.T) { - tu.SetT(t) - - mesh := newTestMesh(t, []string{"alice", "bob"}, nil) - defer mesh.stop() - - alice := mesh.node("alice").svs - bobOnly := NewSvMap[uint64](0) - bobOnly.Set(mesh.node("bob").producer.TlvStr(), testMeshBoot, 1) - partialSv := bobOnly.Encode(func(s uint64) uint64 { return s }) - - // Bob's inline FULL with mismatched mhash; alice is not a strict superset. - alice.onReceiveStateVector(svSyncRecvSvArgs{ - sv: partialSv, - vectorType: optional.Some(spec_svs.VectorTypeFull), - mhash: ComputeMhash(bobOnly), - }) - - // Should not hang or panic from blind pull to unpublished 32=sv prefix. - mesh.wait(100 * time.Millisecond) -} - -func TestSvSyncMhashMismatchSenderAnnounce(t *testing.T) { - tu.SetT(t) - - threshold := 400 - mesh := newTestMesh(t, []string{"alice", "bob"}, func(id string, opts *SvSyncOpts) { - opts.SyncVectorThreshold = threshold - }) - defer mesh.stop() - - seedLargeLocalState(mesh.node("alice").svs, mesh.node("alice").producer, 12) - alice := mesh.node("alice").svs - - bobOnly := NewSvMap[uint64](0) - bobOnly.Set(mesh.node("bob").producer.TlvStr(), testMeshBoot, 1) - partialSv := bobOnly.Encode(func(s uint64) uint64 { return s }) - - // Bob sends PARTIAL; Alice has strict superset membership and announces recovery. - alice.onReceiveStateVector(svSyncRecvSvArgs{ - sv: partialSv, - vectorType: optional.Some(spec_svs.VectorTypePartial), - mhash: ComputeMhash(bobOnly), - }) - - mesh.wait(400 * time.Millisecond) - mesh.waitUpdates("bob", 1, 2*time.Second) - require.Greater(t, mesh.node("bob").memberCount(), 1) -} - // --- shared test helpers and encode tests --- func testSvMapAliceBob() SvMap[uint64] { From 98e290bdfe255d0be4c9cdcab647b2d91f7eb9ab Mon Sep 17 00:00:00 2001 From: Taranum Wasu Date: Thu, 9 Jul 2026 13:03:45 +0530 Subject: [PATCH 15/24] sync: inline tiny encode helpers Fold buildLegacySvsData, buildInlineSvsData, inlineSvsDataSize, exceedsSyncThreshold, buildInlineFullFromState and inlineFullSize into the call sites that actually use them. The inline FULL/PARTIAL SvsData structs are now visible at the build sites, matching the rest of the encode path. --- std/sync/svs_encode.go | 68 ++++++++++++++++-------------------------- std/sync/svs_pull.go | 15 ++++++++-- std/sync/svs_test.go | 63 ++++++++++++++++++++++++++------------ 3 files changed, 82 insertions(+), 64 deletions(-) diff --git a/std/sync/svs_encode.go b/std/sync/svs_encode.go index c3a1f343..f9fe541b 100644 --- a/std/sync/svs_encode.go +++ b/std/sync/svs_encode.go @@ -30,43 +30,6 @@ type PartialEncodeOpts struct { Mtime map[string]time.Time } -// buildLegacySvsData is the pre-revision SVS v3 wire format used when threshold is 0. -func buildLegacySvsData(state SvMap[uint64]) *spec_svs.SvsData { - sv := state.Encode(func(seq uint64) uint64 { return seq }) - return &spec_svs.SvsData{StateVector: sv} -} - -// buildInlineSvsData constructs inline SvsData (mhash + VectorType + StateVector). -func buildInlineSvsData(state SvMap[uint64], vectorType uint64, sv *spec_svs.StateVector) *spec_svs.SvsData { - return &spec_svs.SvsData{ - MemberSetHash: ComputeMhash(state), - VectorType: optional.Some(vectorType), - StateVector: sv, - } -} - -// inlineSvsDataSize returns the encoded byte length of SvsData content. -func inlineSvsDataSize(data *spec_svs.SvsData) int { - return len(data.Encode().Join()) -} - -// exceedsSyncThreshold reports whether size is over the configured inline budget. -// threshold 0 disables the large-group size limit (legacy mode). -func exceedsSyncThreshold(threshold, size int) bool { - return threshold > 0 && size > threshold -} - -// buildInlineFullFromState builds inline FULL SvsData for the complete local state. -func buildInlineFullFromState(state SvMap[uint64]) *spec_svs.SvsData { - sv := state.Encode(func(seq uint64) uint64 { return seq }) - return buildInlineSvsData(state, spec_svs.VectorTypeFull, sv) -} - -// inlineFullSize returns encoded inline FULL SvsData size for threshold checks. -func inlineFullSize(state SvMap[uint64]) int { - return inlineSvsDataSize(buildInlineFullFromState(state)) -} - // svsSendInput carries everything needed to build inline Sync Data for send. type svsSendInput struct { State SvMap[uint64] @@ -79,15 +42,20 @@ type svsSendInput struct { } // buildSvsDataForSend picks inline FULL or PARTIAL SvsData for an outgoing Sync message. +// If Threshold <= 0, the legacy StateVector-only wire is used (no mhash). func buildSvsDataForSend(in svsSendInput) *spec_svs.SvsData { if in.Threshold <= 0 { - return buildLegacySvsData(in.State) + return &spec_svs.SvsData{StateVector: in.State.Encode(func(seq uint64) uint64 { return seq })} } fullSv := in.State.Encode(func(seq uint64) uint64 { return seq }) - fullData := buildInlineSvsData(in.State, spec_svs.VectorTypeFull, fullSv) + fullData := &spec_svs.SvsData{ + MemberSetHash: ComputeMhash(in.State), + VectorType: optional.Some(spec_svs.VectorTypeFull), + StateVector: fullSv, + } - if in.Reason != syncSendPublication || !exceedsSyncThreshold(in.Threshold, inlineSvsDataSize(fullData)) { + if in.Reason != syncSendPublication || len(fullData.Encode().Join()) <= in.Threshold { return fullData } @@ -98,7 +66,11 @@ func buildSvsDataForSend(in svsSendInput) *spec_svs.SvsData { Propagation: in.Propagation, Mtime: in.Mtime, }) - return buildInlineSvsData(in.State, spec_svs.VectorTypePartial, partialSv) + return &spec_svs.SvsData{ + MemberSetHash: ComputeMhash(in.State), + VectorType: optional.Some(spec_svs.VectorTypePartial), + StateVector: partialSv, + } } // encodePartialStateVector builds a PARTIAL StateVector for new publication. @@ -118,7 +90,12 @@ func encodePartialStateVector(state SvMap[uint64], opts PartialEncodeOpts) *spec // Sender-only baseline must always fit when possible. baseline := &spec_svs.StateVector{Entries: []*spec_svs.StateVectorEntry{senderEntry}} - if inlineSvsDataSize(buildInlineSvsData(state, spec_svs.VectorTypePartial, baseline)) > opts.Threshold { + baselineData := &spec_svs.SvsData{ + MemberSetHash: ComputeMhash(state), + VectorType: optional.Some(spec_svs.VectorTypePartial), + StateVector: baseline, + } + if len(baselineData.Encode().Join()) > opts.Threshold { return baseline } @@ -139,7 +116,12 @@ func encodePartialStateVector(state SvMap[uint64], opts PartialEncodeOpts) *spec trial := append(slices.Clone(entries), entry) sortPartialTail(trial) trialSv := &spec_svs.StateVector{Entries: trial} - if exceedsSyncThreshold(opts.Threshold, inlineSvsDataSize(buildInlineSvsData(state, spec_svs.VectorTypePartial, trialSv))) { + trialData := &spec_svs.SvsData{ + MemberSetHash: ComputeMhash(state), + VectorType: optional.Some(spec_svs.VectorTypePartial), + StateVector: trialSv, + } + if len(trialData.Encode().Join()) > opts.Threshold { break } diff --git a/std/sync/svs_pull.go b/std/sync/svs_pull.go index 77f9b163..5b95d3b1 100644 --- a/std/sync/svs_pull.go +++ b/std/sync/svs_pull.go @@ -63,7 +63,13 @@ func shouldUseAnnouncePull(reason syncSendReason, threshold int, state SvMap[uin if reason == syncSendPublication { return false } - return exceedsSyncThreshold(threshold, inlineFullSize(state)) + sv := state.Encode(func(seq uint64) uint64 { return seq }) + full := &spec_svs.SvsData{ + MemberSetHash: ComputeMhash(state), + VectorType: optional.Some(spec_svs.VectorTypeFull), + StateVector: sv, + } + return len(full.Encode().Join()) > threshold } // publishFullVectorData produces retrievable inline FULL SvsData at .../32=sv/. @@ -71,7 +77,12 @@ func (s *SvSync) publishFullVectorData(state SvMap[uint64]) (enc.Name, error) { if len(s.fullVectorPrefix) == 0 { return nil, fmt.Errorf("full vector prefix unset") } - content := buildInlineFullFromState(state).Encode() + sv := state.Encode(func(seq uint64) uint64 { return seq }) + content := (&spec_svs.SvsData{ + MemberSetHash: ComputeMhash(state), + VectorType: optional.Some(spec_svs.VectorTypeFull), + StateVector: sv, + }).Encode() name := s.fullVectorPrefix.WithVersion(enc.VersionUnixMicro) return s.o.Client.Produce(ndn.ProduceArgs{ Name: name, diff --git a/std/sync/svs_test.go b/std/sync/svs_test.go index d62ed687..1f38c90b 100644 --- a/std/sync/svs_test.go +++ b/std/sync/svs_test.go @@ -29,7 +29,11 @@ func TestBuildInlineFullSvsData(t *testing.T) { m := testSvMapAliceBob() sv := m.Encode(func(s uint64) uint64 { return s }) - data := buildInlineSvsData(m, spec_svs.VectorTypeFull, sv) + data := &spec_svs.SvsData{ + MemberSetHash: ComputeMhash(m), + VectorType: optional.Some(spec_svs.VectorTypeFull), + StateVector: sv, + } require.Equal(t, ComputeMhash(m), data.MemberSetHash) vt, ok := data.VectorType.Get() @@ -39,14 +43,6 @@ func TestBuildInlineFullSvsData(t *testing.T) { require.Len(t, data.StateVector.Entries, 2) } -func TestExceedsSyncThreshold(t *testing.T) { - tu.SetT(t) - - require.False(t, exceedsSyncThreshold(0, 99999)) - require.False(t, exceedsSyncThreshold(1000, 500)) - require.True(t, exceedsSyncThreshold(1000, 1001)) -} - func TestOnReceivePartialSkipsMissingNameOutdated(t *testing.T) { tu.SetT(t) @@ -117,8 +113,12 @@ func TestEncodePartialSenderFirst(t *testing.T) { m.Set(carol.TlvStr(), 150, 7) // Threshold large enough for sender + one peer. - full := buildInlineSvsData(m, spec_svs.VectorTypeFull, m.Encode(func(s uint64) uint64 { return s })) - threshold := inlineSvsDataSize(full) - 1 + full := &spec_svs.SvsData{ + MemberSetHash: ComputeMhash(m), + VectorType: optional.Some(spec_svs.VectorTypeFull), + StateVector: m.Encode(func(s uint64) uint64 { return s }), + } + threshold := len(full.Encode().Join()) - 1 partial := encodePartialStateVector(m, PartialEncodeOpts{ Sender: carol, @@ -147,8 +147,12 @@ func TestBuildSvsDataForSendPublicationPartial(t *testing.T) { m.Set(name.TlvStr(), 150, uint64(i+1)) } - full := buildInlineSvsData(m, spec_svs.VectorTypeFull, m.Encode(func(s uint64) uint64 { return s })) - threshold := inlineSvsDataSize(full) / 2 + full := &spec_svs.SvsData{ + MemberSetHash: ComputeMhash(m), + VectorType: optional.Some(spec_svs.VectorTypeFull), + StateVector: m.Encode(func(s uint64) uint64 { return s }), + } + threshold := len(full.Encode().Join()) / 2 pub := buildSvsDataForSend(svsSendInput{ State: m, Reason: syncSendPublication, Threshold: threshold, Sender: alice, @@ -361,7 +365,12 @@ func TestShouldUseAnnouncePull(t *testing.T) { tu.SetT(t) m := testSvMapAliceBob() - fullSize := inlineFullSize(m) + sv := m.Encode(func(s uint64) uint64 { return s }) + fullSize := len((&spec_svs.SvsData{ + MemberSetHash: ComputeMhash(m), + VectorType: optional.Some(spec_svs.VectorTypeFull), + StateVector: sv, + }).Encode().Join()) require.False(t, shouldUseAnnouncePull(syncSendPublication, fullSize-1, m)) require.False(t, shouldUseAnnouncePull(syncSendPeriodic, 0, m)) @@ -389,8 +398,11 @@ func TestParseFullVectorContentRejectsBadMhash(t *testing.T) { tu.SetT(t) m := testSvMapAliceBob() - inline := buildInlineFullFromState(m) - inline.MemberSetHash = []byte("not-a-valid-mhash-padding-000000") + inline := &spec_svs.SvsData{ + MemberSetHash: []byte("not-a-valid-mhash-padding-000000"), + VectorType: optional.Some(spec_svs.VectorTypeFull), + StateVector: m.Encode(func(s uint64) uint64 { return s }), + } wire := inline.Encode().Join() _, err := parseFullVectorContent(wire) @@ -401,7 +413,11 @@ func TestParseFullVectorContent(t *testing.T) { tu.SetT(t) m := testSvMapAliceBob() - inline := buildInlineFullFromState(m) + inline := &spec_svs.SvsData{ + MemberSetHash: ComputeMhash(m), + VectorType: optional.Some(spec_svs.VectorTypeFull), + StateVector: m.Encode(func(s uint64) uint64 { return s }), + } wire := inline.Encode().Join() parsed, err := parseFullVectorContent(wire) @@ -429,7 +445,11 @@ func TestOnPulledFullVectorMergesState(t *testing.T) { s.state.Set(alice.TlvStr(), 100, 1) remote := testSvMapAliceBob() - content := buildInlineFullFromState(remote).Encode().Join() + content := (&spec_svs.SvsData{ + MemberSetHash: ComputeMhash(remote), + VectorType: optional.Some(spec_svs.VectorTypeFull), + StateVector: remote.Encode(func(s uint64) uint64 { return s }), + }).Encode().Join() go func() { s.onPulledFullVector(content) @@ -446,7 +466,12 @@ func TestEncodeSyncDataAnnounceMode(t *testing.T) { tu.SetT(t) m := testSvMapAliceBob() - require.True(t, shouldUseAnnouncePull(syncSendPeriodic, inlineFullSize(m)-1, m)) + fullSize := len((&spec_svs.SvsData{ + MemberSetHash: ComputeMhash(m), + VectorType: optional.Some(spec_svs.VectorTypeFull), + StateVector: m.Encode(func(s uint64) uint64 { return s }), + }).Encode().Join()) + require.True(t, shouldUseAnnouncePull(syncSendPeriodic, fullSize-1, m)) announce := buildAnnounceSvsData(m, tu.NoErr(enc.NameFromStr("/ndn/svs/alice/1/32=sv/2"))) require.Nil(t, announce.StateVector) From 25e3d9122cbf8fdf26ac918dfdca206c406674e0 Mon Sep 17 00:00:00 2001 From: Taranum Wasu Date: Thu, 9 Jul 2026 13:04:10 +0530 Subject: [PATCH 16/24] sync: make FullVectorPrefix configurable on SvSyncOpts Add SvSyncOpts.FullVectorPrefix next to SyncDataName. When set, it is used directly as the publish/serve prefix for retrievable FULL StateVector Data used by announce+pull recovery. When unset, the previous default (SyncDataName with trailing "32=svs" -> "32=sv") is preserved. --- std/sync/svs.go | 9 ++++++++- std/sync/svs_pull.go | 9 +++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/std/sync/svs.go b/std/sync/svs.go index 3dc7caf5..724d48bb 100644 --- a/std/sync/svs.go +++ b/std/sync/svs.go @@ -61,6 +61,13 @@ type SvSyncOpts struct { // If not provided, the GroupPrefix will be used instead. SyncDataName enc.Name + // FullVectorPrefix is the publish/serve prefix for retrievable FULL + // StateVector Data used by SvsDataRef announce+pull recovery. The + // version component is appended when producing the Data. + // If not provided, it defaults to SyncDataName with the trailing + // "32=svs" component (if present) replaced by "32=sv". + FullVectorPrefix enc.Name + // Initial state vector from persistence InitialState *spec_svs.StateVector // Boot time from persistence @@ -155,7 +162,7 @@ func NewSvSync(opts SvSyncOpts) *SvSync { recvSv: make(chan svSyncRecvSvArgs, 128), - fullVectorPrefix: deriveFullVectorPrefix(opts.SyncDataName), + fullVectorPrefix: resolveFullVectorPrefix(opts.FullVectorPrefix, opts.SyncDataName), faceCancel: func() {}, } diff --git a/std/sync/svs_pull.go b/std/sync/svs_pull.go index 5b95d3b1..6036a530 100644 --- a/std/sync/svs_pull.go +++ b/std/sync/svs_pull.go @@ -29,6 +29,15 @@ func deriveFullVectorPrefix(syncDataName enc.Name) enc.Name { return base.Append(enc.NewKeywordComponent(fullVectorKeyword)) } +// resolveFullVectorPrefix returns the explicit FullVectorPrefix if set, +// otherwise derives it from SyncDataName. +func resolveFullVectorPrefix(explicit, syncDataName enc.Name) enc.Name { + if len(explicit) > 0 { + return explicit.Clone() + } + return deriveFullVectorPrefix(syncDataName) +} + func pullRefFromSyncDataWire(dataWire enc.Wire) enc.Name { data, _, err := spec.Spec{}.ReadData(enc.NewWireView(dataWire)) if err != nil { From e7e87bf165640bf2f1c52a06cd6988488d21dccc Mon Sep 17 00:00:00 2001 From: Taranum Wasu Date: Thu, 9 Jul 2026 13:05:59 +0530 Subject: [PATCH 17/24] sync: encode membership tuple as standard TLV Introduce spec_svs.MembershipTuple (Tuple-T 0xcc with Name and BootstrapTime) in the v3 TLV definitions and use the standard TLV codec in ComputeMhash instead of inlining a bespoke 0xD4 TLV byte. --- std/ndn/svs/v3/definitions.go | 8 ++ std/ndn/svs/v3/zz_generated.go | 166 +++++++++++++++++++++++++++++++++ std/sync/svs_mhash.go | 39 +++----- 3 files changed, 188 insertions(+), 25 deletions(-) diff --git a/std/ndn/svs/v3/definitions.go b/std/ndn/svs/v3/definitions.go index a3fcf415..4412d154 100644 --- a/std/ndn/svs/v3/definitions.go +++ b/std/ndn/svs/v3/definitions.go @@ -42,6 +42,14 @@ type SeqNoEntry struct { SeqNo uint64 `tlv:"0xd6"` } +// MembershipTuple is one (Name, BootstrapTime) pair used to compute MemberSetHash. +type MembershipTuple struct { + //+field:name + Name enc.Name `tlv:"0x07"` + //+field:natural + BootstrapTime uint64 `tlv:"0xd4"` +} + // +tlv-model:nocopy type PassiveState struct { //+field:sequence:[]byte:binary:[]byte diff --git a/std/ndn/svs/v3/zz_generated.go b/std/ndn/svs/v3/zz_generated.go index 138dbed3..6d4590e7 100644 --- a/std/ndn/svs/v3/zz_generated.go +++ b/std/ndn/svs/v3/zz_generated.go @@ -837,6 +837,172 @@ func ParseSeqNoEntry(reader enc.WireView, ignoreCritical bool) (*SeqNoEntry, err return context.Parse(reader, ignoreCritical) } +type MembershipTupleEncoder struct { + Length uint + + Name_length uint +} + +type MembershipTupleParsingContext struct { +} + +func (encoder *MembershipTupleEncoder) Init(value *MembershipTuple) { + if value.Name != nil { + encoder.Name_length = 0 + for _, c := range value.Name { + encoder.Name_length += uint(c.EncodingLength()) + } + } + + l := uint(0) + if value.Name != nil { + l += 1 + l += uint(enc.TLNum(encoder.Name_length).EncodingLength()) + l += encoder.Name_length + } + l += 1 + l += uint(1 + enc.Nat(value.BootstrapTime).EncodingLength()) + encoder.Length = l + +} + +func (context *MembershipTupleParsingContext) Init() { + +} + +func (encoder *MembershipTupleEncoder) EncodeInto(value *MembershipTuple, buf []byte) { + + pos := uint(0) + + if value.Name != nil { + buf[pos] = byte(7) + pos += 1 + pos += uint(enc.TLNum(encoder.Name_length).EncodeInto(buf[pos:])) + for _, c := range value.Name { + pos += uint(c.EncodeInto(buf[pos:])) + } + } + buf[pos] = byte(212) + pos += 1 + + buf[pos] = byte(enc.Nat(value.BootstrapTime).EncodeInto(buf[pos+1:])) + pos += uint(1 + buf[pos]) +} + +func (encoder *MembershipTupleEncoder) Encode(value *MembershipTuple) enc.Wire { + + wire := make(enc.Wire, 1) + wire[0] = make([]byte, encoder.Length) + buf := wire[0] + encoder.EncodeInto(value, buf) + + return wire +} + +func (context *MembershipTupleParsingContext) Parse(reader enc.WireView, ignoreCritical bool) (*MembershipTuple, error) { + + var handled_Name bool = false + var handled_BootstrapTime bool = false + + progress := -1 + _ = progress + + value := &MembershipTuple{} + var err error + var startPos int + for { + startPos = reader.Pos() + if startPos >= reader.Length() { + break + } + typ := enc.TLNum(0) + l := enc.TLNum(0) + typ, err = reader.ReadTLNum() + if err != nil { + return nil, enc.ErrFailToParse{TypeNum: 0, Err: err} + } + l, err = reader.ReadTLNum() + if err != nil { + return nil, enc.ErrFailToParse{TypeNum: 0, Err: err} + } + + err = nil + if handled := false; true { + switch typ { + case 7: + if true { + handled = true + handled_Name = true + delegate := reader.Delegate(int(l)) + value.Name, err = delegate.ReadName() + } + case 212: + if true { + handled = true + handled_BootstrapTime = true + value.BootstrapTime = uint64(0) + { + for i := 0; i < int(l); i++ { + x := byte(0) + x, err = reader.ReadByte() + if err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + break + } + value.BootstrapTime = uint64(value.BootstrapTime<<8) | uint64(x) + } + } + } + default: + if !ignoreCritical && ((typ <= 31) || ((typ & 1) == 1)) { + return nil, enc.ErrUnrecognizedField{TypeNum: typ} + } + handled = true + err = reader.Skip(int(l)) + } + if err == nil && !handled { + } + if err != nil { + return nil, enc.ErrFailToParse{TypeNum: typ, Err: err} + } + } + } + + startPos = reader.Pos() + err = nil + + if !handled_Name && err == nil { + value.Name = nil + } + if !handled_BootstrapTime && err == nil { + err = enc.ErrSkipRequired{Name: "BootstrapTime", TypeNum: 212} + } + + if err != nil { + return nil, err + } + + return value, nil +} + +func (value *MembershipTuple) Encode() enc.Wire { + encoder := MembershipTupleEncoder{} + encoder.Init(value) + return encoder.Encode(value) +} + +func (value *MembershipTuple) Bytes() []byte { + return value.Encode().Join() +} + +func ParseMembershipTuple(reader enc.WireView, ignoreCritical bool) (*MembershipTuple, error) { + context := MembershipTupleParsingContext{} + context.Init() + return context.Parse(reader, ignoreCritical) +} + type PassiveStateEncoder struct { Length uint diff --git a/std/sync/svs_mhash.go b/std/sync/svs_mhash.go index 510c16cc..53c07fbd 100644 --- a/std/sync/svs_mhash.go +++ b/std/sync/svs_mhash.go @@ -4,51 +4,40 @@ import ( "crypto/sha256" "slices" - enc "github.com/named-data/ndnd/std/encoding" + spec_svs "github.com/named-data/ndnd/std/ndn/svs/v3" ) -// membershipTuple is one (Name, BootstrapTime) pair in the sync group. -type membershipTuple struct { - name enc.Name - boot uint64 -} - // ComputeMhash returns the membership hash over all (Name, BootstrapTime) pairs in state. +// Each tuple is encoded as a TLV structure (Tuple-T 0xcc with Name and BootstrapTime +// children) using the ndnd standard TLV codec. func ComputeMhash(state SvMap[uint64]) []byte { - members := make([]membershipTuple, 0) + tuples := make([]*spec_svs.MembershipTuple, 0) for name, vals := range state.Iter() { for _, val := range vals { - members = append(members, membershipTuple{name: name, boot: val.Boot}) + tuples = append(tuples, &spec_svs.MembershipTuple{ + Name: name, + BootstrapTime: val.Boot, + }) } } - slices.SortFunc(members, func(a, b membershipTuple) int { - if c := a.name.Compare(b.name); c != 0 { + slices.SortFunc(tuples, func(a, b *spec_svs.MembershipTuple) int { + if c := a.Name.Compare(b.Name); c != 0 { return c } - if a.boot < b.boot { + if a.BootstrapTime < b.BootstrapTime { return -1 } - if a.boot > b.boot { + if a.BootstrapTime > b.BootstrapTime { return 1 } return 0 }) h := sha256.New() - for _, m := range members { - h.Write(m.name.Bytes()) - h.Write(bootstrapTimeTLV(m.boot)) + for _, t := range tuples { + h.Write(t.Encode().Join()) } return h.Sum(nil) } -// bootstrapTimeTLV encodes BOOTSTRAP-TIME-TYPE (0xD4) matching SVS v3 SeqNoEntry layout. -func bootstrapTimeTLV(boot uint64) []byte { - valLen := enc.Nat(boot).EncodingLength() - buf := make([]byte, 2+valLen) - buf[0] = 0xD4 - buf[1] = byte(valLen) - enc.Nat(boot).EncodeInto(buf[2:]) - return buf -} From 9762c7386cdb6a9a056e9fbd19c48d8e6a9c0a73 Mon Sep 17 00:00:00 2001 From: Taranum Wasu Date: Thu, 9 Jul 2026 13:08:17 +0530 Subject: [PATCH 18/24] sync: drop SyncVectorThreshold legacy compatibility MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All Sync messages now carry mhash and VectorType. The legacy StateVector-only wire (and the threshold-0 paths that guarded it) is removed from encode, shouldUseAnnouncePull and handleMhashMismatch. SyncVectorThreshold defaults to 1200 bytes when 0 is supplied; the gating is preserved for switching between inline FULL / PARTIAL / announce+pull in large groups. The spec's §4.3 and §8 are updated to document the no-backward-compat posture. --- docs/svs-v3-revision.md | 11 ++++++----- std/sync/svs.go | 9 +++++++-- std/sync/svs_encode.go | 9 --------- std/sync/svs_pull.go | 12 ++---------- std/sync/svs_test.go | 2 -- 5 files changed, 15 insertions(+), 28 deletions(-) diff --git a/docs/svs-v3-revision.md b/docs/svs-v3-revision.md index f32d480f..e276c9bb 100644 --- a/docs/svs-v3-revision.md +++ b/docs/svs-v3-revision.md @@ -225,9 +225,10 @@ Stop adding entries when estimated inline `SvsData` size approaches `SyncVectorT ### 4.3 `SyncVectorThreshold` -- Configurable implementation parameter (application packet size budget). -- Default value is implementation-defined. -- When `encoded_size(FULL) ≤ SyncVectorThreshold`, nodes use **inline FULL** only (existing SVS v3 behavior). +- Configurable implementation parameter (application packet size budget) in bytes. +- When `encoded_size(FULL) ≤ SyncVectorThreshold`, nodes use **inline FULL** (with `mhash` and `VectorType=FULL`). +- When `encoded_size(FULL) > SyncVectorThreshold`, nodes switch to PARTIAL (publication) or announce+pull (periodic sync and recovery). +- This revision does **not** retain the legacy StateVector-only wire. All Sync messages carry `mhash` and a `VectorType`. --- @@ -368,9 +369,9 @@ Group exceeds `SyncVectorThreshold`. Producer `P` publishes: --- -## 8. Open Items +## 8. Interoperability -1. **Mixed-version interoperability** — how revised nodes coexist with plain SVS v3 peers in the same sync group, if at all. +This revision defines a single wire profile; it does not interoperate with plain SVS v3 peers in the same sync group. Deployments must upgrade all nodes to a revision-conformant implementation at the same time. --- diff --git a/std/sync/svs.go b/std/sync/svs.go index 724d48bb..c2f50064 100644 --- a/std/sync/svs.go +++ b/std/sync/svs.go @@ -84,8 +84,10 @@ type SvSyncOpts struct { // IgnoreValidity ignores validity period in the validation chain IgnoreValidity optional.Optional[bool] - // SyncVectorThreshold is the max inline SvsData size (bytes). - // 0 = legacy mode (StateVector-only wire, no mhash or announce+pull recovery). + // SyncVectorThreshold is the max inline SvsData size (bytes) above + // which the sender switches to PARTIAL (on publication) or + // announce+pull (on periodic sync and recovery). When 0, the default + // (1200 bytes) is used. SyncVectorThreshold int } @@ -141,6 +143,9 @@ func NewSvSync(opts SvSyncOpts) *SvSync { if len(opts.SyncDataName) == 0 { opts.SyncDataName = opts.GroupPrefix } + if opts.SyncVectorThreshold <= 0 { + opts.SyncVectorThreshold = 1200 + } return &SvSync{ o: opts, diff --git a/std/sync/svs_encode.go b/std/sync/svs_encode.go index f9fe541b..c4e97f12 100644 --- a/std/sync/svs_encode.go +++ b/std/sync/svs_encode.go @@ -42,12 +42,7 @@ type svsSendInput struct { } // buildSvsDataForSend picks inline FULL or PARTIAL SvsData for an outgoing Sync message. -// If Threshold <= 0, the legacy StateVector-only wire is used (no mhash). func buildSvsDataForSend(in svsSendInput) *spec_svs.SvsData { - if in.Threshold <= 0 { - return &spec_svs.SvsData{StateVector: in.State.Encode(func(seq uint64) uint64 { return seq })} - } - fullSv := in.State.Encode(func(seq uint64) uint64 { return seq }) fullData := &spec_svs.SvsData{ MemberSetHash: ComputeMhash(in.State), @@ -84,10 +79,6 @@ func encodePartialStateVector(state SvMap[uint64], opts PartialEncodeOpts) *spec senderEntry = &spec_svs.StateVectorEntry{Name: opts.Sender} } - if opts.Threshold <= 0 { - return &spec_svs.StateVector{Entries: []*spec_svs.StateVectorEntry{senderEntry}} - } - // Sender-only baseline must always fit when possible. baseline := &spec_svs.StateVector{Entries: []*spec_svs.StateVectorEntry{senderEntry}} baselineData := &spec_svs.SvsData{ diff --git a/std/sync/svs_pull.go b/std/sync/svs_pull.go index 6036a530..382ebc25 100644 --- a/std/sync/svs_pull.go +++ b/std/sync/svs_pull.go @@ -60,12 +60,9 @@ func buildAnnounceSvsData(state SvMap[uint64], ref enc.Name) *spec_svs.SvsData { } } -// shouldUseAnnouncePull reports whether the sender should publish at 32=sv and announce-only Sync Data. -// Large-group announce+pull is disabled when threshold <= 0 (legacy mode). +// shouldUseAnnouncePull reports whether the sender should publish at 32=sv and announce-only Sync Data +// when the local FULL encoding exceeds the configured threshold. func shouldUseAnnouncePull(reason syncSendReason, threshold int, state SvMap[uint64]) bool { - if threshold <= 0 { - return false - } if reason == syncSendRecovery { return true } @@ -186,11 +183,6 @@ func (s *SvSync) sendRecoveryAnnounce() { // handleMhashMismatch schedules announce or pull recovery on membership mismatch. func (s *SvSync) handleMhashMismatch(args svSyncRecvSvArgs, recvSv SvMap[uint64]) { - // [Spec] Legacy mode (threshold 0): inline FULL is merged above; large-group recovery is off. - if s.o.SyncVectorThreshold <= 0 { - return - } - localMhash := ComputeMhash(s.state) if bytes.Equal(localMhash, args.mhash) { return diff --git a/std/sync/svs_test.go b/std/sync/svs_test.go index 1f38c90b..bff48f2b 100644 --- a/std/sync/svs_test.go +++ b/std/sync/svs_test.go @@ -373,11 +373,9 @@ func TestShouldUseAnnouncePull(t *testing.T) { }).Encode().Join()) require.False(t, shouldUseAnnouncePull(syncSendPublication, fullSize-1, m)) - require.False(t, shouldUseAnnouncePull(syncSendPeriodic, 0, m)) require.False(t, shouldUseAnnouncePull(syncSendPeriodic, fullSize+1, m)) require.True(t, shouldUseAnnouncePull(syncSendPeriodic, fullSize-1, m)) require.True(t, shouldUseAnnouncePull(syncSendOther, fullSize-1, m)) - require.False(t, shouldUseAnnouncePull(syncSendRecovery, 0, m)) require.True(t, shouldUseAnnouncePull(syncSendRecovery, fullSize-1, m)) } From f0724bfb08ac0766e2f9b170bbb8df5f72cfb1e6 Mon Sep 17 00:00:00 2001 From: Taranum Wasu Date: Thu, 9 Jul 2026 13:14:27 +0530 Subject: [PATCH 19/24] fix lint: run gofmt and goimports on touched files The previous batch of commits touched svs_test.go and svs_mhash.go but left them non-canonical for gofmt (trailing blank line in svs_mhash.go) and goimports (third-party testify imports not separated from local named-data/ndnd imports in svs_test.go). Apply both. --- std/sync/svs_mhash.go | 1 - std/sync/svs_test.go | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/std/sync/svs_mhash.go b/std/sync/svs_mhash.go index 53c07fbd..31435d45 100644 --- a/std/sync/svs_mhash.go +++ b/std/sync/svs_mhash.go @@ -40,4 +40,3 @@ func ComputeMhash(state SvMap[uint64]) []byte { } return h.Sum(nil) } - diff --git a/std/sync/svs_test.go b/std/sync/svs_test.go index bff48f2b..6bb959d1 100644 --- a/std/sync/svs_test.go +++ b/std/sync/svs_test.go @@ -5,6 +5,8 @@ import ( "testing" "time" + "github.com/stretchr/testify/require" + enc "github.com/named-data/ndnd/std/encoding" "github.com/named-data/ndnd/std/ndn" spec "github.com/named-data/ndnd/std/ndn/spec_2022" @@ -12,7 +14,6 @@ import ( sig "github.com/named-data/ndnd/std/security/signer" "github.com/named-data/ndnd/std/types/optional" tu "github.com/named-data/ndnd/std/utils/testutils" - "github.com/stretchr/testify/require" ) // --- shared test helpers and encode tests --- From fee8d9f112e0504d346b739e7b65955b6eeab9dd Mon Sep 17 00:00:00 2001 From: Taranum Wasu Date: Thu, 9 Jul 2026 13:08:17 +0530 Subject: [PATCH 20/24] docs: confirm SVS v3 revision drops legacy wire regardless of SyncVectorThreshold MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per Tianyuan's clarification on the review, this revision always emits mhash + VectorType on Sync Data and never falls back to the legacy StateVector-only SvsData. SyncVectorThreshold == 0 just uses the default 1200-byte inline budget. Update §4.3 / §8 of the spec and the doc comment on SvSyncOpts.SyncVectorThreshold to state this explicitly. Co-authored-by: Cursor --- docs/svs-v3-revision.md | 2 +- std/sync/svs.go | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/svs-v3-revision.md b/docs/svs-v3-revision.md index e276c9bb..f316c872 100644 --- a/docs/svs-v3-revision.md +++ b/docs/svs-v3-revision.md @@ -371,7 +371,7 @@ Group exceeds `SyncVectorThreshold`. Producer `P` publishes: ## 8. Interoperability -This revision defines a single wire profile; it does not interoperate with plain SVS v3 peers in the same sync group. Deployments must upgrade all nodes to a revision-conformant implementation at the same time. +This revision defines a single wire profile; it does not interoperate with plain SVS v3 peers in the same sync group. Deployments must upgrade all nodes to a revision-conformant implementation at the same time. Concretely, every Sync Data carries `mhash` and a `VectorType`, and the implementation does not emit a legacy `StateVector`-only `SvsData` under any `SyncVectorThreshold` value (a `Threshold` of 0 falls back to the default 1200-byte budget). --- diff --git a/std/sync/svs.go b/std/sync/svs.go index c2f50064..edb69ef1 100644 --- a/std/sync/svs.go +++ b/std/sync/svs.go @@ -87,7 +87,9 @@ type SvSyncOpts struct { // SyncVectorThreshold is the max inline SvsData size (bytes) above // which the sender switches to PARTIAL (on publication) or // announce+pull (on periodic sync and recovery). When 0, the default - // (1200 bytes) is used. + // (1200 bytes) is used. This revision always emits `mhash` + + // `VectorType` on the wire; there is no legacy `StateVector`-only + // mode regardless of this value. SyncVectorThreshold int } From fe435b25d0f65592a8458918d5d0b61db7e565a3 Mon Sep 17 00:00:00 2001 From: Taranum Wasu Date: Mon, 13 Jul 2026 15:10:53 +0530 Subject: [PATCH 21/24] sync: bypass metadata fetch when pulling full state vector pullFullVector previously failed because Client.ConsumeExt expects RDR metadata (/name/32=metadata) that publishFullVectorData never produces. Pass NoMetadata: true so the consumer fetches segments directly by name. --- docs/svs-v3-revision.md | 11 +-- std/ndn/svs/v3/definitions.go | 8 -- std/ndn/svs/v3/zz_generated.go | 166 --------------------------------- std/sync/svs_encode.go | 9 ++ std/sync/svs_mhash.go | 40 +++++--- std/sync/svs_pull.go | 13 ++- std/sync/svs_test.go | 5 +- 7 files changed, 54 insertions(+), 198 deletions(-) diff --git a/docs/svs-v3-revision.md b/docs/svs-v3-revision.md index f316c872..f32d480f 100644 --- a/docs/svs-v3-revision.md +++ b/docs/svs-v3-revision.md @@ -225,10 +225,9 @@ Stop adding entries when estimated inline `SvsData` size approaches `SyncVectorT ### 4.3 `SyncVectorThreshold` -- Configurable implementation parameter (application packet size budget) in bytes. -- When `encoded_size(FULL) ≤ SyncVectorThreshold`, nodes use **inline FULL** (with `mhash` and `VectorType=FULL`). -- When `encoded_size(FULL) > SyncVectorThreshold`, nodes switch to PARTIAL (publication) or announce+pull (periodic sync and recovery). -- This revision does **not** retain the legacy StateVector-only wire. All Sync messages carry `mhash` and a `VectorType`. +- Configurable implementation parameter (application packet size budget). +- Default value is implementation-defined. +- When `encoded_size(FULL) ≤ SyncVectorThreshold`, nodes use **inline FULL** only (existing SVS v3 behavior). --- @@ -369,9 +368,9 @@ Group exceeds `SyncVectorThreshold`. Producer `P` publishes: --- -## 8. Interoperability +## 8. Open Items -This revision defines a single wire profile; it does not interoperate with plain SVS v3 peers in the same sync group. Deployments must upgrade all nodes to a revision-conformant implementation at the same time. Concretely, every Sync Data carries `mhash` and a `VectorType`, and the implementation does not emit a legacy `StateVector`-only `SvsData` under any `SyncVectorThreshold` value (a `Threshold` of 0 falls back to the default 1200-byte budget). +1. **Mixed-version interoperability** — how revised nodes coexist with plain SVS v3 peers in the same sync group, if at all. --- diff --git a/std/ndn/svs/v3/definitions.go b/std/ndn/svs/v3/definitions.go index 4412d154..a3fcf415 100644 --- a/std/ndn/svs/v3/definitions.go +++ b/std/ndn/svs/v3/definitions.go @@ -42,14 +42,6 @@ type SeqNoEntry struct { SeqNo uint64 `tlv:"0xd6"` } -// MembershipTuple is one (Name, BootstrapTime) pair used to compute MemberSetHash. -type MembershipTuple struct { - //+field:name - Name enc.Name `tlv:"0x07"` - //+field:natural - BootstrapTime uint64 `tlv:"0xd4"` -} - // +tlv-model:nocopy type PassiveState struct { //+field:sequence:[]byte:binary:[]byte diff --git a/std/ndn/svs/v3/zz_generated.go b/std/ndn/svs/v3/zz_generated.go index 6d4590e7..138dbed3 100644 --- a/std/ndn/svs/v3/zz_generated.go +++ b/std/ndn/svs/v3/zz_generated.go @@ -837,172 +837,6 @@ func ParseSeqNoEntry(reader enc.WireView, ignoreCritical bool) (*SeqNoEntry, err return context.Parse(reader, ignoreCritical) } -type MembershipTupleEncoder struct { - Length uint - - Name_length uint -} - -type MembershipTupleParsingContext struct { -} - -func (encoder *MembershipTupleEncoder) Init(value *MembershipTuple) { - if value.Name != nil { - encoder.Name_length = 0 - for _, c := range value.Name { - encoder.Name_length += uint(c.EncodingLength()) - } - } - - l := uint(0) - if value.Name != nil { - l += 1 - l += uint(enc.TLNum(encoder.Name_length).EncodingLength()) - l += encoder.Name_length - } - l += 1 - l += uint(1 + enc.Nat(value.BootstrapTime).EncodingLength()) - encoder.Length = l - -} - -func (context *MembershipTupleParsingContext) Init() { - -} - -func (encoder *MembershipTupleEncoder) EncodeInto(value *MembershipTuple, buf []byte) { - - pos := uint(0) - - if value.Name != nil { - buf[pos] = byte(7) - pos += 1 - pos += uint(enc.TLNum(encoder.Name_length).EncodeInto(buf[pos:])) - for _, c := range value.Name { - pos += uint(c.EncodeInto(buf[pos:])) - } - } - buf[pos] = byte(212) - pos += 1 - - buf[pos] = byte(enc.Nat(value.BootstrapTime).EncodeInto(buf[pos+1:])) - pos += uint(1 + buf[pos]) -} - -func (encoder *MembershipTupleEncoder) Encode(value *MembershipTuple) enc.Wire { - - wire := make(enc.Wire, 1) - wire[0] = make([]byte, encoder.Length) - buf := wire[0] - encoder.EncodeInto(value, buf) - - return wire -} - -func (context *MembershipTupleParsingContext) Parse(reader enc.WireView, ignoreCritical bool) (*MembershipTuple, error) { - - var handled_Name bool = false - var handled_BootstrapTime bool = false - - progress := -1 - _ = progress - - value := &MembershipTuple{} - var err error - var startPos int - for { - startPos = reader.Pos() - if startPos >= reader.Length() { - break - } - typ := enc.TLNum(0) - l := enc.TLNum(0) - typ, err = reader.ReadTLNum() - if err != nil { - return nil, enc.ErrFailToParse{TypeNum: 0, Err: err} - } - l, err = reader.ReadTLNum() - if err != nil { - return nil, enc.ErrFailToParse{TypeNum: 0, Err: err} - } - - err = nil - if handled := false; true { - switch typ { - case 7: - if true { - handled = true - handled_Name = true - delegate := reader.Delegate(int(l)) - value.Name, err = delegate.ReadName() - } - case 212: - if true { - handled = true - handled_BootstrapTime = true - value.BootstrapTime = uint64(0) - { - for i := 0; i < int(l); i++ { - x := byte(0) - x, err = reader.ReadByte() - if err != nil { - if err == io.EOF { - err = io.ErrUnexpectedEOF - } - break - } - value.BootstrapTime = uint64(value.BootstrapTime<<8) | uint64(x) - } - } - } - default: - if !ignoreCritical && ((typ <= 31) || ((typ & 1) == 1)) { - return nil, enc.ErrUnrecognizedField{TypeNum: typ} - } - handled = true - err = reader.Skip(int(l)) - } - if err == nil && !handled { - } - if err != nil { - return nil, enc.ErrFailToParse{TypeNum: typ, Err: err} - } - } - } - - startPos = reader.Pos() - err = nil - - if !handled_Name && err == nil { - value.Name = nil - } - if !handled_BootstrapTime && err == nil { - err = enc.ErrSkipRequired{Name: "BootstrapTime", TypeNum: 212} - } - - if err != nil { - return nil, err - } - - return value, nil -} - -func (value *MembershipTuple) Encode() enc.Wire { - encoder := MembershipTupleEncoder{} - encoder.Init(value) - return encoder.Encode(value) -} - -func (value *MembershipTuple) Bytes() []byte { - return value.Encode().Join() -} - -func ParseMembershipTuple(reader enc.WireView, ignoreCritical bool) (*MembershipTuple, error) { - context := MembershipTupleParsingContext{} - context.Init() - return context.Parse(reader, ignoreCritical) -} - type PassiveStateEncoder struct { Length uint diff --git a/std/sync/svs_encode.go b/std/sync/svs_encode.go index c4e97f12..f9fe541b 100644 --- a/std/sync/svs_encode.go +++ b/std/sync/svs_encode.go @@ -42,7 +42,12 @@ type svsSendInput struct { } // buildSvsDataForSend picks inline FULL or PARTIAL SvsData for an outgoing Sync message. +// If Threshold <= 0, the legacy StateVector-only wire is used (no mhash). func buildSvsDataForSend(in svsSendInput) *spec_svs.SvsData { + if in.Threshold <= 0 { + return &spec_svs.SvsData{StateVector: in.State.Encode(func(seq uint64) uint64 { return seq })} + } + fullSv := in.State.Encode(func(seq uint64) uint64 { return seq }) fullData := &spec_svs.SvsData{ MemberSetHash: ComputeMhash(in.State), @@ -79,6 +84,10 @@ func encodePartialStateVector(state SvMap[uint64], opts PartialEncodeOpts) *spec senderEntry = &spec_svs.StateVectorEntry{Name: opts.Sender} } + if opts.Threshold <= 0 { + return &spec_svs.StateVector{Entries: []*spec_svs.StateVectorEntry{senderEntry}} + } + // Sender-only baseline must always fit when possible. baseline := &spec_svs.StateVector{Entries: []*spec_svs.StateVectorEntry{senderEntry}} baselineData := &spec_svs.SvsData{ diff --git a/std/sync/svs_mhash.go b/std/sync/svs_mhash.go index 31435d45..510c16cc 100644 --- a/std/sync/svs_mhash.go +++ b/std/sync/svs_mhash.go @@ -4,39 +4,51 @@ import ( "crypto/sha256" "slices" - spec_svs "github.com/named-data/ndnd/std/ndn/svs/v3" + enc "github.com/named-data/ndnd/std/encoding" ) +// membershipTuple is one (Name, BootstrapTime) pair in the sync group. +type membershipTuple struct { + name enc.Name + boot uint64 +} + // ComputeMhash returns the membership hash over all (Name, BootstrapTime) pairs in state. -// Each tuple is encoded as a TLV structure (Tuple-T 0xcc with Name and BootstrapTime -// children) using the ndnd standard TLV codec. func ComputeMhash(state SvMap[uint64]) []byte { - tuples := make([]*spec_svs.MembershipTuple, 0) + members := make([]membershipTuple, 0) for name, vals := range state.Iter() { for _, val := range vals { - tuples = append(tuples, &spec_svs.MembershipTuple{ - Name: name, - BootstrapTime: val.Boot, - }) + members = append(members, membershipTuple{name: name, boot: val.Boot}) } } - slices.SortFunc(tuples, func(a, b *spec_svs.MembershipTuple) int { - if c := a.Name.Compare(b.Name); c != 0 { + slices.SortFunc(members, func(a, b membershipTuple) int { + if c := a.name.Compare(b.name); c != 0 { return c } - if a.BootstrapTime < b.BootstrapTime { + if a.boot < b.boot { return -1 } - if a.BootstrapTime > b.BootstrapTime { + if a.boot > b.boot { return 1 } return 0 }) h := sha256.New() - for _, t := range tuples { - h.Write(t.Encode().Join()) + for _, m := range members { + h.Write(m.name.Bytes()) + h.Write(bootstrapTimeTLV(m.boot)) } return h.Sum(nil) } + +// bootstrapTimeTLV encodes BOOTSTRAP-TIME-TYPE (0xD4) matching SVS v3 SeqNoEntry layout. +func bootstrapTimeTLV(boot uint64) []byte { + valLen := enc.Nat(boot).EncodingLength() + buf := make([]byte, 2+valLen) + buf[0] = 0xD4 + buf[1] = byte(valLen) + enc.Nat(boot).EncodeInto(buf[2:]) + return buf +} diff --git a/std/sync/svs_pull.go b/std/sync/svs_pull.go index 382ebc25..6ba35f9c 100644 --- a/std/sync/svs_pull.go +++ b/std/sync/svs_pull.go @@ -60,9 +60,12 @@ func buildAnnounceSvsData(state SvMap[uint64], ref enc.Name) *spec_svs.SvsData { } } -// shouldUseAnnouncePull reports whether the sender should publish at 32=sv and announce-only Sync Data -// when the local FULL encoding exceeds the configured threshold. +// shouldUseAnnouncePull reports whether the sender should publish at 32=sv and announce-only Sync Data. +// Large-group announce+pull is disabled when threshold <= 0 (legacy mode). func shouldUseAnnouncePull(reason syncSendReason, threshold int, state SvMap[uint64]) bool { + if threshold <= 0 { + return false + } if reason == syncSendRecovery { return true } @@ -110,6 +113,7 @@ func (s *SvSync) pullFullVector(ref enc.Name, trustPrefix enc.Name) { s.o.Client.ConsumeExt(ndn.ConsumeExtArgs{ Name: ref.Clone(), TryStore: true, + NoMetadata: true, UseSignatureTime: s.o.UseSignatureTime, IgnoreValidity: s.o.IgnoreValidity, Callback: func(st ndn.ConsumeState) { @@ -183,6 +187,11 @@ func (s *SvSync) sendRecoveryAnnounce() { // handleMhashMismatch schedules announce or pull recovery on membership mismatch. func (s *SvSync) handleMhashMismatch(args svSyncRecvSvArgs, recvSv SvMap[uint64]) { + // [Spec] Legacy mode (threshold 0): inline FULL is merged above; large-group recovery is off. + if s.o.SyncVectorThreshold <= 0 { + return + } + localMhash := ComputeMhash(s.state) if bytes.Equal(localMhash, args.mhash) { return diff --git a/std/sync/svs_test.go b/std/sync/svs_test.go index 6bb959d1..1f38c90b 100644 --- a/std/sync/svs_test.go +++ b/std/sync/svs_test.go @@ -5,8 +5,6 @@ import ( "testing" "time" - "github.com/stretchr/testify/require" - enc "github.com/named-data/ndnd/std/encoding" "github.com/named-data/ndnd/std/ndn" spec "github.com/named-data/ndnd/std/ndn/spec_2022" @@ -14,6 +12,7 @@ import ( sig "github.com/named-data/ndnd/std/security/signer" "github.com/named-data/ndnd/std/types/optional" tu "github.com/named-data/ndnd/std/utils/testutils" + "github.com/stretchr/testify/require" ) // --- shared test helpers and encode tests --- @@ -374,9 +373,11 @@ func TestShouldUseAnnouncePull(t *testing.T) { }).Encode().Join()) require.False(t, shouldUseAnnouncePull(syncSendPublication, fullSize-1, m)) + require.False(t, shouldUseAnnouncePull(syncSendPeriodic, 0, m)) require.False(t, shouldUseAnnouncePull(syncSendPeriodic, fullSize+1, m)) require.True(t, shouldUseAnnouncePull(syncSendPeriodic, fullSize-1, m)) require.True(t, shouldUseAnnouncePull(syncSendOther, fullSize-1, m)) + require.False(t, shouldUseAnnouncePull(syncSendRecovery, 0, m)) require.True(t, shouldUseAnnouncePull(syncSendRecovery, fullSize-1, m)) } From f9935ed55f2a3df38c49a75f5c922080e7153420 Mon Sep 17 00:00:00 2001 From: Taranum Wasu Date: Mon, 13 Jul 2026 15:26:43 +0530 Subject: [PATCH 22/24] sync: restore reviewed SVS behavior around pull fix Undo stale staged changes accidentally included with the metadata-fetch fix so the branch retains the reviewed no-legacy-wire design while keeping direct full-vector segment retrieval. Co-authored-by: Cursor --- docs/svs-v3-revision.md | 11 ++- std/ndn/svs/v3/definitions.go | 8 ++ std/ndn/svs/v3/zz_generated.go | 166 +++++++++++++++++++++++++++++++++ std/sync/svs_encode.go | 9 -- std/sync/svs_mhash.go | 40 +++----- std/sync/svs_pull.go | 12 +-- std/sync/svs_test.go | 5 +- 7 files changed, 198 insertions(+), 53 deletions(-) diff --git a/docs/svs-v3-revision.md b/docs/svs-v3-revision.md index f32d480f..f316c872 100644 --- a/docs/svs-v3-revision.md +++ b/docs/svs-v3-revision.md @@ -225,9 +225,10 @@ Stop adding entries when estimated inline `SvsData` size approaches `SyncVectorT ### 4.3 `SyncVectorThreshold` -- Configurable implementation parameter (application packet size budget). -- Default value is implementation-defined. -- When `encoded_size(FULL) ≤ SyncVectorThreshold`, nodes use **inline FULL** only (existing SVS v3 behavior). +- Configurable implementation parameter (application packet size budget) in bytes. +- When `encoded_size(FULL) ≤ SyncVectorThreshold`, nodes use **inline FULL** (with `mhash` and `VectorType=FULL`). +- When `encoded_size(FULL) > SyncVectorThreshold`, nodes switch to PARTIAL (publication) or announce+pull (periodic sync and recovery). +- This revision does **not** retain the legacy StateVector-only wire. All Sync messages carry `mhash` and a `VectorType`. --- @@ -368,9 +369,9 @@ Group exceeds `SyncVectorThreshold`. Producer `P` publishes: --- -## 8. Open Items +## 8. Interoperability -1. **Mixed-version interoperability** — how revised nodes coexist with plain SVS v3 peers in the same sync group, if at all. +This revision defines a single wire profile; it does not interoperate with plain SVS v3 peers in the same sync group. Deployments must upgrade all nodes to a revision-conformant implementation at the same time. Concretely, every Sync Data carries `mhash` and a `VectorType`, and the implementation does not emit a legacy `StateVector`-only `SvsData` under any `SyncVectorThreshold` value (a `Threshold` of 0 falls back to the default 1200-byte budget). --- diff --git a/std/ndn/svs/v3/definitions.go b/std/ndn/svs/v3/definitions.go index a3fcf415..4412d154 100644 --- a/std/ndn/svs/v3/definitions.go +++ b/std/ndn/svs/v3/definitions.go @@ -42,6 +42,14 @@ type SeqNoEntry struct { SeqNo uint64 `tlv:"0xd6"` } +// MembershipTuple is one (Name, BootstrapTime) pair used to compute MemberSetHash. +type MembershipTuple struct { + //+field:name + Name enc.Name `tlv:"0x07"` + //+field:natural + BootstrapTime uint64 `tlv:"0xd4"` +} + // +tlv-model:nocopy type PassiveState struct { //+field:sequence:[]byte:binary:[]byte diff --git a/std/ndn/svs/v3/zz_generated.go b/std/ndn/svs/v3/zz_generated.go index 138dbed3..6d4590e7 100644 --- a/std/ndn/svs/v3/zz_generated.go +++ b/std/ndn/svs/v3/zz_generated.go @@ -837,6 +837,172 @@ func ParseSeqNoEntry(reader enc.WireView, ignoreCritical bool) (*SeqNoEntry, err return context.Parse(reader, ignoreCritical) } +type MembershipTupleEncoder struct { + Length uint + + Name_length uint +} + +type MembershipTupleParsingContext struct { +} + +func (encoder *MembershipTupleEncoder) Init(value *MembershipTuple) { + if value.Name != nil { + encoder.Name_length = 0 + for _, c := range value.Name { + encoder.Name_length += uint(c.EncodingLength()) + } + } + + l := uint(0) + if value.Name != nil { + l += 1 + l += uint(enc.TLNum(encoder.Name_length).EncodingLength()) + l += encoder.Name_length + } + l += 1 + l += uint(1 + enc.Nat(value.BootstrapTime).EncodingLength()) + encoder.Length = l + +} + +func (context *MembershipTupleParsingContext) Init() { + +} + +func (encoder *MembershipTupleEncoder) EncodeInto(value *MembershipTuple, buf []byte) { + + pos := uint(0) + + if value.Name != nil { + buf[pos] = byte(7) + pos += 1 + pos += uint(enc.TLNum(encoder.Name_length).EncodeInto(buf[pos:])) + for _, c := range value.Name { + pos += uint(c.EncodeInto(buf[pos:])) + } + } + buf[pos] = byte(212) + pos += 1 + + buf[pos] = byte(enc.Nat(value.BootstrapTime).EncodeInto(buf[pos+1:])) + pos += uint(1 + buf[pos]) +} + +func (encoder *MembershipTupleEncoder) Encode(value *MembershipTuple) enc.Wire { + + wire := make(enc.Wire, 1) + wire[0] = make([]byte, encoder.Length) + buf := wire[0] + encoder.EncodeInto(value, buf) + + return wire +} + +func (context *MembershipTupleParsingContext) Parse(reader enc.WireView, ignoreCritical bool) (*MembershipTuple, error) { + + var handled_Name bool = false + var handled_BootstrapTime bool = false + + progress := -1 + _ = progress + + value := &MembershipTuple{} + var err error + var startPos int + for { + startPos = reader.Pos() + if startPos >= reader.Length() { + break + } + typ := enc.TLNum(0) + l := enc.TLNum(0) + typ, err = reader.ReadTLNum() + if err != nil { + return nil, enc.ErrFailToParse{TypeNum: 0, Err: err} + } + l, err = reader.ReadTLNum() + if err != nil { + return nil, enc.ErrFailToParse{TypeNum: 0, Err: err} + } + + err = nil + if handled := false; true { + switch typ { + case 7: + if true { + handled = true + handled_Name = true + delegate := reader.Delegate(int(l)) + value.Name, err = delegate.ReadName() + } + case 212: + if true { + handled = true + handled_BootstrapTime = true + value.BootstrapTime = uint64(0) + { + for i := 0; i < int(l); i++ { + x := byte(0) + x, err = reader.ReadByte() + if err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + break + } + value.BootstrapTime = uint64(value.BootstrapTime<<8) | uint64(x) + } + } + } + default: + if !ignoreCritical && ((typ <= 31) || ((typ & 1) == 1)) { + return nil, enc.ErrUnrecognizedField{TypeNum: typ} + } + handled = true + err = reader.Skip(int(l)) + } + if err == nil && !handled { + } + if err != nil { + return nil, enc.ErrFailToParse{TypeNum: typ, Err: err} + } + } + } + + startPos = reader.Pos() + err = nil + + if !handled_Name && err == nil { + value.Name = nil + } + if !handled_BootstrapTime && err == nil { + err = enc.ErrSkipRequired{Name: "BootstrapTime", TypeNum: 212} + } + + if err != nil { + return nil, err + } + + return value, nil +} + +func (value *MembershipTuple) Encode() enc.Wire { + encoder := MembershipTupleEncoder{} + encoder.Init(value) + return encoder.Encode(value) +} + +func (value *MembershipTuple) Bytes() []byte { + return value.Encode().Join() +} + +func ParseMembershipTuple(reader enc.WireView, ignoreCritical bool) (*MembershipTuple, error) { + context := MembershipTupleParsingContext{} + context.Init() + return context.Parse(reader, ignoreCritical) +} + type PassiveStateEncoder struct { Length uint diff --git a/std/sync/svs_encode.go b/std/sync/svs_encode.go index f9fe541b..c4e97f12 100644 --- a/std/sync/svs_encode.go +++ b/std/sync/svs_encode.go @@ -42,12 +42,7 @@ type svsSendInput struct { } // buildSvsDataForSend picks inline FULL or PARTIAL SvsData for an outgoing Sync message. -// If Threshold <= 0, the legacy StateVector-only wire is used (no mhash). func buildSvsDataForSend(in svsSendInput) *spec_svs.SvsData { - if in.Threshold <= 0 { - return &spec_svs.SvsData{StateVector: in.State.Encode(func(seq uint64) uint64 { return seq })} - } - fullSv := in.State.Encode(func(seq uint64) uint64 { return seq }) fullData := &spec_svs.SvsData{ MemberSetHash: ComputeMhash(in.State), @@ -84,10 +79,6 @@ func encodePartialStateVector(state SvMap[uint64], opts PartialEncodeOpts) *spec senderEntry = &spec_svs.StateVectorEntry{Name: opts.Sender} } - if opts.Threshold <= 0 { - return &spec_svs.StateVector{Entries: []*spec_svs.StateVectorEntry{senderEntry}} - } - // Sender-only baseline must always fit when possible. baseline := &spec_svs.StateVector{Entries: []*spec_svs.StateVectorEntry{senderEntry}} baselineData := &spec_svs.SvsData{ diff --git a/std/sync/svs_mhash.go b/std/sync/svs_mhash.go index 510c16cc..31435d45 100644 --- a/std/sync/svs_mhash.go +++ b/std/sync/svs_mhash.go @@ -4,51 +4,39 @@ import ( "crypto/sha256" "slices" - enc "github.com/named-data/ndnd/std/encoding" + spec_svs "github.com/named-data/ndnd/std/ndn/svs/v3" ) -// membershipTuple is one (Name, BootstrapTime) pair in the sync group. -type membershipTuple struct { - name enc.Name - boot uint64 -} - // ComputeMhash returns the membership hash over all (Name, BootstrapTime) pairs in state. +// Each tuple is encoded as a TLV structure (Tuple-T 0xcc with Name and BootstrapTime +// children) using the ndnd standard TLV codec. func ComputeMhash(state SvMap[uint64]) []byte { - members := make([]membershipTuple, 0) + tuples := make([]*spec_svs.MembershipTuple, 0) for name, vals := range state.Iter() { for _, val := range vals { - members = append(members, membershipTuple{name: name, boot: val.Boot}) + tuples = append(tuples, &spec_svs.MembershipTuple{ + Name: name, + BootstrapTime: val.Boot, + }) } } - slices.SortFunc(members, func(a, b membershipTuple) int { - if c := a.name.Compare(b.name); c != 0 { + slices.SortFunc(tuples, func(a, b *spec_svs.MembershipTuple) int { + if c := a.Name.Compare(b.Name); c != 0 { return c } - if a.boot < b.boot { + if a.BootstrapTime < b.BootstrapTime { return -1 } - if a.boot > b.boot { + if a.BootstrapTime > b.BootstrapTime { return 1 } return 0 }) h := sha256.New() - for _, m := range members { - h.Write(m.name.Bytes()) - h.Write(bootstrapTimeTLV(m.boot)) + for _, t := range tuples { + h.Write(t.Encode().Join()) } return h.Sum(nil) } - -// bootstrapTimeTLV encodes BOOTSTRAP-TIME-TYPE (0xD4) matching SVS v3 SeqNoEntry layout. -func bootstrapTimeTLV(boot uint64) []byte { - valLen := enc.Nat(boot).EncodingLength() - buf := make([]byte, 2+valLen) - buf[0] = 0xD4 - buf[1] = byte(valLen) - enc.Nat(boot).EncodeInto(buf[2:]) - return buf -} diff --git a/std/sync/svs_pull.go b/std/sync/svs_pull.go index 6ba35f9c..91977536 100644 --- a/std/sync/svs_pull.go +++ b/std/sync/svs_pull.go @@ -60,12 +60,9 @@ func buildAnnounceSvsData(state SvMap[uint64], ref enc.Name) *spec_svs.SvsData { } } -// shouldUseAnnouncePull reports whether the sender should publish at 32=sv and announce-only Sync Data. -// Large-group announce+pull is disabled when threshold <= 0 (legacy mode). +// shouldUseAnnouncePull reports whether the sender should publish at 32=sv and announce-only Sync Data +// when the local FULL encoding exceeds the configured threshold. func shouldUseAnnouncePull(reason syncSendReason, threshold int, state SvMap[uint64]) bool { - if threshold <= 0 { - return false - } if reason == syncSendRecovery { return true } @@ -187,11 +184,6 @@ func (s *SvSync) sendRecoveryAnnounce() { // handleMhashMismatch schedules announce or pull recovery on membership mismatch. func (s *SvSync) handleMhashMismatch(args svSyncRecvSvArgs, recvSv SvMap[uint64]) { - // [Spec] Legacy mode (threshold 0): inline FULL is merged above; large-group recovery is off. - if s.o.SyncVectorThreshold <= 0 { - return - } - localMhash := ComputeMhash(s.state) if bytes.Equal(localMhash, args.mhash) { return diff --git a/std/sync/svs_test.go b/std/sync/svs_test.go index 1f38c90b..6bb959d1 100644 --- a/std/sync/svs_test.go +++ b/std/sync/svs_test.go @@ -5,6 +5,8 @@ import ( "testing" "time" + "github.com/stretchr/testify/require" + enc "github.com/named-data/ndnd/std/encoding" "github.com/named-data/ndnd/std/ndn" spec "github.com/named-data/ndnd/std/ndn/spec_2022" @@ -12,7 +14,6 @@ import ( sig "github.com/named-data/ndnd/std/security/signer" "github.com/named-data/ndnd/std/types/optional" tu "github.com/named-data/ndnd/std/utils/testutils" - "github.com/stretchr/testify/require" ) // --- shared test helpers and encode tests --- @@ -373,11 +374,9 @@ func TestShouldUseAnnouncePull(t *testing.T) { }).Encode().Join()) require.False(t, shouldUseAnnouncePull(syncSendPublication, fullSize-1, m)) - require.False(t, shouldUseAnnouncePull(syncSendPeriodic, 0, m)) require.False(t, shouldUseAnnouncePull(syncSendPeriodic, fullSize+1, m)) require.True(t, shouldUseAnnouncePull(syncSendPeriodic, fullSize-1, m)) require.True(t, shouldUseAnnouncePull(syncSendOther, fullSize-1, m)) - require.False(t, shouldUseAnnouncePull(syncSendRecovery, 0, m)) require.True(t, shouldUseAnnouncePull(syncSendRecovery, fullSize-1, m)) } From 91f49f839c45f9b763d1ab0a1a919ededdda8631 Mon Sep 17 00:00:00 2001 From: Taranum Wasu Date: Mon, 13 Jul 2026 22:35:53 +0530 Subject: [PATCH 23/24] sync: fix e2e on sprint topology Co-authored-by: Cursor --- std/object/client_consume_seg.go | 2 +- std/sync/svs.go | 6 ++++++ std/sync/svs_alo_data.go | 2 ++ std/sync/svs_pull.go | 18 ++++++++++++++++++ 4 files changed, 27 insertions(+), 1 deletion(-) diff --git a/std/object/client_consume_seg.go b/std/object/client_consume_seg.go index 8fa9440e..b5c2978e 100644 --- a/std/object/client_consume_seg.go +++ b/std/object/client_consume_seg.go @@ -56,7 +56,7 @@ func newRrSegFetcher(client *Client) rrSegFetcher { outstanding: 0, retxQueue: list.New(), txCounter: make(map[*ConsumeState]int), - maxRetries: 3, + maxRetries: 5, } } diff --git a/std/sync/svs.go b/std/sync/svs.go index edb69ef1..3e36659d 100644 --- a/std/sync/svs.go +++ b/std/sync/svs.go @@ -43,6 +43,10 @@ type SvSync struct { // Prefix for published full State Vector Data (.../32=sv). fullVectorPrefix enc.Name + // lastPullTime debounces pullFullVector per sender so a sync storm across + // many peers does not generate thousands of redundant segment-0 fetches. + lastPullTime map[string]time.Time + // cancellation for face hook faceCancel func() } @@ -171,6 +175,8 @@ func NewSvSync(opts SvSyncOpts) *SvSync { fullVectorPrefix: resolveFullVectorPrefix(opts.FullVectorPrefix, opts.SyncDataName), + lastPullTime: make(map[string]time.Time), + faceCancel: func() {}, } } diff --git a/std/sync/svs_alo_data.go b/std/sync/svs_alo_data.go index 27543385..8b4df760 100644 --- a/std/sync/svs_alo_data.go +++ b/std/sync/svs_alo_data.go @@ -118,6 +118,8 @@ func (s *SvsALO) consumeObject(node enc.Name, boot uint64, seq uint64) { fetchName := s.objectName(node, boot, seq) s.client.ConsumeExt(ndn.ConsumeExtArgs{ Name: fetchName, + TryStore: true, + NoMetadata: true, // fetch name includes version+seq; metadata would only block on timeout UseSignatureTime: s.opts.Svs.UseSignatureTime, IgnoreValidity: s.opts.Svs.IgnoreValidity, Callback: func(status ndn.ConsumeState) { diff --git a/std/sync/svs_pull.go b/std/sync/svs_pull.go index 91977536..bcfed12a 100644 --- a/std/sync/svs_pull.go +++ b/std/sync/svs_pull.go @@ -3,6 +3,7 @@ package sync import ( "bytes" "fmt" + "time" enc "github.com/named-data/ndnd/std/encoding" "github.com/named-data/ndnd/std/log" @@ -96,6 +97,13 @@ func (s *SvSync) publishFullVectorData(state SvMap[uint64]) (enc.Name, error) { }) } +// pullFullVectorMinInterval debounces pullFullVector per sender. During convergence on a +// large group, every sync that crosses the membership hash boundary schedules a pull; without +// gating, a node can accumulate redundant segment-0 fetches for the same content, which +// exhausts retry budgets under network load. We allow at most one pull per sender per +// pullFullVectorMinInterval. +const pullFullVectorMinInterval = 5 * time.Second + // pullFullVector fetches a published full State Vector and merges it on the main loop. // trustPrefix is the sender's .../32=sv prefix; ref must be equal to or below it. func (s *SvSync) pullFullVector(ref enc.Name, trustPrefix enc.Name) { @@ -107,6 +115,16 @@ func (s *SvSync) pullFullVector(ref enc.Name, trustPrefix enc.Name) { return } + // Debounce per sender: drop the pull if one is already in flight or completed recently. + senderHash := trustPrefix.TlvStr() + s.mutex.Lock() + if last, ok := s.lastPullTime[senderHash]; ok && time.Since(last) < pullFullVectorMinInterval { + s.mutex.Unlock() + return + } + s.lastPullTime[senderHash] = time.Now() + s.mutex.Unlock() + s.o.Client.ConsumeExt(ndn.ConsumeExtArgs{ Name: ref.Clone(), TryStore: true, From 3bb17d6c4a739d5697e26de1bfb3b52a8e2ff3fb Mon Sep 17 00:00:00 2001 From: Taranum Wasu Date: Tue, 14 Jul 2026 02:10:15 +0530 Subject: [PATCH 24/24] object: bump metadata fetch retries 3 to 5 Symmetric with the prior segment fetcher fix (maxRetries 3 to 5). Both fetchMetadata and fetchDataByPrefix carry the same TODO: configurable comment and the same 1-second per-attempt lifetime. Under sprint topology load with CI's resource-constrained runner, 3 retries (~4s budget) was just barely insufficient for the first cat: the FATAL arrived 4.4s after ndnd cat started, exactly exhausting the budget. 5 retries (~6s budget) gives the CI runner margin to absorb the same transient packet loss without false-failing. Co-Authored-By: Claude Co-authored-by: Cursor --- std/object/client_consume.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/std/object/client_consume.go b/std/object/client_consume.go index 64ba3e3e..d5899dbe 100644 --- a/std/object/client_consume.go +++ b/std/object/client_consume.go @@ -119,7 +119,7 @@ func (c *Client) fetchMetadata( MustBeFresh: true, Lifetime: optional.Some(time.Millisecond * 1000), }, - Retries: 3, // TODO: configurable + Retries: 5, // TODO: configurable - increased from 3 for sprint topology reliability TryStore: utils.If(tryStore, c.store, nil), Callback: func(args ndn.ExpressCallbackArgs) { if args.Result == ndn.InterestResultError { @@ -176,7 +176,7 @@ func (c *Client) fetchDataByPrefix( MustBeFresh: true, Lifetime: optional.Some(time.Millisecond * 1000), }, - Retries: 3, // TODO: configurable + Retries: 5, // TODO: configurable - increased from 3 for sprint topology reliability TryStore: utils.If(tryStore, c.store, nil), Callback: func(args ndn.ExpressCallbackArgs) { if args.Result == ndn.InterestResultError {