diff --git a/cmd/mithril/node/node.go b/cmd/mithril/node/node.go index bbf994ca..edb7af08 100644 --- a/cmd/mithril/node/node.go +++ b/cmd/mithril/node/node.go @@ -1139,7 +1139,7 @@ func runLive(c *cobra.Command, args []string) { // Handle explicit --snapshot flag (bypasses all auto-discovery, does NOT delete snapshot files) if snapshotArchivePath != "" { - mlog.Log.Infof("Using snapshot file: %s", snapshotArchivePath) + mlog.Log.Infof("Using full snapshot: %s", snapshotArchivePath) // Parse full snapshot slot from filename for validation fullSnapshotSlot := parseSlotFromSnapshotName(filepath.Base(snapshotArchivePath)) diff --git a/pkg/accountsdb/accountsdb.go b/pkg/accountsdb/accountsdb.go index b8d90a42..c6f701b6 100644 --- a/pkg/accountsdb/accountsdb.go +++ b/pkg/accountsdb/accountsdb.go @@ -40,6 +40,12 @@ type AccountsDb struct { inProgressStoreRequests *list.List storeRequestChan chan *list.Element storeWorkerDone chan struct{} + + // InRewardsWindow is set during partitioned epoch rewards distribution. + // When true, stake accounts are not cached in CommonAcctsCache since they're + // one-shot reads/writes that would evict genuinely hot accounts. + // Atomic for safe concurrent access from RPC goroutines. + InRewardsWindow atomic.Bool } type storeRequest struct { @@ -254,8 +260,12 @@ func (accountsDb *AccountsDb) getStoredAccount(slot uint64, pubkey solana.Public acct.Slot = acctIdxEntry.Slot - if solana.PublicKeyFromBytes(acct.Owner[:]) == addresses.VoteProgramAddr { + owner := solana.PublicKeyFromBytes(acct.Owner[:]) + if owner == addresses.VoteProgramAddr { accountsDb.VoteAcctCache.Set(pubkey, acct) + } else if owner == addresses.StakeProgramAddr && accountsDb.InRewardsWindow.Load() { + // During reward distribution, stake accounts are one-shot reads that would + // evict genuinely hot accounts from the cache. Skip caching them. } else { accountsDb.CommonAcctsCache.Set(pubkey, acct) } @@ -327,13 +337,17 @@ func (accountsDb *AccountsDb) storeAccountsSync(accts []*accounts.Account, slot accountsDb.parallelStoreAccounts(StoreAccountsWorkers, accts, slot) } + inRewardsWindow := accountsDb.InRewardsWindow.Load() for _, acct := range accts { if acct == nil { continue } - // if vote account, do not serialize up and write into accountsdb - just save it in cache. - if solana.PublicKeyFromBytes(acct.Owner[:]) == addresses.VoteProgramAddr { + owner := solana.PublicKeyFromBytes(acct.Owner[:]) + if owner == addresses.VoteProgramAddr { accountsDb.VoteAcctCache.Set(acct.Key, acct) + } else if owner == addresses.StakeProgramAddr && inRewardsWindow { + // During reward distribution, stake accounts are one-shot writes that would + // evict genuinely hot accounts from the cache. Skip caching them. } else { accountsDb.CommonAcctsCache.Set(acct.Key, acct) } diff --git a/pkg/rent/rent.go b/pkg/rent/rent.go index 979d7d45..f554bc49 100644 --- a/pkg/rent/rent.go +++ b/pkg/rent/rent.go @@ -42,12 +42,12 @@ func rentStateFromAcct(acct *accounts.Account, rent *sealevel.SysvarRent) *RentS } } -func NewRentStateInfo(rent *sealevel.SysvarRent, txCtx *sealevel.TransactionCtx, tx *solana.Transaction, f *features.Features) []*RentStateInfo { +func NewRentStateInfo(rent *sealevel.SysvarRent, txCtx *sealevel.TransactionCtx, f *features.Features, programIDSet map[solana.PublicKey]struct{}) []*RentStateInfo { rentStateInfos := make([]*RentStateInfo, 0, len(txCtx.Accounts.Accounts)) acctsMetas := txCtx.Accounts.AcctMetas for idx, acct := range txCtx.Accounts.Accounts { - if sealevel.IsWritable(tx, acctsMetas[idx], f) { + if sealevel.IsWritable(acctsMetas[idx], f, programIDSet) { rentStateInfo := rentStateFromAcct(acct, rent) rentStateInfos = append(rentStateInfos, rentStateInfo) } else { diff --git a/pkg/replay/accounts.go b/pkg/replay/accounts.go index 7cbd518f..9bafcbf4 100644 --- a/pkg/replay/accounts.go +++ b/pkg/replay/accounts.go @@ -2,6 +2,7 @@ package replay import ( "slices" + "sync/atomic" "github.com/Overclock-Validator/mithril/pkg/accounts" "github.com/Overclock-Validator/mithril/pkg/addresses" @@ -11,14 +12,48 @@ import ( "github.com/gagliardetto/solana-go" ) -func loadAndValidateTxAccts(slotCtx *sealevel.SlotCtx, acctMetasPerInstr [][]sealevel.AccountMeta, tx *solana.Transaction, instrs []sealevel.Instruction, instrsAcct *accounts.Account, loadedAcctBytesLimit uint32) (*sealevel.TransactionAccounts, error) { +// Account clone tracking for profiling copy-on-write optimization potential +var ( + // Per-transaction account clone stats (loaded in loadAndValidateTxAcctsSimd186) + TxAcctsCloned atomic.Uint64 // Total accounts cloned across all txs + TxAcctsClonedBytes atomic.Uint64 // Total bytes of account data cloned + + // Per-transaction modification stats (touched in handleModifiedAccounts) + TxAcctsTouched atomic.Uint64 // Total accounts actually modified + TxAcctsTouchedBytes atomic.Uint64 // Total bytes of modified account data + + // Transaction count for averaging + TxCount atomic.Uint64 +) + +// CloneStats holds account clone/modify metrics for reporting +type CloneStats struct { + AcctsCloned uint64 // Accounts loaded (cloned) + AcctsClonedBytes uint64 // Bytes cloned + AcctsTouched uint64 // Accounts modified + AcctsTouchedBytes uint64 // Bytes of modified accounts + TxCount uint64 // Number of transactions +} + +// GetAndResetCloneStats returns current clone stats and resets counters +func GetAndResetCloneStats() CloneStats { + return CloneStats{ + AcctsCloned: TxAcctsCloned.Swap(0), + AcctsClonedBytes: TxAcctsClonedBytes.Swap(0), + AcctsTouched: TxAcctsTouched.Swap(0), + AcctsTouchedBytes: TxAcctsTouchedBytes.Swap(0), + TxCount: TxCount.Swap(0), + } +} + +func loadAndValidateTxAccts(slotCtx *sealevel.SlotCtx, acctMetasPerInstr [][]sealevel.AccountMeta, tx *solana.Transaction, instrs []sealevel.Instruction, instrsAcct *accounts.Account, loadedAcctBytesLimit uint32) (*sealevel.TransactionAccounts, []*solana.AccountMeta, error) { txAcctMetas, err := tx.AccountMetaList() if err != nil { - return nil, err + return nil, nil, err } var programIdIdxs []uint64 - instructionAcctPubkeys := make(map[solana.PublicKey]struct{}) + instructionAcctPubkeys := make(map[solana.PublicKey]struct{}, len(tx.Message.AccountKeys)) for instrIdx, instr := range tx.Message.Instructions { programIdIdxs = append(programIdIdxs, uint64(instr.ProgramIDIndex)) @@ -43,20 +78,20 @@ func loadAndValidateTxAccts(slotCtx *sealevel.SlotCtx, acctMetasPerInstr [][]sea } else if !slotCtx.Features.IsActive(features.DisableAccountLoaderSpecialCase) && slices.Contains(programIdIdxs, uint64(idx)) && !acctMeta.IsWritable && !instrContainsAcctMeta { tmp, err := slotCtx.GetAccount(acctMeta.PublicKey) if err != nil { - return nil, err + return nil, nil, err } acct = &accounts.Account{Key: acctMeta.PublicKey, Owner: tmp.Owner, Executable: true, IsDummy: true} } else { acct, err = slotCtx.GetAccount(acctMeta.PublicKey) if err != nil { - return nil, err + return nil, nil, err } } if !isInstructionsSysvarAcct { loadedBytesAccumulator = safemath.SaturatingAddU32(loadedBytesAccumulator, uint32(len(acct.Data))) if loadedBytesAccumulator > loadedAcctBytesLimit { - return nil, TxErrMaxLoadedAccountsDataSizeExceeded + return nil, nil, TxErrMaxLoadedAccountsDataSizeExceeded } } @@ -69,7 +104,7 @@ func loadAndValidateTxAccts(slotCtx *sealevel.SlotCtx, acctMetasPerInstr [][]sea transactionAccts.AcctMetas = convertedAcctMetas removeAcctsExecutableFlagChecks := slotCtx.Features.IsActive(features.RemoveAccountsExecutableFlagChecks) - validatedLoaders := make(map[solana.PublicKey]struct{}) + validatedLoaders := make(map[solana.PublicKey]struct{}, 4) // Usually ≤4 loaders for _, instr := range instrs { if instr.ProgramId == addresses.NativeLoaderAddr { @@ -78,15 +113,15 @@ func loadAndValidateTxAccts(slotCtx *sealevel.SlotCtx, acctMetasPerInstr [][]sea programAcct, err := slotCtx.GetAccount(instr.ProgramId) if err != nil { - return nil, TxErrProgramAccountNotFound + return nil, nil, TxErrProgramAccountNotFound } if programAcct.Lamports == 0 { - return nil, TxErrProgramAccountNotFound + return nil, nil, TxErrProgramAccountNotFound } if !removeAcctsExecutableFlagChecks && !programAcct.Executable { - return nil, TxErrInvalidProgramForExecution + return nil, nil, TxErrInvalidProgramForExecution } owner := programAcct.Owner @@ -101,24 +136,24 @@ func loadAndValidateTxAccts(slotCtx *sealevel.SlotCtx, acctMetasPerInstr [][]sea if err != nil { ownerAcct, err = slotCtx.GetAccountFromAccountsDb(owner) if err != nil { - return nil, TxErrInvalidProgramForExecution + return nil, nil, TxErrInvalidProgramForExecution } } if ownerAcct.Owner != addresses.NativeLoaderAddr || (!removeAcctsExecutableFlagChecks && !ownerAcct.Executable) { - return nil, TxErrInvalidProgramForExecution + return nil, nil, TxErrInvalidProgramForExecution } loadedBytesAccumulator = safemath.SaturatingAddU32(loadedBytesAccumulator, uint32(len(ownerAcct.Data))) if loadedBytesAccumulator > loadedAcctBytesLimit { - return nil, TxErrMaxLoadedAccountsDataSizeExceeded + return nil, nil, TxErrMaxLoadedAccountsDataSizeExceeded } validatedLoaders[owner] = struct{}{} } } - return transactionAccts, nil + return transactionAccts, txAcctMetas, nil } const ( @@ -207,7 +242,7 @@ func isLoaderAcct(owner solana.PublicKey) bool { owner == addresses.LoaderV4Addr } -func loadAndValidateTxAcctsSimd186(slotCtx *sealevel.SlotCtx, acctMetasPerInstr [][]sealevel.AccountMeta, tx *solana.Transaction, instrs []sealevel.Instruction, instrsAcct *accounts.Account, loadedAcctBytesLimit uint32) (*sealevel.TransactionAccounts, error) { +func loadAndValidateTxAcctsSimd186(slotCtx *sealevel.SlotCtx, acctMetasPerInstr [][]sealevel.AccountMeta, tx *solana.Transaction, instrs []sealevel.Instruction, instrsAcct *accounts.Account, loadedAcctBytesLimit uint32) (*sealevel.TransactionAccounts, []*solana.AccountMeta, error) { acctKeys := tx.Message.AccountKeys accumulator := NewLoadedAcctSizeAccumulatorSimd186(slotCtx, uint64(loadedAcctBytesLimit), @@ -216,30 +251,45 @@ func loadAndValidateTxAcctsSimd186(slotCtx *sealevel.SlotCtx, acctMetasPerInstr addrTableLookupCost := safemath.SaturatingMulU64(uint64(len(tx.Message.AddressTableLookups)), addrLookupTableBaseSize) err := accumulator.add(addrTableLookupCost) if err != nil { - return nil, err + return nil, nil, err } - for _, pubkey := range acctKeys { + // Memoize accounts loaded in Pass 1 to avoid re-cloning in Pass 2 + // Use slice indexed by account position (same ordering as txAcctMetas) + acctCache := make([]*accounts.Account, len(acctKeys)) + + var clonedBytes uint64 + for i, pubkey := range acctKeys { acct, err := slotCtx.GetAccount(pubkey) if err != nil { panic("should be impossible - programming error") } + acctCache[i] = acct // Cache by index for reuse in Pass 2 + clonedBytes += uint64(len(acct.Data)) err = accumulator.collectAcct(acct) if err != nil { - return nil, err + return nil, nil, err } } + // Track clone stats for profiling + TxAcctsCloned.Add(uint64(len(acctKeys))) + TxAcctsClonedBytes.Add(clonedBytes) + txAcctMetas, err := tx.AccountMetaList() if err != nil { - return nil, err + return nil, nil, err } - var programIdIdxs []uint64 - instructionAcctPubkeys := make(map[solana.PublicKey]struct{}) + // Use boolean mask for O(1) program index lookup + isProgramIdx := make([]bool, len(acctKeys)) + instructionAcctPubkeys := make(map[solana.PublicKey]struct{}, len(acctKeys)) for instrIdx, instr := range tx.Message.Instructions { - programIdIdxs = append(programIdIdxs, uint64(instr.ProgramIDIndex)) + i := int(instr.ProgramIDIndex) + if i >= 0 && i < len(isProgramIdx) { + isProgramIdx[i] = true + } ias := acctMetasPerInstr[instrIdx] for _, ia := range ias { instructionAcctPubkeys[ia.Pubkey] = struct{}{} @@ -251,21 +301,17 @@ func loadAndValidateTxAcctsSimd186(slotCtx *sealevel.SlotCtx, acctMetasPerInstr for idx, acctMeta := range txAcctMetas { var acct *accounts.Account + cached := acctCache[idx] // Reuse account from Pass 1 _, instrContainsAcctMeta := instructionAcctPubkeys[acctMeta.PublicKey] if acctMeta.PublicKey == sealevel.SysvarInstructionsAddr { acct = instrsAcct - } else if !slotCtx.Features.IsActive(features.DisableAccountLoaderSpecialCase) && slices.Contains(programIdIdxs, uint64(idx)) && !acctMeta.IsWritable && !instrContainsAcctMeta { - tmp, err := slotCtx.GetAccount(acctMeta.PublicKey) - if err != nil { - return nil, err - } - acct = &accounts.Account{Key: acctMeta.PublicKey, Owner: tmp.Owner, Executable: true, IsDummy: true} + } else if !slotCtx.Features.IsActive(features.DisableAccountLoaderSpecialCase) && isProgramIdx[idx] && !acctMeta.IsWritable && !instrContainsAcctMeta { + // Dummy account case - only need owner from cached account + acct = &accounts.Account{Key: acctMeta.PublicKey, Owner: cached.Owner, Executable: true, IsDummy: true} } else { - acct, err = slotCtx.GetAccount(acctMeta.PublicKey) - if err != nil { - return nil, err - } + // Normal case - use cached account directly + acct = cached } acctsForTx = append(acctsForTx, *acct) @@ -278,32 +324,44 @@ func loadAndValidateTxAcctsSimd186(slotCtx *sealevel.SlotCtx, acctMetasPerInstr removeAcctsExecutableFlagChecks := slotCtx.Features.IsActive(features.RemoveAccountsExecutableFlagChecks) - for _, instr := range instrs { + for instrIdx, instr := range instrs { if instr.ProgramId == addresses.NativeLoaderAddr { continue } - programAcct, err := slotCtx.GetAccount(instr.ProgramId) - if err != nil { - programAcct, err = slotCtx.GetAccountFromAccountsDb(instr.ProgramId) + // Use cached account via ProgramIDIndex from tx.Message + programIdx := int(tx.Message.Instructions[instrIdx].ProgramIDIndex) + var programAcct *accounts.Account + if programIdx >= 0 && programIdx < len(acctCache) { + programAcct = acctCache[programIdx] + } + + // Fallback if not in cache or out of bounds + if programAcct == nil { + var err error + programAcct, err = slotCtx.GetAccount(instr.ProgramId) if err != nil { - return nil, TxErrProgramAccountNotFound + programAcct, err = slotCtx.GetAccountFromAccountsDb(instr.ProgramId) + if err != nil { + return nil, nil, TxErrProgramAccountNotFound + } } } if programAcct.Lamports == 0 { - return nil, TxErrProgramAccountNotFound + return nil, nil, TxErrProgramAccountNotFound } if !removeAcctsExecutableFlagChecks && !programAcct.Executable { - return nil, TxErrInvalidProgramForExecution + return nil, nil, TxErrInvalidProgramForExecution } owner := programAcct.Owner if owner != addresses.NativeLoaderAddr && !isLoaderAcct(owner) { - return nil, TxErrInvalidProgramForExecution + return nil, nil, TxErrInvalidProgramForExecution } } - return transactionAccts, nil + TxCount.Add(1) + return transactionAccts, txAcctMetas, nil } diff --git a/pkg/replay/block.go b/pkg/replay/block.go index 8d56b8bc..5f132da2 100644 --- a/pkg/replay/block.go +++ b/pkg/replay/block.go @@ -1696,6 +1696,19 @@ func ReplayBlocks( mlog.Log.InfofPrecise(" execution: median %.3fs, min %.3fs, max %.3fs | wait: median %.3fs, min %.3fs, max %.3fs | replay total: median %.3fs", medExec, minExec, maxExec, medWait, minWait, maxWait, medTotal) + // Account clone stats for copy-on-write optimization profiling + cloneStats := GetAndResetCloneStats() + if cloneStats.TxCount > 0 { + modifyRatio := float64(cloneStats.AcctsTouched) / float64(cloneStats.AcctsCloned) * 100 + avgAcctsPerTx := float64(cloneStats.AcctsCloned) / float64(cloneStats.TxCount) + avgTouchedPerTx := float64(cloneStats.AcctsTouched) / float64(cloneStats.TxCount) + clonedMB := float64(cloneStats.AcctsClonedBytes) / 1024 / 1024 + touchedMB := float64(cloneStats.AcctsTouchedBytes) / 1024 / 1024 + mlog.Log.InfofPrecise(" clone stats: %.1f%% modified (%d/%d accts) | %.1fMB cloned, %.1fMB modified | avg/tx: %.1f cloned, %.1f modified", + modifyRatio, cloneStats.AcctsTouched, cloneStats.AcctsCloned, + clonedMB, touchedMB, avgAcctsPerTx, avgTouchedPerTx) + } + // Line 4: RPC/fetch debugging info if fetchStats.Attempts > 0 { retryRate := float64(fetchStats.Retries) / float64(fetchStats.Attempts) * 100 @@ -1792,7 +1805,7 @@ func runIncinerator(slotCtx *sealevel.SlotCtx) { func compileWritableAndModifiedAccts(slotCtx *sealevel.SlotCtx, block *b.Block, rentAccts []*accounts.Account) ([]*accounts.Account, []*accounts.Account) { writableAccts := make([]*accounts.Account, 0, len(slotCtx.WritableAccts)+len(block.UpdatedAccts)+len(rentAccts)+4) modifiedAccts := make([]*accounts.Account, 0, len(slotCtx.ModifiedAccts)+len(block.UpdatedAccts)+len(rentAccts)+4) - alreadyAdded := make(map[solana.PublicKey]bool) + alreadyAdded := make(map[solana.PublicKey]bool, len(slotCtx.WritableAccts)) for pk := range slotCtx.WritableAccts { acct, _ := slotCtx.GetAccount(pk) diff --git a/pkg/replay/rewards.go b/pkg/replay/rewards.go index 42894686..806d8aec 100644 --- a/pkg/replay/rewards.go +++ b/pkg/replay/rewards.go @@ -218,7 +218,17 @@ func distributePartitionedEpochRewardsForSlot(acctsDb *accountsdb.AccountsDb, ep epochRewards.MustUnmarshalWithDecoder(decoder) partitionIdx := currentBlockHeight - epochRewards.DistributionStartingBlockHeight - distributedAccts, parentDistributedAccts, distributedLamports := rewards.DistributeStakingRewardsForPartition(acctsDb, partitionedEpochRewardsInfo.RewardPartitions.Partition(partitionIdx), partitionedEpochRewardsInfo.StakingRewards, currentSlot) + + // Initialize shared worker pool on first partition (reused across all 243 partitions) + if partitionedEpochRewardsInfo.WorkerPool == nil { + if err := partitionedEpochRewardsInfo.InitWorkerPool(); err != nil { + panic(fmt.Sprintf("unable to initialize reward distribution worker pool: %s", err)) + } + } + + // Set flag to prevent stake account cache pollution during one-shot reward reads/writes + acctsDb.InRewardsWindow.Store(true) + distributedAccts, parentDistributedAccts, distributedLamports := rewards.DistributeStakingRewardsForPartition(acctsDb, partitionedEpochRewardsInfo.RewardPartitions.Partition(partitionIdx), partitionedEpochRewardsInfo.StakingRewards, currentSlot, partitionedEpochRewardsInfo.WorkerPool) parentDistributedAccts = append(parentDistributedAccts, epochRewardsAcct.Clone()) epochRewards.Distribute(distributedLamports) @@ -226,6 +236,8 @@ func distributePartitionedEpochRewardsForSlot(acctsDb *accountsdb.AccountsDb, ep if partitionedEpochRewardsInfo.NumRewardPartitionsRemaining == 0 { epochRewards.Active = false + acctsDb.InRewardsWindow.Store(false) + partitionedEpochRewardsInfo.ReleaseWorkerPool() } writer := new(bytes.Buffer) diff --git a/pkg/replay/topsort_planner.go b/pkg/replay/topsort_planner.go index 8df560b5..e5767951 100644 --- a/pkg/replay/topsort_planner.go +++ b/pkg/replay/topsort_planner.go @@ -65,7 +65,7 @@ func blockToDependencyGraph(b *block.Block) (adjacencyList [][]tx, inDegree []in //start := time.Now() // Map between pubkeys and account indices var acctToPk []solana.PublicKey - pkToAcct := make(map[solana.PublicKey]acct) + pkToAcct := make(map[solana.PublicKey]acct, len(b.Transactions)*4) for i, txMeta := range b.TxMetas { tx := b.Transactions[i] diff --git a/pkg/replay/transaction.go b/pkg/replay/transaction.go index 31b3cec0..533bcfd5 100644 --- a/pkg/replay/transaction.go +++ b/pkg/replay/transaction.go @@ -6,7 +6,6 @@ import ( "fmt" "math" "runtime/trace" - "slices" "strings" "sync" "time" @@ -67,28 +66,38 @@ func newExecCtx(slotCtx *sealevel.SlotCtx, transactionAccts *sealevel.Transactio execCtx.Accounts = accounts.NewMemAccounts() execCtx.SlotCtx = slotCtx execCtx.TransactionContext.ComputeBudgetLimits = computeBudgetLimits - execCtx.ModifiedVoteStates = make(map[solana.PublicKey]*sealevel.VoteStateVersions) + execCtx.ModifiedVoteStates = make(map[solana.PublicKey]*sealevel.VoteStateVersions, 8) return execCtx } -func instrsAndAcctMetasFromTx(tx *solana.Transaction, f *features.Features) ([]sealevel.Instruction, [][]sealevel.AccountMeta, error) { +func instrsAndAcctMetasFromTx(tx *solana.Transaction, f *features.Features) ([]sealevel.Instruction, [][]sealevel.AccountMeta, map[solana.PublicKey]struct{}, error) { instrs := make([]sealevel.Instruction, 0, len(tx.Message.Instructions)) acctMetasPerInstr := make([][]sealevel.AccountMeta, 0, len(tx.Message.Instructions)) + // Build programIDSet once for O(1) lookups in isWritable + programIDs, err := tx.GetProgramIDs() + if err != nil { + return nil, nil, nil, err + } + programIDSet := make(map[solana.PublicKey]struct{}, len(programIDs)) + for _, pid := range programIDs { + programIDSet[pid] = struct{}{} + } + for _, compiledInstr := range tx.Message.Instructions { programId, err := tx.ResolveProgramIDIndex(compiledInstr.ProgramIDIndex) if err != nil { - return nil, nil, err + return nil, nil, nil, err } ams, err := compiledInstr.ResolveInstructionAccounts(&tx.Message) if err != nil { - return nil, nil, err + return nil, nil, nil, err } - var acctMetas []sealevel.AccountMeta + acctMetas := make([]sealevel.AccountMeta, 0, len(ams)) for _, am := range ams { - acctMeta := sealevel.AccountMeta{Pubkey: am.PublicKey, IsSigner: am.IsSigner, IsWritable: isWritable(tx, am, f)} + acctMeta := sealevel.AccountMeta{Pubkey: am.PublicKey, IsSigner: am.IsSigner, IsWritable: isWritable(am, f, programIDSet)} acctMetas = append(acctMetas, acctMeta) } @@ -97,7 +106,7 @@ func instrsAndAcctMetasFromTx(tx *solana.Transaction, f *features.Features) ([]s acctMetasPerInstr = append(acctMetasPerInstr, acctMetas) } - return instrs, acctMetasPerInstr, nil + return instrs, acctMetasPerInstr, programIDSet, nil } func fixupInstructionsSysvarAcct(execCtx *sealevel.ExecutionCtx, instrIdx uint16) error { @@ -115,11 +124,7 @@ func fixupInstructionsSysvarAcct(execCtx *sealevel.ExecutionCtx, instrIdx uint16 return nil } -var newReservedAccts = []solana.PublicKey{a.AddressLookupTableAddr, a.ComputeBudgetProgramAddr, - a.Ed25519PrecompileAddr, a.LoaderV4Addr, a.Secp256kPrecompileAddr, a.ZkElgamalProofProgramAddr, - a.ZkTokenProofProgramAddr, sealevel.SysvarEpochRewardsAddr, sealevel.SysvarLastRestartSlotAddr, a.SysvarOwnerAddr} - -func isWritable(tx *solana.Transaction, am *solana.AccountMeta, f *features.Features) bool { +func isWritable(am *solana.AccountMeta, f *features.Features, programIDSet map[solana.PublicKey]struct{}) bool { if !am.IsWritable { return false } @@ -129,7 +134,7 @@ func isWritable(tx *solana.Transaction, am *solana.AccountMeta, f *features.Feat } if f.IsActive(features.AddNewReservedAccountKeys) { - if slices.Contains(newReservedAccts, am.PublicKey) { + if _, isReserved := sealevel.NewReservedAcctsSet[am.PublicKey]; isReserved { return false } } @@ -140,15 +145,8 @@ func isWritable(tx *solana.Transaction, am *solana.AccountMeta, f *features.Feat } } - programIds, err := tx.GetProgramIDs() - if err != nil { - panic(err) - } - - for _, programId := range programIds { - if am.PublicKey == programId { - return false - } + if _, isProgramID := programIDSet[am.PublicKey]; isProgramID { + return false } return true @@ -156,8 +154,13 @@ func isWritable(tx *solana.Transaction, am *solana.AccountMeta, f *features.Feat func handleModifiedAccounts(slotCtx *sealevel.SlotCtx, execCtx *sealevel.ExecutionCtx) { // update account states in slotCtx for all accounts 'touched' during the tx's execution + var touchedCount, touchedBytes uint64 for idx, newAcctState := range execCtx.TransactionContext.Accounts.Accounts { if execCtx.TransactionContext.Accounts.Touched[idx] { + // Track touched account stats for profiling + touchedCount++ + touchedBytes += uint64(len(newAcctState.Data)) + // clean up accounts closed during the tx (garbage collection) if newAcctState.Lamports == 0 { newAcctState = &accounts.Account{Key: newAcctState.Key, RentEpoch: math.MaxUint64} @@ -171,6 +174,10 @@ func handleModifiedAccounts(slotCtx *sealevel.SlotCtx, execCtx *sealevel.Executi //mlog.Log.Debugf("modified account %s after tx", newAcctState.Key) } } + + // Record touched stats for clone optimization profiling + TxAcctsTouched.Add(touchedCount) + TxAcctsTouchedBytes.Add(touchedBytes) } func recordStakeDelegation(acct *accounts.Account) { @@ -221,11 +228,11 @@ func recordVoteTimestampAndSlot(slotCtx *sealevel.SlotCtx, acct *accounts.Accoun slotCtx.VoteTimestamps[acct.Key] = timestamp } -func recordStakeAndVoteAccounts(slotCtx *sealevel.SlotCtx, execCtx *sealevel.ExecutionCtx, writablePubkeys []solana.PublicKey) { +func recordStakeAndVoteAccounts(slotCtx *sealevel.SlotCtx, execCtx *sealevel.ExecutionCtx, writablePubkeySet map[solana.PublicKey]struct{}) { modifiedVoteAccts := execCtx.TransactionContext.ModifiedVoteAccts for _, acct := range execCtx.TransactionContext.Accounts.Accounts { - if !slices.Contains(writablePubkeys, acct.Key) { + if _, isWritable := writablePubkeySet[acct.Key]; !isWritable { continue } @@ -320,7 +327,7 @@ func ProcessTransaction(slotCtx *sealevel.SlotCtx, sigverifyWg *sync.WaitGroup, defer mlog.Log.DisableInfLogging() } - instrs, acctMetasPerInstr, err := instrsAndAcctMetasFromTx(tx, slotCtx.Features) + instrs, acctMetasPerInstr, programIDSet, err := instrsAndAcctMetasFromTx(tx, slotCtx.Features) if err != nil { return nil, err } @@ -341,11 +348,12 @@ func ProcessTransaction(slotCtx *sealevel.SlotCtx, sigverifyWg *sync.WaitGroup, start = time.Now() var transactionAccts *sealevel.TransactionAccounts + var txAcctMetas []*solana.AccountMeta if slotCtx.Features.IsActive(features.FormalizeLoadedTransactionDataSize) { - transactionAccts, err = loadAndValidateTxAcctsSimd186(slotCtx, acctMetasPerInstr, tx, instrs, instrsAcct, computeBudgetLimits.LoadedAccountBytes) + transactionAccts, txAcctMetas, err = loadAndValidateTxAcctsSimd186(slotCtx, acctMetasPerInstr, tx, instrs, instrsAcct, computeBudgetLimits.LoadedAccountBytes) } else { - transactionAccts, err = loadAndValidateTxAccts(slotCtx, acctMetasPerInstr, tx, instrs, instrsAcct, computeBudgetLimits.LoadedAccountBytes) + transactionAccts, txAcctMetas, err = loadAndValidateTxAccts(slotCtx, acctMetasPerInstr, tx, instrs, instrsAcct, computeBudgetLimits.LoadedAccountBytes) } if err == TxErrMaxLoadedAccountsDataSizeExceeded || err == TxErrInvalidProgramForExecution || err == TxErrProgramAccountNotFound { return handleFailedTx(slotCtx, tx, instrs, computeBudgetLimits, err, nil) @@ -410,7 +418,7 @@ func ProcessTransaction(slotCtx *sealevel.SlotCtx, sigverifyWg *sync.WaitGroup, start = time.Now() rent.MaybeSetRentExemptRentEpochMax(slotCtx, &rentSysvar, &execCtx.Features, &execCtx.TransactionContext.Accounts) - preTxRentStates := rent.NewRentStateInfo(&rentSysvar, execCtx.TransactionContext, tx, &execCtx.Features) + preTxRentStates := rent.NewRentStateInfo(&rentSysvar, execCtx.TransactionContext, &execCtx.Features, programIDSet) metrics.GlobalBlockReplay.PreTxRentStates.AddTimingSince(start) var instrErr error @@ -477,7 +485,7 @@ func ProcessTransaction(slotCtx *sealevel.SlotCtx, sigverifyWg *sync.WaitGroup, } start = time.Now() - postTxRentStates := rent.NewRentStateInfo(&rentSysvar, execCtx.TransactionContext, tx, &execCtx.Features) + postTxRentStates := rent.NewRentStateInfo(&rentSysvar, execCtx.TransactionContext, &execCtx.Features, programIDSet) rentStateErr := rent.VerifyRentStateChanges(preTxRentStates, postTxRentStates, execCtx.TransactionContext) metrics.GlobalBlockReplay.PostTxRentStates.AddTimingSince(start) @@ -518,14 +526,13 @@ func ProcessTransaction(slotCtx *sealevel.SlotCtx, sigverifyWg *sync.WaitGroup, return handleFailedTx(slotCtx, tx, instrs, computeBudgetLimits, instrErr, rentStateErr) } - txAcctMetas, err := tx.AccountMetaList() - if err != nil { - panic(err) - } - + // Reuse txAcctMetas from loadAndValidateTxAccts* (already built once per tx) + // Build writablePubkeySet inline to avoid second loop + writablePubkeySet := make(map[solana.PublicKey]struct{}, len(txAcctMetas)) for _, txAcctMeta := range txAcctMetas { - if isWritable(tx, txAcctMeta, &execCtx.Features) { + if isWritable(txAcctMeta, &execCtx.Features, programIDSet) { writablePubkeys = append(writablePubkeys, txAcctMeta.PublicKey) + writablePubkeySet[txAcctMeta.PublicKey] = struct{}{} } } @@ -535,7 +542,8 @@ func ProcessTransaction(slotCtx *sealevel.SlotCtx, sigverifyWg *sync.WaitGroup, handleModifiedAccounts(slotCtx, execCtx) writablePubkeys = append(writablePubkeys, payerAcct.Key) - recordStakeAndVoteAccounts(slotCtx, execCtx, writablePubkeys) + writablePubkeySet[payerAcct.Key] = struct{}{} + recordStakeAndVoteAccounts(slotCtx, execCtx, writablePubkeySet) metrics.GlobalBlockReplay.TxUpdateAccounts.AddTimingSince(start) return txFeeInfo, nil diff --git a/pkg/rewards/rewards.go b/pkg/rewards/rewards.go index cb073eff..df72efc4 100644 --- a/pkg/rewards/rewards.go +++ b/pkg/rewards/rewards.go @@ -36,6 +36,87 @@ type PartitionedRewardDistributionInfo struct { Credits map[solana.PublicKey]CalculatedStakePoints RewardPartitions Partitions StakingRewards map[solana.PublicKey]*CalculatedStakeRewards + WorkerPool *ants.PoolWithFunc +} + +// rewardDistributionTask carries all context needed for processing one stake account. +// Used with the shared worker pool to avoid per-partition pool creation overhead. +type rewardDistributionTask struct { + acctsDb *accountsdb.AccountsDb + slot uint64 + stakingRewards map[solana.PublicKey]*CalculatedStakeRewards + accts []*accounts.Account + parentAccts []*accounts.Account + distributedLamports *atomic.Uint64 + wg *sync.WaitGroup + idx int + pubkey solana.PublicKey +} + +// rewardDistributionWorker is the shared worker function for stake reward distribution. +func rewardDistributionWorker(i interface{}) { + task := i.(*rewardDistributionTask) + defer task.wg.Done() + + reward, ok := task.stakingRewards[task.pubkey] + if !ok { + return + } + + stakeAcct, err := task.acctsDb.GetAccount(task.slot, task.pubkey) + if err != nil { + panic(fmt.Sprintf("unable to get acct %s from acctsdb for partitioned epoch rewards distribution in slot %d", task.pubkey, task.slot)) + } + task.parentAccts[task.idx] = stakeAcct.Clone() + + stakeState, err := sealevel.UnmarshalStakeState(stakeAcct.Data) + if err != nil { + return + } + + stakeState.Stake.Stake.CreditsObserved = reward.NewCreditsObserved + stakeState.Stake.Stake.Delegation.StakeLamports = safemath.SaturatingAddU64(stakeState.Stake.Stake.Delegation.StakeLamports, uint64(reward.StakerRewards)) + + err = sealevel.MarshalStakeStakeInto(stakeState, stakeAcct.Data) + if err != nil { + panic(fmt.Sprintf("unable to serialize new stake account state in distributing partitioned rewards: %s", err)) + } + + stakeAcct.Lamports, err = safemath.CheckedAddU64(stakeAcct.Lamports, uint64(reward.StakerRewards)) + if err != nil { + panic(fmt.Sprintf("overflow in partitioned epoch rewards distribution in slot %d to acct %s: %s", task.slot, task.pubkey, err)) + } + + task.accts[task.idx] = stakeAcct + task.distributedLamports.Add(reward.StakerRewards) + + // update the stake cache + delegationToCache := stakeState.Stake.Stake.Delegation + delegationToCache.CreditsObserved = stakeState.Stake.Stake.CreditsObserved + global.PutStakeCacheItem(task.pubkey, &delegationToCache) +} + +// InitWorkerPool creates the shared worker pool for reward distribution. +// Call once at the start of partitioned rewards, before processing any partition. +func (info *PartitionedRewardDistributionInfo) InitWorkerPool() error { + if info.WorkerPool != nil { + return nil + } + size := runtime.GOMAXPROCS(0) * 8 + pool, err := ants.NewPoolWithFunc(size, rewardDistributionWorker) + if err != nil { + return err + } + info.WorkerPool = pool + return nil +} + +// ReleaseWorkerPool releases the shared pool. Call when NumRewardPartitionsRemaining == 0. +func (info *PartitionedRewardDistributionInfo) ReleaseWorkerPool() { + if info.WorkerPool != nil { + info.WorkerPool.Release() + info.WorkerPool = nil + } } type CalculatedStakePoints struct { @@ -163,7 +244,6 @@ func DistributeVotingRewards(acctsDb *accountsdb.AccountsDb, validatorRewards ma wg.Wait() workerPool.Release() - ants.Release() err := acctsDb.StoreAccounts(updatedAccts, slot, nil) if err != nil { @@ -173,77 +253,30 @@ func DistributeVotingRewards(acctsDb *accountsdb.AccountsDb, validatorRewards ma return updatedAccts, parentUpdatedAccts, totalVotingRewards.Load() } -type idxAndPubkey struct { - idx int - pubkey solana.PublicKey -} - -func DistributeStakingRewardsForPartition(acctsDb *accountsdb.AccountsDb, partition *Partition, stakingRewards map[solana.PublicKey]*CalculatedStakeRewards, slot uint64) ([]*accounts.Account, []*accounts.Account, uint64) { +func DistributeStakingRewardsForPartition(acctsDb *accountsdb.AccountsDb, partition *Partition, stakingRewards map[solana.PublicKey]*CalculatedStakeRewards, slot uint64, workerPool *ants.PoolWithFunc) ([]*accounts.Account, []*accounts.Account, uint64) { var distributedLamports atomic.Uint64 accts := make([]*accounts.Account, partition.NumPubkeys()) parentAccts := make([]*accounts.Account, partition.NumPubkeys()) var wg sync.WaitGroup - size := runtime.GOMAXPROCS(0) * 8 - workerPool, _ := ants.NewPoolWithFunc(size, func(i interface{}) { - defer wg.Done() - - ip := i.(idxAndPubkey) - idx := ip.idx - stakePk := ip.pubkey - - reward, ok := stakingRewards[stakePk] - if !ok { - return - } - - stakeAcct, err := acctsDb.GetAccount(slot, stakePk) - if err != nil { - panic(fmt.Sprintf("unable to get acct %s from acctsdb for partitioned epoch rewards distribution in slot %d", stakePk, slot)) - } - parentAccts[idx] = stakeAcct.Clone() - - // update the delegation in the stake account state - stakeState, err := sealevel.UnmarshalStakeState(stakeAcct.Data) - if err != nil { - return - } - - stakeState.Stake.Stake.CreditsObserved = reward.NewCreditsObserved - stakeState.Stake.Stake.Delegation.StakeLamports = safemath.SaturatingAddU64(stakeState.Stake.Stake.Delegation.StakeLamports, uint64(reward.StakerRewards)) - - newStakeStateBytes, err := sealevel.MarshalStakeStake(stakeState) - if err != nil { - panic(fmt.Sprintf("unable to serialize new stake account state in distributing partitioned rewards: %s", err)) - } - copy(stakeAcct.Data, newStakeStateBytes) - - // update lamports in stake account - stakeAcct.Lamports, err = safemath.CheckedAddU64(stakeAcct.Lamports, uint64(reward.StakerRewards)) - if err != nil { - panic(fmt.Sprintf("overflow in partitioned epoch rewards distribution in slot %d to acct %s: %s", slot, stakePk, err)) - } - - accts[idx] = stakeAcct - distributedLamports.Add(reward.StakerRewards) - - // update the stake cache - delegationToCache := stakeState.Stake.Stake.Delegation - delegationToCache.CreditsObserved = stakeState.Stake.Stake.CreditsObserved - global.PutStakeCacheItem(stakePk, &delegationToCache) - }) - for idx, stakePk := range partition.Pubkeys() { - ip := idxAndPubkey{idx: idx, pubkey: stakePk} + task := &rewardDistributionTask{ + acctsDb: acctsDb, + slot: slot, + stakingRewards: stakingRewards, + accts: accts, + parentAccts: parentAccts, + distributedLamports: &distributedLamports, + wg: &wg, + idx: idx, + pubkey: stakePk, + } wg.Add(1) - workerPool.Invoke(ip) + workerPool.Invoke(task) } wg.Wait() - workerPool.Release() - ants.Release() - err := acctsDb.StoreAccounts(accts, slot, nil) if err != nil { panic(fmt.Sprintf("error updating accounts for partitioned epoch rewards in slot %d: %s", slot, err)) @@ -356,7 +389,6 @@ func CalculateStakeRewardsAndPartitions(pointsPerStakeAcct map[solana.PublicKey] } wg.Wait() partitionCalcWorkerPool.Release() - ants.Release() return stakeInfoResults, validatorRewards, partitions } @@ -503,7 +535,6 @@ func CalculateStakePoints( wg.Wait() workerPool.Release() - ants.Release() return pointsAccum.CalculatedStakePoints(), pointsAccum.TotalPoints() } diff --git a/pkg/sealevel/sealevel.go b/pkg/sealevel/sealevel.go index ac5807b6..ce70e5df 100644 --- a/pkg/sealevel/sealevel.go +++ b/pkg/sealevel/sealevel.go @@ -2,7 +2,6 @@ package sealevel import ( "bytes" - "slices" a "github.com/Overclock-Validator/mithril/pkg/addresses" "github.com/Overclock-Validator/mithril/pkg/features" @@ -35,7 +34,22 @@ func (t *TransactionCtx) newVMOpts(params *Params) *sbpf.VMOpts { } } -func IsWritable(tx *solana.Transaction, am *AccountMeta, f *features.Features) bool { +// NewReservedAcctsSet contains reserved account addresses that should not be writable. +// Exported so transaction.go can use the same set (avoiding duplication/drift). +var NewReservedAcctsSet = map[solana.PublicKey]struct{}{ + a.AddressLookupTableAddr: {}, + a.ComputeBudgetProgramAddr: {}, + a.Ed25519PrecompileAddr: {}, + a.LoaderV4Addr: {}, + a.Secp256kPrecompileAddr: {}, + a.ZkElgamalProofProgramAddr: {}, + a.ZkTokenProofProgramAddr: {}, + SysvarEpochRewardsAddr: {}, + SysvarLastRestartSlotAddr: {}, + a.SysvarOwnerAddr: {}, +} + +func IsWritable(am *AccountMeta, f *features.Features, programIDSet map[solana.PublicKey]struct{}) bool { if !am.IsWritable { return false } @@ -45,7 +59,7 @@ func IsWritable(tx *solana.Transaction, am *AccountMeta, f *features.Features) b } if f.IsActive(features.AddNewReservedAccountKeys) { - if slices.Contains(newReservedAccts, am.Pubkey) { + if _, isReserved := NewReservedAcctsSet[am.Pubkey]; isReserved { return false } } @@ -56,24 +70,13 @@ func IsWritable(tx *solana.Transaction, am *AccountMeta, f *features.Features) b } } - programIds, err := tx.GetProgramIDs() - if err != nil { - panic(err) - } - - for _, programId := range programIds { - if am.Pubkey == programId { - return false - } + if _, isProgramID := programIDSet[am.Pubkey]; isProgramID { + return false } return true } -var newReservedAccts = []solana.PublicKey{a.AddressLookupTableAddr, a.ComputeBudgetProgramAddr, - a.Ed25519PrecompileAddr, a.LoaderV4Addr, a.Secp256kPrecompileAddr, a.ZkElgamalProofProgramAddr, - a.ZkTokenProofProgramAddr, SysvarEpochRewardsAddr, SysvarLastRestartSlotAddr, a.SysvarOwnerAddr} - func IsSysvar(pubkey solana.PublicKey) bool { if pubkey == SysvarClockAddr || pubkey == SysvarEpochScheduleAddr || pubkey == SysvarFeesAddr || pubkey == SysvarInstructionsAddr || diff --git a/pkg/sealevel/stake_state.go b/pkg/sealevel/stake_state.go index 873e6c9e..d45043f4 100644 --- a/pkg/sealevel/stake_state.go +++ b/pkg/sealevel/stake_state.go @@ -2,6 +2,7 @@ package sealevel import ( "bytes" + "fmt" "math" "github.com/Overclock-Validator/mithril/pkg/features" @@ -765,6 +766,31 @@ func MarshalStakeStake(state *StakeStateV2) ([]byte, error) { } } +// fixedSliceWriter implements io.Writer over a fixed-size byte slice, +// avoiding allocation during serialization. +type fixedSliceWriter struct { + buf []byte + pos int +} + +func (w *fixedSliceWriter) Write(p []byte) (int, error) { + if w.pos+len(p) > len(w.buf) { + return 0, fmt.Errorf("write exceeds buffer: pos=%d, write=%d, cap=%d", w.pos, len(p), len(w.buf)) + } + copy(w.buf[w.pos:], p) + w.pos += len(p) + return len(p), nil +} + +// MarshalStakeStakeInto writes the stake state directly into dst, avoiding allocation. +// dst should be at least StakeStateV2Size (200) bytes for valid stake accounts. +func MarshalStakeStakeInto(state *StakeStateV2, dst []byte) error { + writer := &fixedSliceWriter{buf: dst, pos: 0} + encoder := bin.NewBinEncoder(writer) + + return state.MarshalWithEncoder(encoder) +} + func setStakeAccountState(acct *BorrowedAccount, stakeState *StakeStateV2, f features.Features) error { stakeStateBytes, err := MarshalStakeStake(stakeState) if err != nil { diff --git a/pkg/snapshot/build_db.go b/pkg/snapshot/build_db.go index e08b34f9..9b6605f2 100644 --- a/pkg/snapshot/build_db.go +++ b/pkg/snapshot/build_db.go @@ -164,21 +164,21 @@ func BuildAccountsDbPaths( // Clean any leftover artifacts from previous incomplete runs (e.g., Ctrl+C) CleanAccountsDbDir(accountsDbDir) - mlog.Log.Infof("Parsing manifest from %s", snapshotFile) + mlog.Log.Infof("Parsing full snapshot manifest...") manifest, err := UnmarshalManifestFromSnapshot(ctx, snapshotFile, accountsDbDir) if err != nil { return nil, nil, fmt.Errorf("reading snapshot manifest: %v", err) } - mlog.Log.Infof("Parsed manifest from full snapshot") + mlog.Log.Infof("Parsed full snapshot manifest") var incrementalManifest *SnapshotManifest if incrementalSnapshotFile != "" { - mlog.Log.Infof("Parsing manifest from %s", incrementalSnapshotFile) + mlog.Log.Infof("Parsing incremental snapshot manifest...") incrementalManifest, err = UnmarshalManifestFromSnapshot(ctx, incrementalSnapshotFile, accountsDbDir) if err != nil { return nil, nil, fmt.Errorf("reading incremental snapshot manifest: %v", err) } - mlog.Log.Infof("Parsed manifest from incremental snapshot") + mlog.Log.Infof("Parsed incremental snapshot manifest") } start := time.Now() diff --git a/pkg/snapshot/build_db_with_incr.go b/pkg/snapshot/build_db_with_incr.go index a8e80cbb..6af452b0 100644 --- a/pkg/snapshot/build_db_with_incr.go +++ b/pkg/snapshot/build_db_with_incr.go @@ -50,12 +50,12 @@ func BuildAccountsDbAuto( // Clean any leftover artifacts from previous incomplete runs (e.g., Ctrl+C) CleanAccountsDbDir(accountsDbDir) - mlog.Log.Infof("Parsing manifest from %s", fullSnapshotFile) + mlog.Log.Infof("Parsing full snapshot manifest...") manifest, err := UnmarshalManifestFromSnapshot(ctx, fullSnapshotFile, accountsDbDir) if err != nil { return nil, nil, fmt.Errorf("reading snapshot manifest: %v", err) } - mlog.Log.Infof("Parsed manifest from full snapshot") + mlog.Log.Infof("Parsed full snapshot manifest") start := time.Now() @@ -170,7 +170,7 @@ func BuildAccountsDbAuto( mlog.Log.Infof("Found new incremental snapshot URL: %s (slot %d)", incrementalSnapshotPath, incrSlot) } - mlog.Log.Infof("Parsing manifest from %s", incrementalSnapshotPath) + mlog.Log.Infof("Parsing incremental snapshot manifest...") incrementalManifestCopy, err := UnmarshalManifestFromSnapshot(ctx, incrementalSnapshotPath, accountsDbDir) if err != nil { mlog.Log.Errorf("reading incremental snapshot manifest: %v", err) @@ -178,7 +178,7 @@ func BuildAccountsDbAuto( } // Copy the manifest so the worker pool's pointer has the value. *incrementalManifest = *incrementalManifestCopy - mlog.Log.Infof("Parsed manifest from incremental snapshot") + mlog.Log.Infof("Parsed incremental snapshot manifest") // Determine save path for incremental snapshot if streaming from HTTP var incrSavePath string