From 7c1dd87ca687cd2714c439440e836210caf93167 Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 2 Nov 2026 15:06:58 +0800 Subject: [PATCH 01/26] fix(rate): generate API keys with cryptographic entropy --- util/rate/util.go | 13 +++++------ util/rate/util_test.go | 50 +++++++++++++++++++++--------------------- 2 files changed, 30 insertions(+), 33 deletions(-) diff --git a/util/rate/util.go b/util/rate/util.go index 803f8e67..d0806be9 100644 --- a/util/rate/util.go +++ b/util/rate/util.go @@ -1,8 +1,7 @@ package rate import ( - "math/rand" - "time" + "crypto/rand" "github.com/btcsuite/btcutil/base58" "github.com/pkg/errors" @@ -13,15 +12,13 @@ const ( ) func GenerateRandomLimitKey(limitType LimitType) (string, error) { - data := make([]byte, LimitKeyLength) - - source := rand.NewSource(time.Now().UnixNano()) - r := rand.New(source) - r.Read(data) - if limitType != LimitTypeByKey && limitType != LimitTypeByIp { return "", errors.New("invalid limit type") } + data := make([]byte, LimitKeyLength) + if _, err := rand.Read(data); err != nil { + return "", errors.Wrap(err, "failed to generate limit key") + } secretb := []byte(base58.Encode(data)) secretb[0] = '0' + uint8(limitType) diff --git a/util/rate/util_test.go b/util/rate/util_test.go index 17ef129b..94a7330b 100644 --- a/util/rate/util_test.go +++ b/util/rate/util_test.go @@ -1,33 +1,33 @@ package rate import ( + "bytes" + "crypto/rand" + "errors" + "github.com/btcsuite/btcutil/base58" + "github.com/stretchr/testify/require" "testing" - - "github.com/stretchr/testify/assert" ) -func TestGenerateRandomLimitKey(t *testing.T) { - // limit type by IP - limitKey, err := GenerateRandomLimitKey(LimitTypeByIp) - assert.Nil(t, err) - - assert.Greater(t, len(limitKey), LimitKeyLength) - assert.Less(t, len(limitKey), LimitKeyLength*2) - - t.Logf("New random limit key of type `byIP`: %v", limitKey) - assert.Equal(t, limitKey[0], uint8('0'+LimitTypeByIp)) +type failingRandomReader struct{} - // limit type by Key - limitKey, err = GenerateRandomLimitKey(LimitTypeByKey) - assert.Nil(t, err) - - assert.Greater(t, len(limitKey), LimitKeyLength) - assert.Less(t, len(limitKey), LimitKeyLength*2) - - t.Logf("New random limit key of type `byKey`: %v", limitKey) - assert.Equal(t, limitKey[0], uint8('0'+LimitTypeByKey)) - - // invalid limit type - _, err = GenerateRandomLimitKey(-1) - assert.NotNil(t, err) +func (failingRandomReader) Read([]byte) (int, error) { return 0, errors.New("entropy unavailable") } +func TestGenerateRandomLimitKey(t *testing.T) { + original := rand.Reader + t.Cleanup(func() { rand.Reader = original }) + for _, kind := range []LimitType{LimitTypeByKey, LimitTypeByIp} { + entropy := bytes.Repeat([]byte{42}, LimitKeyLength) + rand.Reader = bytes.NewReader(entropy) + key, err := GenerateRandomLimitKey(kind) + require.NoError(t, err) + expected := []byte(base58.Encode(entropy)) + expected[0] = '0' + byte(kind) + require.Equal(t, string(expected), key) + } + rand.Reader = failingRandomReader{} + key, err := GenerateRandomLimitKey(LimitTypeByKey) + require.ErrorContains(t, err, "entropy unavailable") + require.Empty(t, key) + _, err = GenerateRandomLimitKey(LimitType(255)) + require.Error(t, err) } From 59467333169706197467e9195712a5ecab46f3bb Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 2 Nov 2026 15:13:05 +0800 Subject: [PATCH 02/26] fix(rate): remove reload lock inversion and preserve fractional rates --- util/rate/registry.go | 26 +++++++++++---- util/rate/registry_concurrency_test.go | 46 ++++++++++++++++++++++++++ util/rate/registry_reload.go | 10 ++++-- 3 files changed, 73 insertions(+), 9 deletions(-) create mode 100644 util/rate/registry_concurrency_test.go diff --git a/util/rate/registry.go b/util/rate/registry.go index 7b74dbd3..0a302b47 100644 --- a/util/rate/registry.go +++ b/util/rate/registry.go @@ -3,6 +3,7 @@ package rate import ( "context" "sync" + "sync/atomic" "time" "github.com/Conflux-Chain/confura/util/acl" @@ -38,8 +39,9 @@ type Registry struct { *http.Registry *aclRegistry - mu sync.Mutex - kloader *KeyLoader + mu sync.Mutex + limiterStrategies atomic.Value // immutable map[string]*Strategy; Create never takes mu + kloader *KeyLoader // all available strategies strategies map[string]*Strategy // strategy name => *Strategy @@ -62,6 +64,7 @@ func NewRegistry(kloader *KeyLoader, valFactory acl.ValidatorFactory) *Registry ), } + m.publishLimiterStrategies() m.Registry = http.NewRegistry(m) go m.ScheduleGC(GCScheduleInterval) @@ -112,10 +115,8 @@ func (r *Registry) GetGroupAndKey( } func (r *Registry) Create(ctx context.Context, resource, group string) (rate.Limiter, error) { - r.mu.Lock() - defer r.mu.Unlock() - - stg, ok := r.strategies[group] + strategies := r.limiterStrategies.Load().(map[string]*Strategy) + stg, ok := strategies[group] if !ok { return nil, errors.New("strategy not found") } @@ -133,10 +134,21 @@ func (r *Registry) createWithOption(option interface{}) (l rate.Limiter, err err case FixedWindowOption: l = rate.NewFixedWindow(opt.Interval, opt.Quota) case TokenBucketOption: - l = rate.NewTokenBucket(int(opt.Rate), opt.Burst) + l = rate.NewTokenBucketRate(opt.Rate, opt.Burst) default: err = errors.New("invalid limit option") } return l, err } + +// publishLimiterStrategies is called with mu held (or before publication at construction). +// The dependency invokes Create with its own lock held. Publish an immutable +// snapshot so Create does not reverse reload's mu -> registry lock order. +func (r *Registry) publishLimiterStrategies() { + snapshot := make(map[string]*Strategy, len(r.strategies)) + for name, strategy := range r.strategies { + snapshot[name] = strategy + } + r.limiterStrategies.Store(snapshot) +} diff --git a/util/rate/registry_concurrency_test.go b/util/rate/registry_concurrency_test.go new file mode 100644 index 00000000..712239b6 --- /dev/null +++ b/util/rate/registry_concurrency_test.go @@ -0,0 +1,46 @@ +package rate + +import ( + "context" + "testing" + "time" + + limiter "github.com/Conflux-Chain/go-conflux-util/rate" +) + +func TestCreateDoesNotAcquireStrategyLock(t *testing.T) { + r := &Registry{strategies: map[string]*Strategy{"default": {Name: "default", LimitOptions: map[string]interface{}{"rpc": TokenBucketOption{Rate: 1, Burst: 1}}}}} + r.publishLimiterStrategies() + // Reload holds mu while waiting for the dependency registry lock. The + // dependency holds that lock when it invokes Create: Create must progress. + r.mu.Lock() + defer r.mu.Unlock() + done := make(chan error, 1) + go func() { _, err := r.Create(context.Background(), "rpc", "default"); done <- err }() + select { + case err := <-done: + if err != nil { + t.Fatal(err) + } + case <-time.After(time.Second): + t.Fatal("Create waits for reload lock") + } +} + +func TestFractionalTokenBucket(t *testing.T) { + r := &Registry{} + l, err := r.createWithOption(TokenBucketOption{Rate: 0.5, Burst: 1}) + if err != nil { + t.Fatal(err) + } + bucket := l.(*limiter.TokenBucket) + // Past times make this test independent of the old dependency's Delay() + // wall-clock implementation as well as its corrected DelayFrom(now). + now := time.Now().Add(-10 * time.Second) + if err := bucket.LimitAt(now, 1); err != nil { + t.Fatal(err) + } + if err := bucket.LimitAt(now.Add(2*time.Second), 1); err != nil { + t.Fatalf("fractional rate did not refill: %v", err) + } +} diff --git a/util/rate/registry_reload.go b/util/rate/registry_reload.go index d83d1475..023cb77f 100644 --- a/util/rate/registry_reload.go +++ b/util/rate/registry_reload.go @@ -101,24 +101,30 @@ func (m *Registry) getVipStrategy(tier handlers.VipTier) (*Strategy, bool) { } func (m *Registry) removeStrategy(s *Strategy) { + delete(m.strategies, s.Name) + delete(m.id2Strategies, s.ID) + m.publishLimiterStrategies() // remove all limiters under this strategy for resource := range s.LimitOptions { m.Remove(resource, s.Name) logrus.WithField("resource", resource).Info("RateLimit rule deleted") } - delete(m.strategies, s.Name) - delete(m.id2Strategies, s.ID) } func (m *Registry) addStrategy(s *Strategy) { m.strategies[s.Name] = s m.id2Strategies[s.ID] = s + m.publishLimiterStrategies() } func (m *Registry) updateStrategy(old, new *Strategy) { m.strategies[new.Name] = new m.id2Strategies[new.ID] = new + if old.Name != new.Name { + delete(m.strategies, old.Name) + } + m.publishLimiterStrategies() if old.Name != new.Name { // strategy name changed? this shall be rare, but we also need to From 4993d11392706abce33b6aa18d489a3602be3fd3 Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 2 Nov 2026 15:13:05 +0800 Subject: [PATCH 03/26] fix(rate): coalesce key loads and redact credential-bearing errors --- util/rate/loader.go | 35 ++++++++++++++++------- util/rate/loader_test.go | 62 ++++++++++++++++++++++++++++++++++++++++ util/rate/resolver.go | 2 +- 3 files changed, 87 insertions(+), 12 deletions(-) create mode 100644 util/rate/loader_test.go diff --git a/util/rate/loader.go b/util/rate/loader.go index 0102169b..2747b364 100644 --- a/util/rate/loader.go +++ b/util/rate/loader.go @@ -1,6 +1,8 @@ package rate import ( + "errors" + "golang.org/x/sync/singleflight" "sync" "time" @@ -13,6 +15,8 @@ const ( LimitKeyExpirationTTL = 60 * time.Second ) +var errKeyLoad = errors.New("key lookup failed") + type KeyInfo struct { SID uint32 // bound strategy ID AclID uint32 // bound allowlist ID @@ -30,7 +34,8 @@ type KeysetFilter struct { type ksLoadFunc func(filter *KeysetFilter) ([]*KeyInfo, error) type KeyLoader struct { - mu sync.Mutex + mu sync.Mutex + loads singleflight.Group // raw keyset load function ksload ksLoadFunc // limit key cache: limit key => *KeyInfo (nil if missing) @@ -54,13 +59,23 @@ func NewKeyLoader(ksload ksLoadFunc) *KeyLoader { // Load loads key info from cache or raw loading from somewhere else // if cache missed. func (l *KeyLoader) Load(key string) (*KeyInfo, bool) { - // load from cache at first - if ki, ok := l.cacheLoad(key); ok { // cache hit - return ki, ok + if cv, expired, found := l.keyCache.GetWithoutExp(key); found && !expired { + return cv.(*KeyInfo), true } - - // otherwise, populate the cache - return l.populateCache(key) + result, err, _ := l.loads.Do(key, func() (interface{}, error) { + if ki, ok := l.cacheLoad(key); ok { + return ki, nil + } + ki, ok := l.populateCache(key) + if !ok { + return (*KeyInfo)(nil), errKeyLoad + } + return ki, nil + }) + if err != nil { + return nil, false + } + return result.(*KeyInfo), true } func (l *KeyLoader) cacheLoad(key string) (*KeyInfo, bool) { @@ -98,9 +113,7 @@ func (l *KeyLoader) populateCache(key string) (*KeyInfo, bool) { l.keyCache.Add(key, (*KeyInfo)(nil)) } - logrus.WithField("key", key). - WithError(err). - Error("Key loader failed to load limit key info") + logrus.Error("Key loader failed to load limit key info") return nil, false } @@ -124,7 +137,7 @@ func (kl *KeyLoader) rawLoad(key string) (*KeyInfo, error) { func (kl *KeyLoader) warmUpKeyCache() { kis, err := kl.ksload(&KeysetFilter{Limit: (LimitKeyCacheSize * 3 / 4)}) if err != nil { - logrus.WithError(err).Warn("Failed to load limit keyset to warm up cache") + logrus.Warn("Failed to load limit keyset to warm up cache") return } diff --git a/util/rate/loader_test.go b/util/rate/loader_test.go new file mode 100644 index 00000000..24f6335c --- /dev/null +++ b/util/rate/loader_test.go @@ -0,0 +1,62 @@ +package rate + +import ( + "bytes" + "errors" + "strings" + "sync" + "sync/atomic" + "testing" + + "github.com/Conflux-Chain/confura/util" + "github.com/sirupsen/logrus" +) + +func TestKeyLoaderCoalescesConcurrentMisses(t *testing.T) { + var calls atomic.Int32 + entered, release := make(chan struct{}), make(chan struct{}) + l := &KeyLoader{keyCache: util.NewExpirableLruCache(LimitKeyCacheSize, LimitKeyExpirationTTL)} + l.ksload = func(*KeysetFilter) ([]*KeyInfo, error) { + if calls.Add(1) == 1 { + close(entered) + } + <-release + return []*KeyInfo{{Key: "fixture", SID: 2}}, nil + } + const n = 32 + var wg sync.WaitGroup + for i := 0; i < n; i++ { + wg.Add(1) + go func() { + defer wg.Done() + ki, ok := l.Load("fixture") + if !ok || ki.SID != 2 { + t.Error("incorrect lookup") + } + }() + } + <-entered + close(release) + wg.Wait() + if calls.Load() != 1 { + t.Fatalf("got %d DB reads", calls.Load()) + } +} + +func TestKeyLoaderDoesNotLogCredentials(t *testing.T) { + const key = "secret-fixture-not-a-real-key" + var out bytes.Buffer + old := logrus.StandardLogger().Out + logrus.SetOutput(&out) + defer logrus.SetOutput(old) + l := &KeyLoader{keyCache: util.NewExpirableLruCache(1, LimitKeyExpirationTTL), ksload: func(*KeysetFilter) ([]*KeyInfo, error) { return nil, errors.New("query key=" + key) }} + if _, ok := l.Load(key); ok { + t.Fatal("error treated as successful load") + } + if strings.Contains(out.String(), key) { + t.Fatalf("credential logged: %s", out.String()) + } + if ki, ok := l.Load(key); !ok || ki != nil { + t.Fatal("missing negative cache") + } +} diff --git a/util/rate/resolver.go b/util/rate/resolver.go index 7d0780ee..512e80a4 100644 --- a/util/rate/resolver.go +++ b/util/rate/resolver.go @@ -112,7 +112,7 @@ func (r *ProvisionedResolver) Resolve(ctx context.Context, reg *Registry) (*Stra stg, _ := reg.id2Strategies[ki.SID] if stg == nil { - logrus.WithField("keyInfo", ki).Debug("Custom rate limit strategy not configured") + logrus.WithField("strategyID", ki.SID).Debug("Custom rate limit strategy not configured") } var limitKey string From 6043fb8b2ff54a0a687a4206d04cb00e75aade30 Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 2 Nov 2026 15:16:59 +0800 Subject: [PATCH 04/26] fix(acl): enforce contract restrictions on filters and subscriptions --- util/acl/validator.go | 49 ++++++++++++++++++++++--- util/acl/validator_boundary_test.go | 56 +++++++++++++++++++++++++++++ util/regex.go | 4 --- util/rpc/middlewares/allowlists.go | 1 + 4 files changed, 101 insertions(+), 9 deletions(-) create mode 100644 util/acl/validator_boundary_test.go diff --git a/util/acl/validator.go b/util/acl/validator.go index ada92d7c..6a893a32 100644 --- a/util/acl/validator.go +++ b/util/acl/validator.go @@ -2,6 +2,7 @@ package acl import ( "context" + "encoding/json" "errors" "regexp" "strings" @@ -28,6 +29,7 @@ type Context struct { context.Context RpcMethod string + RawParams json.RawMessage ExtractRpcParams func() ([]interface{}, error) } @@ -194,8 +196,40 @@ func (v *validatorBase) validateContractAddresses(ctx Context) error { return nil } - if ctx.ExtractRpcParams == nil { - return nil + // A filter ID carries no verifiable contract constraint. Until filter + // ownership is tracked, do not accept IDs created by another identity. + switch ctx.RpcMethod { + case "eth_getFilterLogs", "eth_getFilterChanges", "cfx_getFilterLogs", "cfx_getFilterChanges": + return errInvalidContractAddr + } + if ctx.RpcMethod == "eth_subscribe" || ctx.RpcMethod == "cfx_subscribe" { + var args []json.RawMessage + var kind string + if json.Unmarshal(ctx.RawParams, &args) != nil || len(args) == 0 || json.Unmarshal(args[0], &kind) != nil { + return errBadRpcParams + } + if kind != "logs" { + return nil + } + if len(args) != 2 { + return errBadRpcParams + } + raw := args[1] + if ctx.RpcMethod == "eth_subscribe" { + ctx.RpcMethod = "eth_getLogs" + ctx.ExtractRpcParams = func() ([]interface{}, error) { + var f web3Types.FilterQuery + err := json.Unmarshal(raw, &f) + return []interface{}{f}, err + } + } else { + ctx.RpcMethod = "cfx_getLogs" + ctx.ExtractRpcParams = func() ([]interface{}, error) { + var f cfxTypes.LogFilter + err := json.Unmarshal(raw, &f) + return []interface{}{f}, err + } + } } parseContractAddresses, ok := v.contractAddressParsers[ctx.RpcMethod] @@ -203,6 +237,9 @@ func (v *validatorBase) validateContractAddresses(ctx Context) error { return nil } + if ctx.ExtractRpcParams == nil { + return errBadRpcParams + } inputParams, err := ctx.ExtractRpcParams() if err != nil { return errBadRpcParams @@ -242,6 +279,7 @@ func NewEthValidator(al *AllowList) Validator { "eth_call": v.parseCallRequest, "eth_estimateGas": v.parseCallRequest, "eth_getLogs": v.parseFilterQuery, + "eth_newFilter": v.parseFilterQuery, "eth_scanLogs": parseContractAddressProvider, "eth_scanLogsWithPivotAssumption": parseContractAddressProvider, "eth_getBalance": v.parseAddr, @@ -266,7 +304,7 @@ func (v *EthValidator) parseCallRequest(params []interface{}) (res []string, ok } cr, ok := params[0].(web3Types.CallRequest) - if ok { + if ok && cr.To != nil { res = append(res, cr.To.String()) } @@ -329,6 +367,7 @@ func NewCfxValidator(al *AllowList) Validator { "cfx_call": v.parseCallRequest, "cfx_estimateGasAndCollateral": v.parseCallRequest, "cfx_getLogs": v.parseLogFilter, + "cfx_newFilter": v.parseLogFilter, "cfx_scanLogs": parseContractAddressProvider, "cfx_scanLogsWithPivotAssumption": parseContractAddressProvider, "cfx_getBalance": v.parseAddr, @@ -361,7 +400,7 @@ func (v *CfxValidator) parseCallRequest(params []interface{}) (res []string, ok } cr, ok := params[0].(cfxTypes.CallRequest) - if ok { // only non-verbose base32 address format is accepted + if ok && cr.To != nil { // only non-verbose base32 address format is accepted res = append(res, cr.To.MustGetBase32Address()) } @@ -390,7 +429,7 @@ func (v *CfxValidator) parseAddr(params []interface{}) (res []string, ok bool) { ca, ok := params[0].(cfxTypes.Address) if ok { - res = append(res, ca.MustGetVerboseBase32Address()) + res = append(res, ca.MustGetBase32Address()) } return diff --git a/util/acl/validator_boundary_test.go b/util/acl/validator_boundary_test.go new file mode 100644 index 00000000..74711005 --- /dev/null +++ b/util/acl/validator_boundary_test.go @@ -0,0 +1,56 @@ +package acl + +import ( + "context" + "encoding/json" + "testing" + + "github.com/Conflux-Chain/confura/util/rpc/handlers" + "github.com/Conflux-Chain/go-conflux-sdk/types/cfxaddress" + "github.com/ethereum/go-ethereum/common" + web3Types "github.com/openweb3/web3go/types" + "github.com/stretchr/testify/require" +) + +func TestContractACLFilterAndSubscriptionBoundaries(t *testing.T) { + v := NewEthValidator(&AllowList{ContractAddresses: []string{allowedContractAddr}}) + for _, method := range []string{"eth_getLogs", "eth_newFilter"} { + ctx := ethGetLogsContext(web3Types.FilterQuery{Addresses: []common.Address{common.HexToAddress(allowedContractAddr)}}) + ctx.RpcMethod = method + require.NoError(t, v.Validate(ctx)) + ctx.ExtractRpcParams = nil + require.Error(t, v.Validate(ctx)) + } + for _, tc := range []struct { + params string + allowed bool + }{ + {`["logs",{"address":"` + allowedContractAddr + `"}]`, true}, + {`["logs",{"address":"` + notAllowedContractAddr + `"}]`, false}, + {`["logs",{}]`, false}, {`["logs"]`, false}, {`["logs",null]`, false}, + {`["newHeads"]`, true}, {`[null]`, true}, + } { + ctx := Context{Context: context.Background(), RpcMethod: "eth_subscribe", RawParams: json.RawMessage(tc.params)} + err := v.Validate(ctx) + require.Equal(t, tc.allowed, err == nil, tc.params) + } + for _, method := range []string{"eth_getFilterLogs", "eth_getFilterChanges"} { + require.Error(t, v.Validate(Context{RpcMethod: method})) + require.NoError(t, NewEthValidator(&AllowList{}).Validate(Context{RpcMethod: method})) + } +} + +func TestLiteralOriginAndCoreAddress(t *testing.T) { + v := NewEthValidator(&AllowList{Origins: []string{"https://app.example.com"}}) + for _, origin := range []string{"https://app-example.com", "https://appXexample.com"} { + require.Error(t, v.Validate(Context{Context: context.WithValue(context.Background(), handlers.CtxKeyReqOrigin, origin)})) + } + require.NoError(t, v.Validate(Context{Context: context.WithValue(context.Background(), handlers.CtxKeyReqOrigin, "https://app.example.com")})) + addr := cfxaddress.MustNew(allowedContractAddr, 1) + for _, form := range []string{addr.MustGetBase32Address(), addr.MustGetVerboseBase32Address()} { + v := NewCfxValidator(&AllowList{ContractAddresses: []string{form}}) + for _, method := range []string{"cfx_getBalance", "cfx_getCode", "cfx_getStorageAt", "cfx_getNextNonce"} { + require.NoError(t, v.Validate(Context{RpcMethod: method, ExtractRpcParams: func() ([]interface{}, error) { return []interface{}{addr}, nil }})) + } + } +} diff --git a/util/regex.go b/util/regex.go index 1e6b4da9..c6b9f515 100644 --- a/util/regex.go +++ b/util/regex.go @@ -8,10 +8,6 @@ import ( // WildCardToRegexp converts a wildcard pattern to a regular expression pattern. func WildCardToRegexp(pattern string) string { components := strings.Split(pattern, "*") - if len(components) == 1 { - // if len is 1, there are no *'s, return exact match pattern - return "^" + pattern + "$" - } var result strings.Builder for i, literal := range components { diff --git a/util/rpc/middlewares/allowlists.go b/util/rpc/middlewares/allowlists.go index 438b41bf..f3822c75 100644 --- a/util/rpc/middlewares/allowlists.go +++ b/util/rpc/middlewares/allowlists.go @@ -20,6 +20,7 @@ func Allowlists(next rpc.HandleCallMsgFunc) rpc.HandleCallMsgFunc { aclCtx := acl.Context{ Context: ctx, RpcMethod: msg.Method, + RawParams: msg.Params, } if connH, ok := ctx.Value("handler").(*rpc.ConnHandler); ok { From 6412997aca38553d59b8f7bd839a7de42567509e Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 2 Nov 2026 15:16:59 +0800 Subject: [PATCH 05/26] fix(rpc): preserve and canonicalize IPv6 client identities --- util/rpc/handlers/ip.go | 15 +++++++++++---- util/rpc/handlers/ip_test.go | 23 +++++++++++++++++++++++ 2 files changed, 34 insertions(+), 4 deletions(-) create mode 100644 util/rpc/handlers/ip_test.go diff --git a/util/rpc/handlers/ip.go b/util/rpc/handlers/ip.go index 29b72060..082bef79 100644 --- a/util/rpc/handlers/ip.go +++ b/util/rpc/handlers/ip.go @@ -22,7 +22,7 @@ type ipRange struct { // inRange - check to see if a given ip address is within a range given func inRange(r ipRange, ipAddress net.IP) bool { // strcmp type byte comparison - if bytes.Compare(ipAddress, r.start) >= 0 && bytes.Compare(ipAddress, r.end) < 0 { + if bytes.Compare(ipAddress, r.start) >= 0 && bytes.Compare(ipAddress, r.end) <= 0 { return true } return false @@ -84,12 +84,19 @@ func GetIPAddress(r *http.Request) string { // bad address, go to next continue } - return ip + return realIP.String() } } - if idx := strings.Index(r.RemoteAddr, ":"); idx != -1 { - return r.RemoteAddr[:idx] + host := r.RemoteAddr + if h, _, err := net.SplitHostPort(host); err == nil { + host = h + } + if ip := net.ParseIP(host); ip != nil { + return ip.String() + } + if host != r.RemoteAddr { + return host } return r.RemoteAddr diff --git a/util/rpc/handlers/ip_test.go b/util/rpc/handlers/ip_test.go new file mode 100644 index 00000000..229eba4a --- /dev/null +++ b/util/rpc/handlers/ip_test.go @@ -0,0 +1,23 @@ +package handlers + +import ( + "net/http" + "testing" +) + +func TestRemoteIPCanonicalization(t *testing.T) { + for _, tc := range []struct{ remote, forwarded, want string }{ + {"[2001:db8::1]:1234", "", "2001:db8::1"}, + {"[2001:db8::2]:1234", "", "2001:db8::2"}, + {"2001:db8::1", "", "2001:db8::1"}, + {"192.0.2.1:456", "", "192.0.2.1"}, + {"192.0.2.1:456", "2001:4860:0000:0000:0000:0000:0000:0001", "2001:4860::1"}, + {"192.0.2.1:456", "10.255.255.255", "192.0.2.1"}, + } { + r := &http.Request{RemoteAddr: tc.remote, Header: make(http.Header)} + r.Header.Set("X-Forwarded-For", tc.forwarded) + if got := GetIPAddress(r); got != tc.want { + t.Errorf("%+v: %q", tc, got) + } + } +} From 1fc4d178039d34bc3f4996eb5b67e93b32ac4b5b Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 2 Nov 2026 15:16:59 +0800 Subject: [PATCH 06/26] fix(metrics): bound method labels before routing failures --- util/rpc/middlewares/metrics.go | 25 ++++++++++++++-- util/rpc/middlewares/metrics_test.go | 45 ++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 3 deletions(-) create mode 100644 util/rpc/middlewares/metrics_test.go diff --git a/util/rpc/middlewares/metrics.go b/util/rpc/middlewares/metrics.go index d8ec3afd..915414b3 100644 --- a/util/rpc/middlewares/metrics.go +++ b/util/rpc/middlewares/metrics.go @@ -28,9 +28,9 @@ func Metrics(next rpc.HandleCallMsgFunc) rpc.HandleCallMsgFunc { start := time.Now() resp := next(ctx, msg) - metricMethod := msg.Method - if resp.Error != nil && isMethodNotFoundByError(msg.Method, resp.Error) { - metricMethod = "method_not_found" + metricMethod := registeredMetricMethod(ctx, msg) + if resp == nil { + return nil } // collect rpc QPS/latency, response size etc. @@ -77,3 +77,22 @@ func unwrapJsonError(jsErr *rpc.JsonError) error { return jsErr } + +// Classify against registered callbacks, independently of early routing/auth +// errors. Never use arbitrary client-supplied method names as metric labels. +func registeredMetricMethod(ctx context.Context, msg *rpc.JsonRpcMessage) string { + switch msg.Method { + case "eth_subscribe", "eth_unsubscribe", "cfx_subscribe", "cfx_unsubscribe": + return msg.Method + } + if h, ok := ctx.Value("handler").(*rpc.ConnHandler); ok { + _, err := h.ParseCallArguments(msg) + if err == nil { + return msg.Method + } + if e, ok := err.(rpc.Error); ok && e.ErrorCode() != -32601 { + return msg.Method + } + } + return "method_not_found" +} diff --git a/util/rpc/middlewares/metrics_test.go b/util/rpc/middlewares/metrics_test.go new file mode 100644 index 00000000..e53dc46d --- /dev/null +++ b/util/rpc/middlewares/metrics_test.go @@ -0,0 +1,45 @@ +package middlewares + +import ( + "context" + "fmt" + "net/http/httptest" + "strings" + "testing" + + rpc "github.com/openweb3/go-rpc-provider" +) + +type MetricsFixture struct{} + +func (*MetricsFixture) Ping() bool { return true } + +func TestMetricMethodUsesRegisteredCallbacks(t *testing.T) { + // The provider exposes global hooks; scope the test hook to a context key. + type fixtureKey struct{} + rpc.HookHandleCallMsg(func(next rpc.HandleCallMsgFunc) rpc.HandleCallMsgFunc { + return func(ctx context.Context, msg *rpc.JsonRpcMessage) *rpc.JsonRpcMessage { + if want, ok := ctx.Value(fixtureKey{}).(string); ok { + if got := registeredMetricMethod(ctx, msg); got != want { + t.Errorf("%s: got %s, want %s", msg.Method, got, want) + } + } + return next(ctx, msg) + } + }) + server := rpc.NewServer() + defer server.Stop() + if err := server.RegisterName("fixture", &MetricsFixture{}); err != nil { + t.Fatal(err) + } + for i := 0; i < 200; i++ { + method, want := fmt.Sprintf("untrusted_%d", i), "method_not_found" + if i == 0 { + method, want = "fixture_ping", "fixture_ping" + } + req := httptest.NewRequest("POST", "/", strings.NewReader(fmt.Sprintf(`{"jsonrpc":"2.0","id":1,"method":%q,"params":[]}`, method))) + req.Header.Set("Content-Type", "application/json") + req = req.WithContext(context.WithValue(req.Context(), fixtureKey{}, want)) + server.ServeHTTP(httptest.NewRecorder(), req) + } +} From 44349c3031b1da4526ee34065b0907caab567e93 Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 2 Nov 2026 15:17:44 +0800 Subject: [PATCH 07/26] fix(relay): make concurrent queue admission nonblocking --- util/relay/relay.go | 10 ++++++---- util/relay/relay_test.go | 43 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 4 deletions(-) create mode 100644 util/relay/relay_test.go diff --git a/util/relay/relay.go b/util/relay/relay.go index 651c9f2a..f5d69ca0 100644 --- a/util/relay/relay.go +++ b/util/relay/relay.go @@ -61,12 +61,14 @@ func newTxnRelayer(relayConf *TxnRelayerConfig) *txnRelayer { // Relay relays raw transaction broadcasting asynchronously. func (relayer *txnRelayer) Relay(signedTx hexutil.Bytes) bool { - if len(relayer.txnQueue) == relayer.BufferSize { // queue is full? + // Own queued bytes; callers may reuse their request buffer after return. + raw := append(hexutil.Bytes(nil), signedTx...) + select { + case relayer.txnQueue <- raw: + return true + default: return false } - - relayer.txnQueue <- signedTx - return true } // run starts concurrency worker(s) asynchronously diff --git a/util/relay/relay_test.go b/util/relay/relay_test.go new file mode 100644 index 00000000..baf3c682 --- /dev/null +++ b/util/relay/relay_test.go @@ -0,0 +1,43 @@ +package relay + +import ( + "sync" + "sync/atomic" + "testing" + "time" +) + +func TestRelayConcurrentProducersNeverWaitForConsumer(t *testing.T) { + r := newTxnRelayer(&TxnRelayerConfig{BufferSize: 2}) + raw := []byte{1, 2} + if !r.Relay(raw) { + t.Fatal("initial enqueue failed") + } + raw[0] = 9 + var accepted atomic.Int32 + var wg sync.WaitGroup + start, done := make(chan struct{}), make(chan struct{}) + for i := 0; i < 64; i++ { + wg.Add(1) + go func() { + defer wg.Done() + <-start + if r.Relay([]byte{3}) { + accepted.Add(1) + } + }() + } + close(start) + go func() { wg.Wait(); close(done) }() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("producer blocked without consumer") + } + if accepted.Load() != 1 { + t.Fatalf("accepted %d beyond remaining capacity", accepted.Load()) + } + if (<-r.txnQueue)[0] != 1 { + t.Fatal("caller mutated queued bytes") + } +} From d450d5ac0e5a1acbca5d17a04e42d712cae7c7ac Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 2 Nov 2026 15:17:44 +0800 Subject: [PATCH 08/26] fix(util): enforce topic arity and handle nil values safely --- util/blockchain.go | 5 ++++- util/boundary_test.go | 32 ++++++++++++++++++++++++++++++++ util/types.go | 2 +- 3 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 util/boundary_test.go diff --git a/util/blockchain.go b/util/blockchain.go index e480b298..661d5da5 100644 --- a/util/blockchain.go +++ b/util/blockchain.go @@ -171,7 +171,7 @@ func IsEip155Tx(tx *web3goTypes.TransactionDetail) bool { // IsLegacyEthTx check if the EVM transaction is legacy (pre EIP155) func IsLegacyEthTx(tx *web3goTypes.TransactionDetail) bool { - if tx.V != nil && tx.V.Uint64() == 27 || tx.V.Uint64() == 28 { + if tx != nil && tx.V != nil && (tx.V.Uint64() == 27 || tx.V.Uint64() == 28) { return true } @@ -194,6 +194,9 @@ func GetEthHardforkBlockNumber(chainId uint64) web3goTypes.BlockNumber { // MatchEthLogTopics checks if the eSpace event log matches with the given topics condition func MatchEthLogTopics(log *web3goTypes.Log, topics [][]common.Hash) bool { + if len(topics) > len(log.Topics) { + return false + } find := func(t common.Hash, topics []common.Hash) bool { return slices.Contains(topics, t) } diff --git a/util/boundary_test.go b/util/boundary_test.go new file mode 100644 index 00000000..baad7cf4 --- /dev/null +++ b/util/boundary_test.go @@ -0,0 +1,32 @@ +package util + +import ( + "github.com/ethereum/go-ethereum/common" + web3Types "github.com/openweb3/web3go/types" + "testing" +) + +func TestTopicArity(t *testing.T) { + h := common.HexToHash("0x1") + log := &web3Types.Log{Topics: []common.Hash{h}} + for _, topics := range [][][]common.Hash{{{h}, {}}, {{h}, {h}}, {{}, {}}} { + if MatchEthLogTopics(log, topics) { + t.Fatal("accepted missing topic") + } + } + if !MatchEthLogTopics(log, [][]common.Hash{{}}) { + t.Fatal("rejected existing wildcard position") + } +} + +func TestNilBoundaries(t *testing.T) { + if IsInterfaceValNil([1]int{}) { + t.Fatal("array is not nil") + } + if !IsInterfaceValNil((*int)(nil)) { + t.Fatal("typed nil not recognized") + } + if IsLegacyEthTx(nil) || IsLegacyEthTx(&web3Types.TransactionDetail{}) { + t.Fatal("missing signature is not legacy") + } +} diff --git a/util/types.go b/util/types.go index a38e3095..20e3da5b 100644 --- a/util/types.go +++ b/util/types.go @@ -10,7 +10,7 @@ func IsInterfaceValNil(i interface{}) bool { } switch reflect.TypeOf(i).Kind() { - case reflect.Ptr, reflect.Map, reflect.Array, reflect.Chan, reflect.Slice, reflect.Func: + case reflect.Ptr, reflect.Map, reflect.Chan, reflect.Slice, reflect.Func: return reflect.ValueOf(i).IsNil() } From a61899f721fe1ca89cb4982e3c0dedd6acca813f Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 2 Nov 2026 15:17:44 +0800 Subject: [PATCH 09/26] fix(metrics): register remote metrics by their formatted names --- util/metrics/service/registry.go | 16 ++++++++-------- util/metrics/service/registry_test.go | 17 +++++++++++++++++ 2 files changed, 25 insertions(+), 8 deletions(-) create mode 100644 util/metrics/service/registry_test.go diff --git a/util/metrics/service/registry.go b/util/metrics/service/registry.go index 7e3dab50..bc296e0c 100644 --- a/util/metrics/service/registry.go +++ b/util/metrics/service/registry.go @@ -22,35 +22,35 @@ func NewRegistry(updater Updater) metrics.Registry { func (r *Registry) GetOrRegisterCounter(name string, args ...interface{}) metrics.Counter { metricName := fmt.Sprintf(name, args...) - return r.Registry.GetOrRegister(name, func() metrics.Counter { + return r.Registry.GetOrRegister(metricName, func() metrics.Counter { return &Counter{clientMetric{metricName, r.updater}, metrics.NewCounter()} }).(metrics.Counter) } func (r *Registry) GetOrRegisterGauge(name string, args ...interface{}) metrics.Gauge { metricName := fmt.Sprintf(name, args...) - return r.Registry.GetOrRegister(name, func() metrics.Gauge { + return r.Registry.GetOrRegister(metricName, func() metrics.Gauge { return &Gauge{clientMetric{metricName, r.updater}, metrics.NewGauge()} }).(metrics.Gauge) } func (r *Registry) GetOrRegisterGaugeFloat64(name string, args ...interface{}) metrics.GaugeFloat64 { metricName := fmt.Sprintf(name, args...) - return r.Registry.GetOrRegister(name, func() metrics.GaugeFloat64 { + return r.Registry.GetOrRegister(metricName, func() metrics.GaugeFloat64 { return &GaugeFloat64{clientMetric{metricName, r.updater}, metrics.NewGaugeFloat64()} }).(metrics.GaugeFloat64) } func (r *Registry) GetOrRegisterMeter(name string, args ...interface{}) metrics.Meter { metricName := fmt.Sprintf(name, args...) - return r.Registry.GetOrRegister(name, func() metrics.Meter { + return r.Registry.GetOrRegister(metricName, func() metrics.Meter { return &Meter{clientMetric{metricName, r.updater}, metrics.NewMeter()} }).(metrics.Meter) } func (r *Registry) GetOrRegisterHistogram(name string, args ...interface{}) metrics.Histogram { metricName := fmt.Sprintf(name, args...) - return r.Registry.GetOrRegister(name, func() metrics.Histogram { + return r.Registry.GetOrRegister(metricName, func() metrics.Histogram { return &Histogram{ clientMetric{metricName, r.updater}, metrics.NewHistogram(metrics.NewExpDecaySample(1028, 0.015)), @@ -60,14 +60,14 @@ func (r *Registry) GetOrRegisterHistogram(name string, args ...interface{}) metr func (r *Registry) GetOrRegisterTimer(name string, args ...interface{}) metrics.Timer { metricName := fmt.Sprintf(name, args...) - return r.Registry.GetOrRegister(name, func() metrics.Timer { + return r.Registry.GetOrRegister(metricName, func() metrics.Timer { return &Timer{clientMetric{metricName, r.updater}, metrics.NewTimer()} }).(metrics.Timer) } func (r *Registry) GetOrRegisterPercentage(name string, args ...interface{}) metricUtil.Percentage { metricName := fmt.Sprintf(name, args...) - return r.Registry.GetOrRegister(name, func() metricUtil.Percentage { + return r.Registry.GetOrRegister(metricName, func() metricUtil.Percentage { return &Percentage{clientMetric{metricName, r.updater}, metricUtil.NewPercentage()} }).(metricUtil.Percentage) } @@ -78,7 +78,7 @@ func (r *Registry) GetOrRegisterTimeWindowPercentageDefault(name string, args .. func (r *Registry) GetOrRegisterTimeWindowPercentage(slots int, slotInterval time.Duration, name string, args ...interface{}) metricUtil.Percentage { metricName := fmt.Sprintf(name, args...) - return r.Registry.GetOrRegister(name, func() metricUtil.Percentage { + return r.Registry.GetOrRegister(metricName, func() metricUtil.Percentage { return &TimeWindowPercentage{clientMetric{metricName, r.updater}, metricUtil.NewPercentage(), slots, slotInterval} }).(metricUtil.Percentage) } diff --git a/util/metrics/service/registry_test.go b/util/metrics/service/registry_test.go new file mode 100644 index 00000000..203e2936 --- /dev/null +++ b/util/metrics/service/registry_test.go @@ -0,0 +1,17 @@ +package service + +import "testing" + +func TestFormattedMetricNamesRemainDistinct(t *testing.T) { + r := NewRegistry(nil).(*Registry) + a, b := r.GetOrRegisterGauge("fixture/%s", "a"), r.GetOrRegisterGauge("fixture/%s", "b") + if a == b { + t.Fatal("distinct names alias") + } + if a != r.GetOrRegisterGauge("fixture/%s", "a") { + t.Fatal("same name not reused") + } + if r.Get("fixture/a") != a || r.Get("fixture/b") != b { + t.Fatal("registry contains format instead of name") + } +} From bb8568cd38db6f3b97185a50f979a50f30079c8d Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 2 Nov 2026 15:21:00 +0800 Subject: [PATCH 10/26] fix(rpc): preserve transaction indexes and complete trace paths --- util/rpc/cached_client_eth.go | 34 ++++++++--- util/rpc/cached_client_eth_test.go | 96 ++++++++++++++++++++++++++++++ 2 files changed, 123 insertions(+), 7 deletions(-) create mode 100644 util/rpc/cached_client_eth_test.go diff --git a/util/rpc/cached_client_eth.go b/util/rpc/cached_client_eth.go index 3e0f9ee9..1d5066a7 100644 --- a/util/rpc/cached_client_eth.go +++ b/util/rpc/cached_client_eth.go @@ -2,7 +2,9 @@ package rpc import ( "context" + "math" "math/big" + "slices" "sync" "github.com/Conflux-Chain/confura-data-cache/nearhead" @@ -805,6 +807,9 @@ func (c ethCacheClient) LazyTransactionByBlockHashAndIndex( return res, err }, func() (cacheTypes.Lazy[*web3Types.TransactionDetail], error) { + if uint64(index) > math.MaxUint32 { + return cacheTypes.Lazy[*web3Types.TransactionDetail]{}, nil + } return c.dataCache.GetTransactionByIndex(cacheBlockRef, uint32(index)) }, func() (cacheTypes.Lazy[*web3Types.TransactionDetail], error) { @@ -852,7 +857,7 @@ func (c ethCacheClient) LazyTransactionByBlockNumberAndIndex( return res, err }, func() (res cacheTypes.Lazy[*web3Types.TransactionDetail], err error) { - if blockRef.shouldReadFromDataCache(nearheadCoverage) { + if uint64(index) <= math.MaxUint32 && blockRef.shouldReadFromDataCache(nearheadCoverage) { res, err = c.dataCache.GetTransactionByIndex(blockRef.cacheRef, uint32(index)) } return res, err @@ -886,23 +891,27 @@ func (c traceCacheClient) LazyTrace( transactionHash common.Hash, indexes []uint, ) (res cacheTypes.Lazy[*web3Types.LocalizedTrace], _ error) { - if len(indexes) == 0 { - return res, nil - } - tracker := newRPCCacheTracker(rpcMethodTraceGet) defer tracker.report() return readThroughLazyCacheChain( tracker, func() (res cacheTypes.Lazy[*web3Types.LocalizedTrace], err error) { - if trace := c.nearhead.GetTrace(transactionHash, indexes[0]); trace != nil { + if trace := traceAtAddress(c.nearhead.GetTransactionTraces(transactionHash), indexes); trace != nil { res, err = cacheTypes.NewLazy(trace) } return res, err }, func() (cacheTypes.Lazy[*web3Types.LocalizedTrace], error) { - return c.dataCache.GetTrace(transactionHash, indexes[0]) + traces, err := c.dataCache.GetTransactionTraces(transactionHash) + if err != nil { + return cacheTypes.Lazy[*web3Types.LocalizedTrace]{}, err + } + loaded, err := traces.Load() + if err != nil { + return cacheTypes.Lazy[*web3Types.LocalizedTrace]{}, err + } + return cacheTypes.NewLazy(traceAtAddress(loaded, indexes)) }, func() (cacheTypes.Lazy[*web3Types.LocalizedTrace], error) { hexIndexes := make([]hexutil.Uint, len(indexes)) @@ -1300,3 +1309,14 @@ func readThroughLazyCacheChain[T any]( }, ) } + +// trace_get indexes are a path through the call tree, not an offset in the +// flattened trace_transaction response. An empty path selects the root. +func traceAtAddress(traces []web3Types.LocalizedTrace, path []uint) *web3Types.LocalizedTrace { + for i := range traces { + if slices.Equal(traces[i].TraceAddress, path) { + return &traces[i] + } + } + return nil +} diff --git a/util/rpc/cached_client_eth_test.go b/util/rpc/cached_client_eth_test.go new file mode 100644 index 00000000..b9feb8dc --- /dev/null +++ b/util/rpc/cached_client_eth_test.go @@ -0,0 +1,96 @@ +package rpc + +import ( + "context" + "encoding/json" + "math/big" + "testing" + + "github.com/Conflux-Chain/confura-data-cache/nearhead" + cacheRpc "github.com/Conflux-Chain/confura-data-cache/rpc" + ct "github.com/Conflux-Chain/confura-data-cache/types" + "github.com/Conflux-Chain/go-conflux-util/blockchain/sync/evm" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/openweb3/go-rpc-provider/interfaces" + "github.com/openweb3/web3go/client" + wt "github.com/openweb3/web3go/types" + "github.com/stretchr/testify/require" +) + +type fixtureProvider struct { + interfaces.Provider + call func(context.Context, interface{}, string, ...interface{}) error +} + +func (p *fixtureProvider) CallContext(ctx context.Context, out interface{}, method string, args ...interface{}) error { + return p.call(ctx, out, method, args...) +} + +type fixtureResolver struct{} + +func (fixtureResolver) Resolve(n wt.BlockNumber) (uint64, error) { + if n >= 0 { + return uint64(n), nil + } + return 100, nil +} +func fixtureNearhead() *EthNearHeadCache { + return &EthNearHeadCache{EthCache: nearhead.NewEthCache(nearhead.Config{MaxMemory: 1 << 20, MaxLogs: 100})} +} +func fixtureBlock(n int64, h common.Hash, topics []common.Hash) evm.BlockData { + tx := common.HexToHash("0x1") + return evm.BlockData{Block: &wt.Block{Number: big.NewInt(n), Hash: h, Transactions: *wt.NewTxOrHashListByTxs([]wt.TransactionDetail{{Hash: tx}})}, Receipts: []*wt.Receipt{{TransactionHash: tx, BlockHash: h, Logs: []*wt.Log{{BlockNumber: uint64(n), BlockHash: h, TxHash: tx, Topics: topics}}}}} +} + +type fixtureIndexCache struct { + cacheRpc.Interface + calls int +} + +func (c *fixtureIndexCache) GetTransactionByIndex(_ ct.BlockHashOrNumber, i uint32) (ct.Lazy[*wt.TransactionDetail], error) { + c.calls++ + return ct.NewLazy(&wt.TransactionDetail{Hash: common.HexToHash("0xa")}) +} + +func TestLargeTransactionIndexBypassesNarrowCache(t *testing.T) { + d := &fixtureIndexCache{Interface: cacheRpc.NotFoundImpl} + p := &fixtureProvider{call: func(_ context.Context, out interface{}, method string, args ...interface{}) error { + require.Equal(t, hexutil.Uint(uint64(1)<<32), args[1]) + return json.Unmarshal([]byte("null"), out) + }} + n := fixtureNearhead() + require.NoError(t, n.Put(fixtureBlock(100, common.Hash{}, nil))) + c := ethCacheClient{RpcEthClient: client.NewRpcEthClient(p), cacheClientContext: cacheClientContext{nearhead: n, dataCache: d, resolver: fixtureResolver{}}} + a, err := c.LazyTransactionByBlockHashAndIndex(common.Hash{}, uint(uint64(1)<<32)) + require.NoError(t, err) + require.True(t, a.IsEmptyOrNull()) + b, err := c.LazyTransactionByBlockNumberAndIndex(1, uint(uint64(1)<<32)) + require.NoError(t, err) + require.True(t, b.IsEmptyOrNull()) + require.Zero(t, d.calls) +} + +type fixtureTraceCache struct{ cacheRpc.Interface } + +func (c *fixtureTraceCache) GetTransactionTraces(common.Hash) (ct.Lazy[[]wt.LocalizedTrace], error) { + var traces []wt.LocalizedTrace + for _, path := range [][]uint{{}, {0}, {0, 1}, {0, 2}} { + var trace wt.LocalizedTrace + _ = json.Unmarshal([]byte(`{"type":"call","action":{"from":"0x0000000000000000000000000000000000000000","to":"0x0000000000000000000000000000000000000000","gas":"0x0","input":"0x","value":"0x0","callType":"call"},"traceAddress":[],"subtraces":0,"blockNumber":100,"blockHash":"0x0000000000000000000000000000000000000000000000000000000000000001"}`), &trace) + trace.TraceAddress = path + traces = append(traces, trace) + } + return ct.NewLazy(traces) +} +func TestTraceGetUsesEntirePathIncludingRoot(t *testing.T) { + c := traceCacheClient{cacheClientContext: cacheClientContext{nearhead: fixtureNearhead(), dataCache: &fixtureTraceCache{cacheRpc.NotFoundImpl}}} + for _, path := range [][]uint{{}, {0, 1}, {0, 2}} { + lazy, err := c.LazyTrace(common.Hash{}, path) + require.NoError(t, err) + got, err := lazy.Load() + require.NoError(t, err) + require.NotNil(t, got) + require.Equal(t, path, got.TraceAddress) + } +} From 3cd8ad2c21bcd28ba7cba810688ab25fd37ce874 Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 2 Nov 2026 15:25:46 +0800 Subject: [PATCH 11/26] fix(rpc): snapshot and fence near-head log reads --- util/rpc/cached_client_eth.go | 70 ++++++++++++++--- util/rpc/cached_client_eth_test.go | 4 +- util/rpc/nearhead.go | 121 +++++++++++++++++++++++++++++ util/rpc/nearhead_test.go | 119 ++++++++++++++++++++++++++++ 4 files changed, 300 insertions(+), 14 deletions(-) create mode 100644 util/rpc/nearhead.go create mode 100644 util/rpc/nearhead_test.go diff --git a/util/rpc/cached_client_eth.go b/util/rpc/cached_client_eth.go index 1d5066a7..97563a58 100644 --- a/util/rpc/cached_client_eth.go +++ b/util/rpc/cached_client_eth.go @@ -9,11 +9,13 @@ import ( "github.com/Conflux-Chain/confura-data-cache/nearhead" cacheRpc "github.com/Conflux-Chain/confura-data-cache/rpc" - cacheSync "github.com/Conflux-Chain/confura-data-cache/sync" cacheTypes "github.com/Conflux-Chain/confura-data-cache/types" "github.com/Conflux-Chain/confura/util" "github.com/Conflux-Chain/confura/util/metrics" lruCache "github.com/Conflux-Chain/confura/util/rpc/cache" + "github.com/Conflux-Chain/go-conflux-util/blockchain/sync/evm" + "github.com/Conflux-Chain/go-conflux-util/blockchain/sync/poll" + "github.com/Conflux-Chain/go-conflux-util/blockchain/sync/process" viperutil "github.com/Conflux-Chain/go-conflux-util/viper" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" @@ -55,7 +57,10 @@ const ( type EthNearHeadCache struct { *nearhead.EthCache - stopSync context.CancelFunc + stopSync context.CancelFunc + syncDone sync.WaitGroup + snapshotMu sync.RWMutex + maxLogs int } func newDummyEthNearHeadCache() *EthNearHeadCache { @@ -67,13 +72,17 @@ func newDummyEthNearHeadCache() *EthNearHeadCache { func (c *EthNearHeadCache) Close() { if c.stopSync != nil { c.stopSync() + c.syncDone.Wait() } } func NewEthNearHeadCacheFromViper(url string) (*EthNearHeadCache, error) { var conf struct { - Enabled bool - cacheSync.NearHeadConfig `mapstructure:",squash"` + Enabled bool + Adapter evm.AdapterConfig + Poller poll.Option + Writer nearhead.WriteOption + Cache nearhead.Config } if err := viperutil.UnmarshalKey("nearhead", &conf); err != nil { return nil, errors.WithMessage(err, "failed to unmarshal near head config") @@ -86,17 +95,26 @@ func NewEthNearHeadCacheFromViper(url string) (*EthNearHeadCache, error) { conf.Adapter.URL = url - // Start near head sync. + adapter, err := evm.NewAdapterWithConfig(conf.Adapter) + if err != nil { + return nil, errors.WithMessage(err, "failed to create near head adapter") + } ctx, cancel := context.WithCancel(context.Background()) - var wg sync.WaitGroup - - cache, err := cacheSync.StartNearHead(ctx, &wg, conf.NearHeadConfig) + finalized, err := adapter.GetFinalizedBlockNumber(ctx) if err != nil { cancel() - return nil, errors.WithMessage(err, "Failed to sync near head data") + adapter.Close() + return nil, err } - - return &EthNearHeadCache{EthCache: cache, stopSync: cancel}, nil + poller := poll.NewLatestPoller(adapter, finalized, poll.ReorgWindowParams{}, conf.Poller) + cache := &EthNearHeadCache{EthCache: nearhead.NewEthCache(conf.Cache), stopSync: cancel, maxLogs: conf.Cache.MaxLogs} + writer := &nearHeadWriter{cache: cache, writer: nearhead.NewWriter(cache.EthCache, conf.Writer)} + cache.syncDone.Add(2) + go poller.Poll(ctx, &cache.syncDone) + go process.Process(ctx, &cache.syncDone, poller.DataCh(), writer) + var once sync.Once + cache.stopSync = func() { once.Do(func() { cancel(); cache.syncDone.Wait(); adapter.Close() }) } + return cache, nil } func MustNewEthDataCacheClientFromViper() cacheRpc.Interface { @@ -674,7 +692,7 @@ func (c ethCacheClient) logsByBlockRange(filter web3Types.FilterQuery) (res []we return nil, tracker.withError(err) } - ethLogs, err := c.nearhead.GetLogsByBlockRange( + ethLogs, cachedHash, err := c.nearhead.logsSnapshot( fromBlock, toBlock, nearhead.FilterOpt{ Addresses: filter.Addresses, @@ -701,6 +719,26 @@ func (c ethCacheClient) logsByBlockRange(filter web3Types.FilterQuery) (res []we return nil, tracker.withError(err) } + // A canonical hash at the upper bound commits to the entire requested + // ancestry. Check the cache boundary and fence all full-node range reads. + anchor, err := c.RpcEthClient.BlockByNumber(web3Types.BlockNumber(toBlock), false) + if err != nil { + return nil, tracker.withError(err) + } + if anchor == nil { + return nil, tracker.withError(errors.New("log range anchor unavailable")) + } + cacheAnchor := anchor + if ethLogs.ToBlock != toBlock { + cacheAnchor, err = c.RpcEthClient.BlockByNumber(web3Types.BlockNumber(ethLogs.ToBlock), false) + if err != nil { + return nil, tracker.withError(err) + } + } + if cacheAnchor == nil || cacheAnchor.Hash != cachedHash { + return nil, tracker.withError(errors.New("near head logs differ from canonical chain; retry request")) + } + tracker.withNearHeadHit(true) // Stitches together logs from the near head cache and full node. @@ -729,6 +767,14 @@ func (c ethCacheClient) logsByBlockRange(filter web3Types.FilterQuery) (res []we tracker.withFullNodeHit(true) } + after, err := c.RpcEthClient.BlockByNumber(web3Types.BlockNumber(toBlock), false) + if err != nil { + return nil, tracker.withError(err) + } + if after == nil || after.Hash != anchor.Hash { + return nil, tracker.withError(errors.New("chain changed while reading logs; retry request")) + } + return res, nil } diff --git a/util/rpc/cached_client_eth_test.go b/util/rpc/cached_client_eth_test.go index b9feb8dc..a8358668 100644 --- a/util/rpc/cached_client_eth_test.go +++ b/util/rpc/cached_client_eth_test.go @@ -36,11 +36,11 @@ func (fixtureResolver) Resolve(n wt.BlockNumber) (uint64, error) { return 100, nil } func fixtureNearhead() *EthNearHeadCache { - return &EthNearHeadCache{EthCache: nearhead.NewEthCache(nearhead.Config{MaxMemory: 1 << 20, MaxLogs: 100})} + return &EthNearHeadCache{maxLogs: 100, EthCache: nearhead.NewEthCache(nearhead.Config{MaxMemory: 1 << 20, MaxLogs: 100})} } func fixtureBlock(n int64, h common.Hash, topics []common.Hash) evm.BlockData { tx := common.HexToHash("0x1") - return evm.BlockData{Block: &wt.Block{Number: big.NewInt(n), Hash: h, Transactions: *wt.NewTxOrHashListByTxs([]wt.TransactionDetail{{Hash: tx}})}, Receipts: []*wt.Receipt{{TransactionHash: tx, BlockHash: h, Logs: []*wt.Log{{BlockNumber: uint64(n), BlockHash: h, TxHash: tx, Topics: topics}}}}} + return evm.BlockData{Block: &wt.Block{Number: big.NewInt(n), Difficulty: big.NewInt(0), Hash: h, Transactions: *wt.NewTxOrHashListByTxs([]wt.TransactionDetail{{Hash: tx}})}, Receipts: []*wt.Receipt{{TransactionHash: tx, BlockHash: h, Logs: []*wt.Log{{BlockNumber: uint64(n), BlockHash: h, TxHash: tx, Topics: topics}}}}} } type fixtureIndexCache struct { diff --git a/util/rpc/nearhead.go b/util/rpc/nearhead.go new file mode 100644 index 00000000..eaf695e3 --- /dev/null +++ b/util/rpc/nearhead.go @@ -0,0 +1,121 @@ +package rpc + +import ( + "context" + "fmt" + + "github.com/Conflux-Chain/confura-data-cache/nearhead" + ct "github.com/Conflux-Chain/confura-data-cache/types" + "github.com/Conflux-Chain/confura/util" + "github.com/Conflux-Chain/go-conflux-util/blockchain/sync/evm" + "github.com/Conflux-Chain/go-conflux-util/blockchain/sync/poll" + "github.com/ethereum/go-ethereum/common" + wt "github.com/openweb3/web3go/types" +) + +// Serialize the dependency's Pop/Put pair with multi-block log snapshots. +// Its individual methods are safe, but do not provide a compound read view. +type nearHeadWriter struct { + cache *EthNearHeadCache + writer *nearhead.Writer +} + +func (w *nearHeadWriter) Process(ctx context.Context, data poll.Revertable[evm.BlockData]) { + w.cache.snapshotMu.Lock() + defer w.cache.snapshotMu.Unlock() + w.writer.Process(ctx, data) +} + +func (c *EthNearHeadCache) Put(data evm.BlockData) error { + c.snapshotMu.Lock() + defer c.snapshotMu.Unlock() + return c.EthCache.Put(data) +} +func (c *EthNearHeadCache) Pop(number uint64) int { + c.snapshotMu.Lock() + defer c.snapshotMu.Unlock() + return c.EthCache.Pop(number) +} + +func (c *EthNearHeadCache) GetLogsByBlockHash(hash common.Hash, filter nearhead.FilterOpt) (*nearhead.EthLogs, error) { + c.snapshotMu.RLock() + defer c.snapshotMu.RUnlock() + ref := ct.BlockHashOrNumberWithHash(hash) + block := c.EthCache.GetBlock(ref, false).Result + if block == nil { + return nil, nil + } + receipts := c.EthCache.GetBlockReceipts(ref).Result + if receipts == nil { + return nil, nil + } + logs, err := c.collectLogs(nil, receipts, filter) + if err != nil { + return nil, err + } + return &nearhead.EthLogs{FromBlock: block.Number.Uint64(), ToBlock: block.Number.Uint64(), Logs: logs}, nil +} + +func (c *EthNearHeadCache) logsSnapshot(from, to uint64, filter nearhead.FilterOpt) (*nearhead.EthLogs, common.Hash, error) { + c.snapshotMu.RLock() + defer c.snapshotMu.RUnlock() + if from > to { + return nil, common.Hash{}, fmt.Errorf("invalid block range params") + } + coverage := c.EthCache.GetBlock(ct.BlockHashOrNumberWithNumber(from), false).Coverage + if coverage[0] == coverage[1] || to < coverage[0] || from >= coverage[1] { + return nil, common.Hash{}, nil + } + from, to = max(from, coverage[0]), min(to, coverage[1]-1) + logs := make([]wt.Log, 0) + for number := from; number <= to; number++ { + receipts := c.EthCache.GetBlockReceipts(ct.BlockHashOrNumberWithNumber(number)).Result + if receipts == nil { + return nil, common.Hash{}, nil + } // receipts were not collected + var err error + logs, err = c.collectLogs(logs, receipts, filter) + if err != nil { + return nil, common.Hash{}, err + } + } + last := c.EthCache.GetBlock(ct.BlockHashOrNumberWithNumber(to), false).Result + if last == nil { + return nil, common.Hash{}, nil + } + return &nearhead.EthLogs{FromBlock: from, ToBlock: to, Logs: logs}, last.Hash, nil +} + +func (c *EthNearHeadCache) collectLogs(logs []wt.Log, receipts []*wt.Receipt, filter nearhead.FilterOpt) ([]wt.Log, error) { + if logs == nil { + logs = make([]wt.Log, 0) + } + for _, receipt := range receipts { + if receipt == nil { + return nil, fmt.Errorf("incomplete cached receipt") + } + for _, entry := range receipt.Logs { + if entry == nil { + return nil, fmt.Errorf("invalid cached log") + } + if !util.IncludeEthLogAddrs(entry, filter.Addresses) || !util.MatchEthLogTopics(entry, filter.Topics) { + continue + } + copied := *entry + copied.Topics = append([]common.Hash(nil), entry.Topics...) + copied.Data = append([]byte(nil), entry.Data...) + logs = append(logs, copied) + if len(logs) > c.maxLogs { + return nil, fmt.Errorf("the result set exceeds the max limit of %v logs, please narrow down your filter conditions", c.maxLogs) + } + } + } + return logs, nil +} + +// GetLogsByBlockRange returns a snapshot without using the dependency's topic +// matcher. A caller combining this with other sources must fence the chain view. +func (c *EthNearHeadCache) GetLogsByBlockRange(from, to uint64, filter nearhead.FilterOpt) (*nearhead.EthLogs, error) { + logs, _, err := c.logsSnapshot(from, to, filter) + return logs, err +} diff --git a/util/rpc/nearhead_test.go b/util/rpc/nearhead_test.go new file mode 100644 index 00000000..85c11921 --- /dev/null +++ b/util/rpc/nearhead_test.go @@ -0,0 +1,119 @@ +package rpc + +import ( + "context" + "encoding/json" + "fmt" + "sync" + "testing" + "time" + + "github.com/Conflux-Chain/confura-data-cache/nearhead" + cacheRpc "github.com/Conflux-Chain/confura-data-cache/rpc" + "github.com/ethereum/go-ethereum/common" + "github.com/openweb3/web3go/client" + wt "github.com/openweb3/web3go/types" + "github.com/stretchr/testify/require" +) + +func TestLogsRejectReorgDuringStitch(t *testing.T) { + for _, reorg := range []bool{false, true} { + t.Run(fmt.Sprint(reorg), func(t *testing.T) { + n := fixtureNearhead() + a, b := common.HexToHash("0xa100"), common.HexToHash("0xb100") + require.NoError(t, n.Put(fixtureBlock(100, a, nil))) + canonical := a + p := &fixtureProvider{call: func(_ context.Context, out interface{}, method string, args ...interface{}) error { + var data interface{} + switch method { + case "eth_getBlockByNumber": + block := fixtureBlock(100, canonical, nil).Block + block.Transactions = *wt.NewTxOrHashListByHashes(nil) + data = block + case "eth_getLogs": + if reorg { + canonical = b + require.Equal(t, 1, n.Pop(100)) + require.NoError(t, n.Put(fixtureBlock(100, b, nil))) + } + data = []wt.Log{{BlockNumber: 99, BlockHash: common.HexToHash("0x99")}} + default: + t.Fatalf("unexpected method %s", method) + } + raw, err := json.Marshal(data) + if err != nil { + return err + } + return json.Unmarshal(raw, out) + }} + c := ethCacheClient{RpcEthClient: client.NewRpcEthClient(p), cacheClientContext: cacheClientContext{nearhead: n, dataCache: cacheRpc.NotFoundImpl, resolver: fixtureResolver{}}} + from, to := wt.BlockNumber(99), wt.BlockNumber(100) + logs, err := c.Logs(wt.FilterQuery{FromBlock: &from, ToBlock: &to}) + if reorg { + require.ErrorContains(t, err, "chain changed") + require.Nil(t, logs) + } else { + require.NoError(t, err) + require.Len(t, logs, 2) + require.Equal(t, a, logs[1].BlockHash) + } + }) + } +} + +func TestNearHeadLogTopicsAndOwnership(t *testing.T) { + n := fixtureNearhead() + hash := common.HexToHash("0xa") + require.NoError(t, n.Put(fixtureBlock(100, hash, []common.Hash{hash}))) + for _, topics := range [][][]common.Hash{{{hash}, {}}, {{hash}, {hash}}} { + logs, err := n.GetLogsByBlockHash(hash, nearhead.FilterOpt{Topics: topics}) + require.NoError(t, err) + require.Empty(t, logs.Logs) + } + logs, err := n.GetLogsByBlockHash(hash, nearhead.FilterOpt{}) + require.NoError(t, err) + require.Len(t, logs.Logs, 1) + logs.Logs[0].Topics[0] = common.Hash{} + again, err := n.GetLogsByBlockHash(hash, nearhead.FilterOpt{}) + require.NoError(t, err) + require.Equal(t, hash, again.Logs[0].Topics[0]) + n.maxLogs = 0 + _, err = n.GetLogsByBlockHash(hash, nearhead.FilterOpt{}) + require.Error(t, err) +} + +func TestNearHeadHashReadsAndWriterProgress(t *testing.T) { + n := fixtureNearhead() + hash := common.HexToHash("0xa") + require.NoError(t, n.Put(fixtureBlock(100, hash, nil))) + var wg sync.WaitGroup + for i := 0; i < 4; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < 100; j++ { + _, err := n.GetLogsByBlockHash(hash, nearhead.FilterOpt{}) + if err != nil { + t.Error(err) + } + } + }() + } + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < 100; j++ { + n.Pop(100) + if err := n.Put(fixtureBlock(100, hash, nil)); err != nil { + t.Error(err) + } + } + }() + done := make(chan struct{}) + go func() { wg.Wait(); close(done) }() + select { + case <-done: + case <-time.After(3 * time.Second): + t.Fatal("cache read/write deadlock") + } +} From c6597de1303b96410760b4220e57a4d12679a755 Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 2 Nov 2026 15:27:07 +0800 Subject: [PATCH 12/26] refactor(config): preserve application configuration semantics across upgrades --- cmd/rpc.go | 2 +- config/config.go | 14 ++++- node/config.go | 2 +- rpc/handler/cfx_gasstation.go | 2 +- rpc/handler/cfx_logs.go | 2 +- rpc/handler/cfx_txn.go | 2 +- rpc/handler/eth_gasstation.go | 2 +- rpc/handler/eth_txn.go | 2 +- store/eth_data.go | 2 +- store/log_filter.go | 2 +- store/mysql/config.go | 2 +- store/store.go | 2 +- sync/catchup/syncer.go | 2 +- sync/election/leader_manager.go | 2 +- sync/sync_db.go | 2 +- sync/sync_eth.go | 2 +- sync/sync_trace_log.go | 2 +- util/pprof/pprof.go | 2 +- util/relay/relay.go | 2 +- util/rpc/cache/cache_eth.go | 2 +- util/rpc/cached_client_eth.go | 2 +- util/rpc/config.go | 2 +- util/rpc/middlewares/web3pay.go | 2 +- util/rpc/pubsub_limiter.go | 2 +- util/rpc/server.go | 2 +- util/viper/viper.go | 108 ++++++++++++++++++++++++++++++++ util/viper/viper_test.go | 48 ++++++++++++++ virtualfilter/client/client.go | 2 +- virtualfilter/config.go | 2 +- 29 files changed, 194 insertions(+), 28 deletions(-) create mode 100644 util/viper/viper.go create mode 100644 util/viper/viper_test.go diff --git a/cmd/rpc.go b/cmd/rpc.go index d5b0fcf0..4a3acdef 100644 --- a/cmd/rpc.go +++ b/cmd/rpc.go @@ -19,7 +19,7 @@ import ( "github.com/Conflux-Chain/confura/util/relay" rpcutil "github.com/Conflux-Chain/confura/util/rpc" vfclient "github.com/Conflux-Chain/confura/virtualfilter/client" - viperutil "github.com/Conflux-Chain/go-conflux-util/viper" + viperutil "github.com/Conflux-Chain/confura/util/viper" ) var ( diff --git a/config/config.go b/config/config.go index f74ad9ac..657cf904 100644 --- a/config/config.go +++ b/config/config.go @@ -11,7 +11,10 @@ import ( "github.com/Conflux-Chain/confura/util/pprof" rpcutil "github.com/Conflux-Chain/confura/util/rpc" "github.com/Conflux-Chain/confura/util/rpc/cache" - "github.com/Conflux-Chain/go-conflux-util/config" + "github.com/Conflux-Chain/confura/util/viper" + "github.com/Conflux-Chain/go-conflux-util/alert" + logutil "github.com/Conflux-Chain/go-conflux-util/log" + metricsutil "github.com/Conflux-Chain/go-conflux-util/metrics" "github.com/ethereum/go-ethereum/log" slogrus "github.com/samber/slog-logrus/v2" "github.com/sirupsen/logrus" @@ -23,7 +26,14 @@ const viperEnvPrefix = "infura" func Init() { // init utilities eg., viper, alert, metrics and logging - config.MustInit(viperEnvPrefix) + viper.MustInit(viperEnvPrefix) + var metricConfig metricsutil.Config + viper.MustUnmarshalKey("metrics", &metricConfig) + metricsutil.Init(metricConfig) + alert.MustInitFromViper() + var logConfig logutil.LoggingConfig + viper.MustUnmarshalKey("log", &logConfig) + logutil.MustInit(logConfig) // init pprof pprof.MustInit() diff --git a/node/config.go b/node/config.go index 9df713a6..0ff4b1f2 100644 --- a/node/config.go +++ b/node/config.go @@ -3,7 +3,7 @@ package node import ( "time" - "github.com/Conflux-Chain/go-conflux-util/viper" + "github.com/Conflux-Chain/confura/util/viper" "github.com/buraksezer/consistent" "github.com/sirupsen/logrus" ) diff --git a/rpc/handler/cfx_gasstation.go b/rpc/handler/cfx_gasstation.go index fc027136..0b7504a2 100644 --- a/rpc/handler/cfx_gasstation.go +++ b/rpc/handler/cfx_gasstation.go @@ -10,7 +10,7 @@ import ( "github.com/Conflux-Chain/confura/util" sdk "github.com/Conflux-Chain/go-conflux-sdk" cfxtypes "github.com/Conflux-Chain/go-conflux-sdk/types" - "github.com/Conflux-Chain/go-conflux-util/viper" + "github.com/Conflux-Chain/confura/util/viper" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/openweb3/go-rpc-provider/utils" "github.com/sirupsen/logrus" diff --git a/rpc/handler/cfx_logs.go b/rpc/handler/cfx_logs.go index c72a8310..f8e958e3 100644 --- a/rpc/handler/cfx_logs.go +++ b/rpc/handler/cfx_logs.go @@ -11,7 +11,7 @@ import ( "github.com/Conflux-Chain/confura/util/metrics" sdk "github.com/Conflux-Chain/go-conflux-sdk" "github.com/Conflux-Chain/go-conflux-sdk/types" - viperutil "github.com/Conflux-Chain/go-conflux-util/viper" + viperutil "github.com/Conflux-Chain/confura/util/viper" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/pkg/errors" "github.com/sirupsen/logrus" diff --git a/rpc/handler/cfx_txn.go b/rpc/handler/cfx_txn.go index e010fe0c..83fe5bab 100644 --- a/rpc/handler/cfx_txn.go +++ b/rpc/handler/cfx_txn.go @@ -9,7 +9,7 @@ import ( rpcutil "github.com/Conflux-Chain/confura/util/rpc" sdk "github.com/Conflux-Chain/go-conflux-sdk" "github.com/Conflux-Chain/go-conflux-sdk/types" - "github.com/Conflux-Chain/go-conflux-util/viper" + "github.com/Conflux-Chain/confura/util/viper" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/rpc" "github.com/openweb3/go-rpc-provider/utils" diff --git a/rpc/handler/eth_gasstation.go b/rpc/handler/eth_gasstation.go index 44e19068..911aee4f 100644 --- a/rpc/handler/eth_gasstation.go +++ b/rpc/handler/eth_gasstation.go @@ -7,7 +7,7 @@ import ( "github.com/Conflux-Chain/confura/node" "github.com/Conflux-Chain/confura/types" - "github.com/Conflux-Chain/go-conflux-util/viper" + "github.com/Conflux-Chain/confura/util/viper" "github.com/ethereum/go-ethereum/common" "github.com/openweb3/go-rpc-provider/utils" ethtypes "github.com/openweb3/web3go/types" diff --git a/rpc/handler/eth_txn.go b/rpc/handler/eth_txn.go index 4980edce..c983eac6 100644 --- a/rpc/handler/eth_txn.go +++ b/rpc/handler/eth_txn.go @@ -7,7 +7,7 @@ import ( "github.com/Conflux-Chain/confura/util" "github.com/Conflux-Chain/confura/util/relay" rpcutil "github.com/Conflux-Chain/confura/util/rpc" - "github.com/Conflux-Chain/go-conflux-util/viper" + "github.com/Conflux-Chain/confura/util/viper" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/rpc" diff --git a/store/eth_data.go b/store/eth_data.go index 3157e68c..263e8593 100644 --- a/store/eth_data.go +++ b/store/eth_data.go @@ -8,7 +8,7 @@ import ( "time" "github.com/Conflux-Chain/confura/util/metrics" - "github.com/Conflux-Chain/go-conflux-util/viper" + "github.com/Conflux-Chain/confura/util/viper" "github.com/ethereum/go-ethereum/common" "github.com/openweb3/go-rpc-provider/utils" "github.com/openweb3/web3go" diff --git a/store/log_filter.go b/store/log_filter.go index fe2190d1..9a470351 100644 --- a/store/log_filter.go +++ b/store/log_filter.go @@ -7,7 +7,7 @@ import ( citypes "github.com/Conflux-Chain/confura/types" "github.com/Conflux-Chain/go-conflux-sdk/types" - "github.com/Conflux-Chain/go-conflux-util/viper" + "github.com/Conflux-Chain/confura/util/viper" web3Types "github.com/openweb3/web3go/types" "github.com/pkg/errors" diff --git a/store/mysql/config.go b/store/mysql/config.go index b5d48ef9..197b042c 100644 --- a/store/mysql/config.go +++ b/store/mysql/config.go @@ -8,7 +8,7 @@ import ( "github.com/Conflux-Chain/confura/store" "github.com/Conflux-Chain/go-conflux-util/dlock" - "github.com/Conflux-Chain/go-conflux-util/viper" + "github.com/Conflux-Chain/confura/util/viper" gosql "github.com/go-sql-driver/mysql" "github.com/sirupsen/logrus" "gorm.io/driver/mysql" diff --git a/store/store.go b/store/store.go index 292c695b..7731757a 100644 --- a/store/store.go +++ b/store/store.go @@ -1,7 +1,7 @@ package store import ( - "github.com/Conflux-Chain/go-conflux-util/viper" + "github.com/Conflux-Chain/confura/util/viper" "github.com/sirupsen/logrus" ) diff --git a/sync/catchup/syncer.go b/sync/catchup/syncer.go index c8dc41f4..02c8d850 100644 --- a/sync/catchup/syncer.go +++ b/sync/catchup/syncer.go @@ -11,7 +11,7 @@ import ( "github.com/Conflux-Chain/confura/sync/election" "github.com/Conflux-Chain/confura/sync/monitor" sdk "github.com/Conflux-Chain/go-conflux-sdk" - viperutil "github.com/Conflux-Chain/go-conflux-util/viper" + viperutil "github.com/Conflux-Chain/confura/util/viper" "github.com/openweb3/web3go" "github.com/pkg/errors" "github.com/sirupsen/logrus" diff --git a/sync/election/leader_manager.go b/sync/election/leader_manager.go index 5025d21f..07bc703f 100644 --- a/sync/election/leader_manager.go +++ b/sync/election/leader_manager.go @@ -9,7 +9,7 @@ import ( "github.com/Conflux-Chain/confura/store" "github.com/Conflux-Chain/go-conflux-util/dlock" "github.com/Conflux-Chain/go-conflux-util/health" - "github.com/Conflux-Chain/go-conflux-util/viper" + "github.com/Conflux-Chain/confura/util/viper" "github.com/google/uuid" "github.com/pkg/errors" "github.com/sirupsen/logrus" diff --git a/sync/sync_db.go b/sync/sync_db.go index 911ab7b8..62fd2070 100644 --- a/sync/sync_db.go +++ b/sync/sync_db.go @@ -17,7 +17,7 @@ import ( "github.com/Conflux-Chain/go-conflux-sdk/types" "github.com/Conflux-Chain/go-conflux-util/dlock" "github.com/Conflux-Chain/go-conflux-util/health" - viperutil "github.com/Conflux-Chain/go-conflux-util/viper" + viperutil "github.com/Conflux-Chain/confura/util/viper" "github.com/pkg/errors" "github.com/sirupsen/logrus" "gorm.io/gorm" diff --git a/sync/sync_eth.go b/sync/sync_eth.go index dc44ef7f..aeae8f0f 100644 --- a/sync/sync_eth.go +++ b/sync/sync_eth.go @@ -14,7 +14,7 @@ import ( "github.com/Conflux-Chain/confura/util/metrics" "github.com/Conflux-Chain/go-conflux-util/dlock" "github.com/Conflux-Chain/go-conflux-util/health" - viperutil "github.com/Conflux-Chain/go-conflux-util/viper" + viperutil "github.com/Conflux-Chain/confura/util/viper" "github.com/openweb3/web3go" ethtypes "github.com/openweb3/web3go/types" "github.com/pkg/errors" diff --git a/sync/sync_trace_log.go b/sync/sync_trace_log.go index b12f1768..cd4851aa 100644 --- a/sync/sync_trace_log.go +++ b/sync/sync_trace_log.go @@ -14,7 +14,7 @@ import ( "github.com/Conflux-Chain/go-conflux-util/blockchain/sync/poll" "github.com/Conflux-Chain/go-conflux-util/blockchain/sync/process/db" "github.com/Conflux-Chain/go-conflux-util/ctxutil" - viperutil "github.com/Conflux-Chain/go-conflux-util/viper" + viperutil "github.com/Conflux-Chain/confura/util/viper" "github.com/sirupsen/logrus" ) diff --git a/util/pprof/pprof.go b/util/pprof/pprof.go index a7f54ca4..1fdc5ccf 100644 --- a/util/pprof/pprof.go +++ b/util/pprof/pprof.go @@ -5,7 +5,7 @@ import ( "net/http" _ "net/http/pprof" - "github.com/Conflux-Chain/go-conflux-util/viper" + "github.com/Conflux-Chain/confura/util/viper" "github.com/sirupsen/logrus" ) diff --git a/util/relay/relay.go b/util/relay/relay.go index f5d69ca0..202a93c9 100644 --- a/util/relay/relay.go +++ b/util/relay/relay.go @@ -6,7 +6,7 @@ import ( "github.com/Conflux-Chain/confura/util" rpcutil "github.com/Conflux-Chain/confura/util/rpc" sdk "github.com/Conflux-Chain/go-conflux-sdk" - "github.com/Conflux-Chain/go-conflux-util/viper" + "github.com/Conflux-Chain/confura/util/viper" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/openweb3/web3go" "github.com/sirupsen/logrus" diff --git a/util/rpc/cache/cache_eth.go b/util/rpc/cache/cache_eth.go index e7f62b9c..61c52e8e 100644 --- a/util/rpc/cache/cache_eth.go +++ b/util/rpc/cache/cache_eth.go @@ -9,7 +9,7 @@ import ( "sync/atomic" "time" - "github.com/Conflux-Chain/go-conflux-util/viper" + "github.com/Conflux-Chain/confura/util/viper" "github.com/ethereum/go-ethereum/common" "github.com/mcuadros/go-defaults" "github.com/openweb3/go-rpc-provider/utils" diff --git a/util/rpc/cached_client_eth.go b/util/rpc/cached_client_eth.go index 97563a58..94358e1e 100644 --- a/util/rpc/cached_client_eth.go +++ b/util/rpc/cached_client_eth.go @@ -16,7 +16,7 @@ import ( "github.com/Conflux-Chain/go-conflux-util/blockchain/sync/evm" "github.com/Conflux-Chain/go-conflux-util/blockchain/sync/poll" "github.com/Conflux-Chain/go-conflux-util/blockchain/sync/process" - viperutil "github.com/Conflux-Chain/go-conflux-util/viper" + viperutil "github.com/Conflux-Chain/confura/util/viper" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" providers "github.com/openweb3/go-rpc-provider/provider_wrapper" diff --git a/util/rpc/config.go b/util/rpc/config.go index c7b1e9b1..29a0d750 100644 --- a/util/rpc/config.go +++ b/util/rpc/config.go @@ -3,7 +3,7 @@ package rpc import ( "time" - "github.com/Conflux-Chain/go-conflux-util/viper" + "github.com/Conflux-Chain/confura/util/viper" ) var ( diff --git a/util/rpc/middlewares/web3pay.go b/util/rpc/middlewares/web3pay.go index a0425032..68494648 100644 --- a/util/rpc/middlewares/web3pay.go +++ b/util/rpc/middlewares/web3pay.go @@ -4,7 +4,7 @@ import ( "strings" "github.com/Conflux-Chain/confura/util/rpc/handlers" - "github.com/Conflux-Chain/go-conflux-util/viper" + "github.com/Conflux-Chain/confura/util/viper" web3pay "github.com/Conflux-Chain/web3pay-service/client" web3paymw "github.com/Conflux-Chain/web3pay-service/client/middleware" "github.com/openweb3/go-rpc-provider" diff --git a/util/rpc/pubsub_limiter.go b/util/rpc/pubsub_limiter.go index b812f6e1..35765ecd 100644 --- a/util/rpc/pubsub_limiter.go +++ b/util/rpc/pubsub_limiter.go @@ -8,7 +8,7 @@ import ( "sync" "github.com/Conflux-Chain/confura/util/rpc/handlers" - "github.com/Conflux-Chain/go-conflux-util/viper" + "github.com/Conflux-Chain/confura/util/viper" "github.com/sirupsen/logrus" ) diff --git a/util/rpc/server.go b/util/rpc/server.go index f675d969..fea1acd8 100644 --- a/util/rpc/server.go +++ b/util/rpc/server.go @@ -9,7 +9,7 @@ import ( "time" "github.com/Conflux-Chain/confura/util/rpc/handlers" - viperutil "github.com/Conflux-Chain/go-conflux-util/viper" + viperutil "github.com/Conflux-Chain/confura/util/viper" "github.com/openweb3/go-rpc-provider" "github.com/sirupsen/logrus" "github.com/spf13/viper" diff --git a/util/viper/viper.go b/util/viper/viper.go new file mode 100644 index 00000000..25010ad3 --- /dev/null +++ b/util/viper/viper.go @@ -0,0 +1,108 @@ +// Package viper preserves Confura's configuration contract independently of +// changes to dependency initialization: require a config file, never load .env, +// and apply default tags before decoding explicit values (including 0/false). +package viper + +import ( + "os" + "sort" + "strings" + + "github.com/mcuadros/go-defaults" + "github.com/mitchellh/mapstructure" + "github.com/sirupsen/logrus" + base "github.com/spf13/viper" +) + +var envKeyPrefix string + +type ValueResolver func(string) (interface{}, bool) + +func MustInit(prefix string, paths ...string) { + if err := Init(prefix, paths...); err != nil { + logrus.WithError(err).Fatal("Failed to read configuration") + } +} + +// Init is startup-only; Viper's global state must not be changed concurrently. +func Init(prefix string, paths ...string) error { + envKeyPrefix = strings.ToUpper(prefix + "_") + base.AutomaticEnv() + base.SetEnvPrefix(prefix) + base.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) + // Bind environment-only settings so dependency decoders can discover them + // through AllKeys without running their dotenv-loading initializer. + for key, env := range environmentKeys() { + if err := base.BindEnv(key, env); err != nil { + return err + } + } + if len(paths) > 0 { + base.SetConfigFile(paths[0]) + } else { + base.SetConfigName("config") + base.AddConfigPath(".") + base.AddConfigPath("./config") + } + return base.ReadInConfig() +} + +func environmentKeys() map[string]string { + keys := make(map[string]string) + if envKeyPrefix == "" { + return keys + } + for _, entry := range os.Environ() { + name, _, ok := strings.Cut(entry, "=") + if !ok || !strings.HasPrefix(name, envKeyPrefix) { + continue + } + key := strings.ToLower(strings.ReplaceAll(strings.TrimPrefix(name, envKeyPrefix), "_", ".")) + keys[key] = name + } + return keys +} + +func MustUnmarshalKey(key string, out interface{}, resolvers ...ValueResolver) { + if err := UnmarshalKey(key, out, resolvers...); err != nil { + logrus.WithError(err).WithField("key", key).Fatal("Failed to decode configuration") + } +} + +func UnmarshalKey(key string, out interface{}, resolvers ...ValueResolver) error { + defaults.SetDefaults(out) + prefix := strings.ToLower(key) + if prefix != "" { + prefix += "." + } + keys := make(map[string]bool) + for _, k := range base.AllKeys() { + if strings.HasPrefix(k, prefix) { + keys[k] = true + } + } + for k := range environmentKeys() { + if strings.HasPrefix(k, prefix) { + keys[k] = true + } + } + ordered := make([]string, 0, len(keys)) + for k := range keys { + ordered = append(ordered, k) + } + sort.Strings(ordered) + sub := base.New() + for _, k := range ordered { + value := base.Get(k) + if len(resolvers) > 0 { + if v, ok := resolvers[0](k); ok { + value = v + } + } + if value != nil { + sub.Set(strings.TrimPrefix(k, prefix), value) + } + } + hook := mapstructure.ComposeDecodeHookFunc(mapstructure.StringToTimeDurationHookFunc(), mapstructure.StringToSliceHookFunc(","), mapstructure.TextUnmarshallerHookFunc()) + return sub.Unmarshal(out, base.DecodeHook(hook)) +} diff --git a/util/viper/viper_test.go b/util/viper/viper_test.go new file mode 100644 index 00000000..b6ae41b5 --- /dev/null +++ b/util/viper/viper_test.go @@ -0,0 +1,48 @@ +package viper + +import ( + "os" + "path/filepath" + "testing" + "time" + + base "github.com/spf13/viper" + "github.com/stretchr/testify/require" +) + +func TestConfigurationContract(t *testing.T) { + base.Reset() + t.Cleanup(base.Reset) + dir := t.TempDir() + file := filepath.Join(dir, "config.yaml") + require.NoError(t, os.WriteFile(file, []byte("fixture:\n count: 0\n enabled: false\n names: [explicit]\n"), 0600)) + t.Setenv("UTILFIX_FIXTURE_DELAY", "2s") + require.NoError(t, Init("UTILFIX", file)) + var conf struct { + Count int `default:"3"` + Enabled bool `default:"true"` + Names []string `default:"[one,two]"` + Delay time.Duration `default:"1s"` + } + require.NoError(t, UnmarshalKey("fixture", &conf)) + require.Zero(t, conf.Count) + require.False(t, conf.Enabled) + require.Equal(t, []string{"explicit", "two"}, conf.Names) + require.Equal(t, 2*time.Second, conf.Delay) +} + +func TestMissingConfigAndDotenv(t *testing.T) { + base.Reset() + t.Cleanup(base.Reset) + dir := t.TempDir() + old, err := os.Getwd() + require.NoError(t, err) + require.NoError(t, os.Chdir(dir)) + t.Cleanup(func() { _ = os.Chdir(old) }) + require.NoError(t, os.WriteFile(".env", []byte("UTILFIX_FROM_DOTENV=unexpected\n"), 0600)) + require.Error(t, Init("UTILFIX")) + require.Empty(t, os.Getenv("UTILFIX_FROM_DOTENV")) + require.NoError(t, os.WriteFile("config.yaml", []byte("{}"), 0600)) + require.NoError(t, Init("UTILFIX")) + require.Empty(t, os.Getenv("UTILFIX_FROM_DOTENV")) +} diff --git a/virtualfilter/client/client.go b/virtualfilter/client/client.go index dfb2e792..c30a31d5 100644 --- a/virtualfilter/client/client.go +++ b/virtualfilter/client/client.go @@ -5,7 +5,7 @@ import ( "time" cfxtypes "github.com/Conflux-Chain/go-conflux-sdk/types" - "github.com/Conflux-Chain/go-conflux-util/viper" + "github.com/Conflux-Chain/confura/util/viper" "github.com/openweb3/go-rpc-provider" "github.com/openweb3/go-rpc-provider/interfaces" providers "github.com/openweb3/go-rpc-provider/provider_wrapper" diff --git a/virtualfilter/config.go b/virtualfilter/config.go index 5401fd67..a4f06005 100644 --- a/virtualfilter/config.go +++ b/virtualfilter/config.go @@ -3,7 +3,7 @@ package virtualfilter import ( "time" - "github.com/Conflux-Chain/go-conflux-util/viper" + "github.com/Conflux-Chain/confura/util/viper" ) // ethConfig represents the configuration of the EVM space virtual filter system. From 2fd5fd08d7f759e75c92e9b2d8d472603bc93778 Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 2 Nov 2026 15:30:47 +0800 Subject: [PATCH 13/26] build(deps): pin hardened go-conflux-util and handle invalid limiter options --- go.mod | 7 ++++--- go.sum | 9 +++++++++ util/rate/registry.go | 10 ++++++++++ util/rate/registry_concurrency_test.go | 19 +++++++++++++++++++ util/rate/strategy.go | 7 ++++++- util/rpc/cached_client_eth.go | 9 +++++++-- 6 files changed, 55 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index a5a53c3a..9e51f2df 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ toolchain go1.23.10 require ( github.com/Conflux-Chain/confura-data-cache v1.0.1-0.20260414083601-cfb736638339 github.com/Conflux-Chain/go-conflux-sdk v1.6.2-0.20260723071534-0585f1e2e56b - github.com/Conflux-Chain/go-conflux-util v0.7.4 + github.com/Conflux-Chain/go-conflux-util v0.7.12-0.20261016064239-f3074203c485 github.com/Conflux-Chain/web3pay-service v1.0.2 github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce github.com/buraksezer/consistent v0.9.0 @@ -18,6 +18,7 @@ require ( github.com/google/uuid v1.6.0 github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d github.com/mcuadros/go-defaults v1.2.0 + github.com/mitchellh/mapstructure v1.4.3 github.com/montanaflynn/stats v0.6.6 github.com/openweb3/go-rpc-provider v0.3.7 github.com/openweb3/web3go v0.3.5 @@ -38,6 +39,7 @@ require ( google.golang.org/grpc v1.75.0 google.golang.org/protobuf v1.36.6 gorm.io/driver/mysql v1.6.0 + gorm.io/driver/sqlite v1.6.0 gorm.io/gorm v1.30.0 gorm.io/hints v1.1.0 ) @@ -111,6 +113,7 @@ require ( github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect + github.com/joho/godotenv v1.5.1 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kilic/bls12-381 v0.1.0 // indirect github.com/klauspost/compress v1.16.0 // indirect @@ -123,7 +126,6 @@ require ( github.com/mattn/go-runewidth v0.0.13 // indirect github.com/mattn/go-sqlite3 v1.14.22 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect - github.com/mitchellh/mapstructure v1.4.3 // indirect github.com/mitchellh/pointerstructure v1.2.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect @@ -176,7 +178,6 @@ require ( gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - gorm.io/driver/sqlite v1.6.0 // indirect gotest.tools v2.2.0+incompatible // indirect ) diff --git a/go.sum b/go.sum index e87bb72a..24dab0b5 100644 --- a/go.sum +++ b/go.sum @@ -46,6 +46,8 @@ github.com/Conflux-Chain/go-conflux-sdk v1.6.2-0.20260723071534-0585f1e2e56b h1: github.com/Conflux-Chain/go-conflux-sdk v1.6.2-0.20260723071534-0585f1e2e56b/go.mod h1:8jfCPUR8MgY+yvqj2rPTHAtQLtt1tvmXuto9VV2eIJg= github.com/Conflux-Chain/go-conflux-util v0.7.4 h1:v1iC+NPEmyBBZHVV3QjYuuaXmiTwMZrAip6t3t+7wOE= github.com/Conflux-Chain/go-conflux-util v0.7.4/go.mod h1:JujW3dd5moi4Vgusd3Xwqbw5sW89MuatoyUSVf/J9Yk= +github.com/Conflux-Chain/go-conflux-util v0.7.12-0.20261016064239-f3074203c485 h1:m+WfWDqKoCcUSARVoHaDxxwf0+XAbwLyrVbS2c0nFCw= +github.com/Conflux-Chain/go-conflux-util v0.7.12-0.20261016064239-f3074203c485/go.mod h1:Mv/ijdSN0UcnWAwctYAB8Y5T3VVEYNSfefK4MJ4Nrmo= github.com/Conflux-Chain/web3pay-service v1.0.2 h1:GiZhlvtF2NfQYjYsl5qNGz2sHg53mIhsAGhEaZrtC5E= github.com/Conflux-Chain/web3pay-service v1.0.2/go.mod h1:jgw6atcXYp0i8pNOs/MhJyvbXPbfDebE5TGIZMcIH/w= github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= @@ -375,8 +377,11 @@ github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.2/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -430,6 +435,7 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-sqlite3 v1.14.8/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -1058,8 +1064,11 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gorm.io/driver/mysql v1.6.0 h1:eNbLmNTpPpTOVZi8MMxCi2aaIm0ZpInbORNXDwyLGvg= gorm.io/driver/mysql v1.6.0/go.mod h1:D/oCC2GWK3M/dqoLxnOlaNKmXz8WNTfcS9y5ovaSqKo= +gorm.io/driver/sqlite v1.1.6/go.mod h1:W8LmC/6UvVbHKah0+QOC7Ja66EaZXHwUTjgXY8YNWX8= gorm.io/driver/sqlite v1.6.0 h1:WHRRrIiulaPiPFmDcod6prc4l2VGVWHz80KspNsxSfQ= gorm.io/driver/sqlite v1.6.0/go.mod h1:AO9V1qIQddBESngQUKWL9yoH93HIeA1X6V633rBwyT8= +gorm.io/gorm v1.21.15/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0= +gorm.io/gorm v1.22.2/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0= gorm.io/gorm v1.30.0 h1:qbT5aPv1UH8gI99OsRlvDToLxW5zR7FzS9acZDOZcgs= gorm.io/gorm v1.30.0/go.mod h1:8Z33v652h4//uMA76KjeDH8mJXPm1QNCYrMeatR0DOE= gorm.io/hints v1.1.0 h1:Lp4z3rxREufSdxn4qmkK3TLDltrM10FLTHiuqwDPvXw= diff --git a/util/rate/registry.go b/util/rate/registry.go index 0a302b47..58b1fd4c 100644 --- a/util/rate/registry.go +++ b/util/rate/registry.go @@ -2,6 +2,7 @@ package rate import ( "context" + "math" "sync" "sync/atomic" "time" @@ -132,8 +133,17 @@ func (r *Registry) Create(ctx context.Context, resource, group string) (rate.Lim func (r *Registry) createWithOption(option interface{}) (l rate.Limiter, err error) { switch opt := option.(type) { case FixedWindowOption: + if opt.Interval <= 0 || opt.Quota <= 0 { + return nil, errors.New("fixed window interval and quota must be positive") + } l = rate.NewFixedWindow(opt.Interval, opt.Quota) case TokenBucketOption: + if err := rate.ValidateTokenBucketConfig(float64(opt.Rate), opt.Burst, 1); err != nil { + return nil, err + } + if float64(opt.Burst)/float64(opt.Rate) >= float64(math.MaxInt64/int64(time.Second)) { + return nil, errors.New("token bucket refill interval is too large") + } l = rate.NewTokenBucketRate(opt.Rate, opt.Burst) default: err = errors.New("invalid limit option") diff --git a/util/rate/registry_concurrency_test.go b/util/rate/registry_concurrency_test.go index 712239b6..ceff67a8 100644 --- a/util/rate/registry_concurrency_test.go +++ b/util/rate/registry_concurrency_test.go @@ -2,6 +2,9 @@ package rate import ( "context" + "encoding/json" + xrate "golang.org/x/time/rate" + "math" "testing" "time" @@ -44,3 +47,19 @@ func TestFractionalTokenBucket(t *testing.T) { t.Fatalf("fractional rate did not refill: %v", err) } } + +func TestInvalidLimiterOptionsReturnErrors(t *testing.T) { + r := &Registry{} + for _, opt := range []interface{}{TokenBucketOption{Rate: 0, Burst: 1}, TokenBucketOption{Rate: -1, Burst: 1}, TokenBucketOption{Rate: 1, Burst: 0}, TokenBucketOption{Rate: xrate.Limit(math.NaN()), Burst: 1}, TokenBucketOption{Rate: 1e-20, Burst: 1}, FixedWindowOption{}, FixedWindowOption{Interval: -1, Quota: 1}} { + if _, err := r.createWithOption(opt); err == nil { + t.Errorf("accepted invalid option %+v", opt) + } + } + var s Strategy + if err := json.Unmarshal([]byte(`{"rpc":null}`), &s); err == nil { + t.Fatal("accepted null rule") + } + if err := json.Unmarshal([]byte(`{"rpc":{"algo":"token_bucket","option":{"rate":0.5,"burst":1}}}`), &s); err != nil { + t.Fatal(err) + } +} diff --git a/util/rate/strategy.go b/util/rate/strategy.go index bc574bd7..a05c21a9 100644 --- a/util/rate/strategy.go +++ b/util/rate/strategy.go @@ -63,9 +63,14 @@ func (s *Strategy) UnmarshalJSON(data []byte) error { return errors.WithMessage(err, "malformed json format") } + options := make(map[string]interface{}, len(tmpRules)) for resource, rule := range tmpRules { - s.LimitOptions[resource] = rule.Option + if rule == nil { + return errors.New("null rate limit rule") + } + options[resource] = rule.Option } + s.LimitOptions = options return nil } diff --git a/util/rpc/cached_client_eth.go b/util/rpc/cached_client_eth.go index 94358e1e..38354875 100644 --- a/util/rpc/cached_client_eth.go +++ b/util/rpc/cached_client_eth.go @@ -13,10 +13,10 @@ import ( "github.com/Conflux-Chain/confura/util" "github.com/Conflux-Chain/confura/util/metrics" lruCache "github.com/Conflux-Chain/confura/util/rpc/cache" + viperutil "github.com/Conflux-Chain/confura/util/viper" "github.com/Conflux-Chain/go-conflux-util/blockchain/sync/evm" "github.com/Conflux-Chain/go-conflux-util/blockchain/sync/poll" "github.com/Conflux-Chain/go-conflux-util/blockchain/sync/process" - viperutil "github.com/Conflux-Chain/confura/util/viper" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" providers "github.com/openweb3/go-rpc-provider/provider_wrapper" @@ -106,7 +106,12 @@ func NewEthNearHeadCacheFromViper(url string) (*EthNearHeadCache, error) { adapter.Close() return nil, err } - poller := poll.NewLatestPoller(adapter, finalized, poll.ReorgWindowParams{}, conf.Poller) + poller, err := poll.NewLatestPoller(adapter, finalized, poll.ReorgWindowParams{}, conf.Poller) + if err != nil { + cancel() + adapter.Close() + return nil, errors.WithMessage(err, "failed to create near head poller") + } cache := &EthNearHeadCache{EthCache: nearhead.NewEthCache(conf.Cache), stopSync: cancel, maxLogs: conf.Cache.MaxLogs} writer := &nearHeadWriter{cache: cache, writer: nearhead.NewWriter(cache.EthCache, conf.Writer)} cache.syncDone.Add(2) From 80bb1ef5cd16d0346a3c28e03a54ebc9174351e3 Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 2 Nov 2026 15:34:38 +0800 Subject: [PATCH 14/26] test(util): cover dependency lock ordering and HTTP batch WebSocket ACLs --- cmd/rpc.go | 2 +- rpc/acl_boundary_test.go | 110 +++++++++++++++++++++++++ rpc/handler/cfx_gasstation.go | 2 +- rpc/handler/cfx_logs.go | 2 +- rpc/handler/cfx_txn.go | 2 +- store/log_filter.go | 2 +- store/mysql/config.go | 2 +- sync/catchup/syncer.go | 2 +- sync/election/leader_manager.go | 2 +- sync/sync_db.go | 2 +- sync/sync_eth.go | 2 +- sync/sync_trace_log.go | 2 +- util/boundary_test.go | 3 +- util/rate/loader.go | 2 +- util/rate/registry_concurrency_test.go | 45 +++++++++- util/rate/registry_reload.go | 1 - util/rate/util_test.go | 3 +- util/relay/relay.go | 2 +- util/rpc/middlewares/metrics.go | 7 -- virtualfilter/client/client.go | 2 +- 20 files changed, 172 insertions(+), 25 deletions(-) create mode 100644 rpc/acl_boundary_test.go diff --git a/cmd/rpc.go b/cmd/rpc.go index 4a3acdef..f1dae508 100644 --- a/cmd/rpc.go +++ b/cmd/rpc.go @@ -18,8 +18,8 @@ import ( "github.com/Conflux-Chain/confura/util/rate" "github.com/Conflux-Chain/confura/util/relay" rpcutil "github.com/Conflux-Chain/confura/util/rpc" - vfclient "github.com/Conflux-Chain/confura/virtualfilter/client" viperutil "github.com/Conflux-Chain/confura/util/viper" + vfclient "github.com/Conflux-Chain/confura/virtualfilter/client" ) var ( diff --git a/rpc/acl_boundary_test.go b/rpc/acl_boundary_test.go new file mode 100644 index 00000000..40f1a4bd --- /dev/null +++ b/rpc/acl_boundary_test.go @@ -0,0 +1,110 @@ +package rpc + +import ( + "context" + "encoding/json" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/Conflux-Chain/confura/util/acl" + "github.com/Conflux-Chain/confura/util/rate" + "github.com/Conflux-Chain/confura/util/rpc/handlers" + "github.com/Conflux-Chain/confura/util/rpc/middlewares" + "github.com/ethereum/go-ethereum/common" + rpc "github.com/openweb3/go-rpc-provider" + wt "github.com/openweb3/web3go/types" + "github.com/stretchr/testify/require" +) + +type ACLBoundaryAPI struct{} + +func (*ACLBoundaryAPI) GetLogs(context.Context, wt.FilterQuery) ([]wt.Log, error) { + return []wt.Log{{Address: common.HexToAddress("0x2222222222222222222222222222222222222222")}}, nil +} +func (*ACLBoundaryAPI) NewFilter(context.Context, wt.FilterQuery) (string, error) { return "0x1", nil } +func (*ACLBoundaryAPI) Echo(ctx context.Context) (string, error) { + id, _ := handlers.GetAuthIdFromContext(ctx) + return id, nil +} +func (*ACLBoundaryAPI) Logs(ctx context.Context, f wt.FilterQuery) (*rpc.Subscription, error) { + n, _ := rpc.NotifierFromContext(ctx) + return n.CreateSubscription(), nil +} + +func TestContractACLHTTPBatchAndWebSocket(t *testing.T) { + // Same production registration order; no config loading, external services, or real credentials. + rpc.HookHandleCallMsg(middlewares.Recover) + rpc.HookHandleCallMsg(middlewares.AntiInjection) + rpc.HookHandleCallMsg(middlewares.Authenticate) + rpc.HookHandleCallMsg(middlewares.Allowlists) + rpc.HookHandleCallMsg(middlewares.DailyMaxReqRateLimit) + rpc.HookHandleCallMsg(middlewares.QpsRateLimit) + rpc.HookHandleCallMsg(middlewares.Metrics) + rpc.HookHandleCallMsg(middlewares.Log) + rpc.HookHandleCallMsg(clientMiddleware) + rpc.HookHandleCallMsg(middlewares.UniformError) + rpc.HookHandleCallMsg(rpc.PreventMessagesWithouID) + const key = "FixtureProvisionedSynthetic0123456789" + loader := rate.NewKeyLoader(func(f *rate.KeysetFilter) ([]*rate.KeyInfo, error) { + if len(f.KeySet) == 0 || f.KeySet[0] == key { + return []*rate.KeyInfo{{Key: key, SID: 1, AclID: 1}}, nil + } + return nil, nil + }) + r := rate.NewRegistry(loader, acl.NewEthValidator) + s := rate.NewStrategy(1, "default") + s.LimitOptions["rpc_all_qps"] = rate.NewTokenBucketOption(10000, 10000) + s.LimitOptions["rpc_all_daily"] = rate.FixedWindowOption{Interval: time.Hour, Quota: 10000} + config := &rate.Config{Strategies: map[uint32]*rate.Strategy{1: s}, AllowLists: map[uint32]*acl.AllowList{1: {ID: 1, Name: "default", ContractAddresses: []string{"0x1111111111111111111111111111111111111111"}}}} + go r.AutoReload(time.Hour, func() (*rate.Config, error) { return config, nil }) + require.Eventually(t, func() bool { + return len(r.Strategies()) == 1 && r.Allow(acl.Context{Context: context.Background(), RpcMethod: "eth_getLogs", ExtractRpcParams: func() ([]interface{}, error) { return []interface{}{wt.FilterQuery{}}, nil }}) != nil + }, time.Second, time.Millisecond) + h := rpc.NewServer() + defer h.Stop() + require.NoError(t, h.RegisterName("eth", &ACLBoundaryAPI{})) + httpH := httpMiddleware("eth", r, nil)(h) + request := func(path, token, body string) string { + q := httptest.NewRequest("POST", "http://fixture.invalid"+path, strings.NewReader(body)) + q.Header.Set("Content-Type", "application/json") + q.Header.Set("Access-Token", token) + w := httptest.NewRecorder() + httpH.ServeHTTP(w, q) + return w.Body.String() + } + for _, tc := range []struct{ name, path, token, body, want string }{ + {"guest", "/", "", `{"jsonrpc":"2.0","id":1,"method":"eth_echo","params":[]}`, `"result":""`}, + {"unknown", "/", "FixtureUnknownSynthetic0123456789", `{"jsonrpc":"2.0","id":1,"method":"eth_echo","params":[]}`, "invalid api key"}, + {"provisioned", "/", key, `{"jsonrpc":"2.0","id":1,"method":"eth_echo","params":[]}`, key}, + {"path beats header", "/FixtureUnknownSynthetic0123456789", key, `{"jsonrpc":"2.0","id":1,"method":"eth_echo","params":[]}`, "invalid api key"}, + {"deny getLogs", "/", "", `{"jsonrpc":"2.0","id":1,"method":"eth_getLogs","params":[{"address":"0x2222222222222222222222222222222222222222"}]}`, "invalid contract address"}, + {"malformed params", "/", "", `{"jsonrpc":"2.0","id":1,"method":"eth_getLogs","params":[9]}`, "bad RPC parameters"}, + {"filter denied", "/", "", `{"jsonrpc":"2.0","id":1,"method":"eth_newFilter","params":[{"address":"0x2222222222222222222222222222222222222222"}]}`, "invalid contract address"}, + {"batch", "/", "", `[{"jsonrpc":"2.0","id":1,"method":"eth_getLogs","params":[{}]},{"jsonrpc":"2.0","id":2,"method":"eth_newFilter","params":[{}]}]`, "invalid contract address"}, + {"notification", "/", "", `{"jsonrpc":"2.0","method":"eth_echo","params":[]}`, "invalid request"}, + } { + t.Run(tc.name, func(t *testing.T) { + out := request(tc.path, tc.token, tc.body) + require.Contains(t, out, tc.want) + if tc.name == "batch" { + require.NotContains(t, out, `"result"`) + } + }) + } + // Real WebSocket upgrade and provider subscription dispatch, with identical HTTP context injector. + ws := httptest.NewServer(httpMiddleware("eth", r, nil)(h.WebsocketHandler([]string{"*"}))) + defer ws.Close() + conn, err := rpc.Dial("ws" + strings.TrimPrefix(ws.URL, "http")) + require.NoError(t, err) + defer conn.Close() + logs := make(chan json.RawMessage, 1) + sub, err := conn.Subscribe(context.Background(), "eth", logs, "logs", map[string]interface{}{"address": "0x2222222222222222222222222222222222222222"}) + require.ErrorContains(t, err, "invalid contract address") + require.Nil(t, sub) + sub, err = conn.Subscribe(context.Background(), "eth", logs, "logs", map[string]interface{}{"address": "0x1111111111111111111111111111111111111111"}) + require.NoError(t, err) + sub.Unsubscribe() + +} diff --git a/rpc/handler/cfx_gasstation.go b/rpc/handler/cfx_gasstation.go index 0b7504a2..4130137a 100644 --- a/rpc/handler/cfx_gasstation.go +++ b/rpc/handler/cfx_gasstation.go @@ -8,9 +8,9 @@ import ( "github.com/Conflux-Chain/confura/node" "github.com/Conflux-Chain/confura/types" "github.com/Conflux-Chain/confura/util" + "github.com/Conflux-Chain/confura/util/viper" sdk "github.com/Conflux-Chain/go-conflux-sdk" cfxtypes "github.com/Conflux-Chain/go-conflux-sdk/types" - "github.com/Conflux-Chain/confura/util/viper" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/openweb3/go-rpc-provider/utils" "github.com/sirupsen/logrus" diff --git a/rpc/handler/cfx_logs.go b/rpc/handler/cfx_logs.go index f8e958e3..9f2cbc18 100644 --- a/rpc/handler/cfx_logs.go +++ b/rpc/handler/cfx_logs.go @@ -9,9 +9,9 @@ import ( "github.com/Conflux-Chain/confura/store/mysql" citypes "github.com/Conflux-Chain/confura/types" "github.com/Conflux-Chain/confura/util/metrics" + viperutil "github.com/Conflux-Chain/confura/util/viper" sdk "github.com/Conflux-Chain/go-conflux-sdk" "github.com/Conflux-Chain/go-conflux-sdk/types" - viperutil "github.com/Conflux-Chain/confura/util/viper" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/pkg/errors" "github.com/sirupsen/logrus" diff --git a/rpc/handler/cfx_txn.go b/rpc/handler/cfx_txn.go index 83fe5bab..708f9eaf 100644 --- a/rpc/handler/cfx_txn.go +++ b/rpc/handler/cfx_txn.go @@ -7,9 +7,9 @@ import ( "github.com/Conflux-Chain/confura/util" "github.com/Conflux-Chain/confura/util/relay" rpcutil "github.com/Conflux-Chain/confura/util/rpc" + "github.com/Conflux-Chain/confura/util/viper" sdk "github.com/Conflux-Chain/go-conflux-sdk" "github.com/Conflux-Chain/go-conflux-sdk/types" - "github.com/Conflux-Chain/confura/util/viper" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/rpc" "github.com/openweb3/go-rpc-provider/utils" diff --git a/store/log_filter.go b/store/log_filter.go index 9a470351..3dc8d042 100644 --- a/store/log_filter.go +++ b/store/log_filter.go @@ -6,8 +6,8 @@ import ( "time" citypes "github.com/Conflux-Chain/confura/types" - "github.com/Conflux-Chain/go-conflux-sdk/types" "github.com/Conflux-Chain/confura/util/viper" + "github.com/Conflux-Chain/go-conflux-sdk/types" web3Types "github.com/openweb3/web3go/types" "github.com/pkg/errors" diff --git a/store/mysql/config.go b/store/mysql/config.go index 197b042c..a1829cbc 100644 --- a/store/mysql/config.go +++ b/store/mysql/config.go @@ -7,8 +7,8 @@ import ( "time" "github.com/Conflux-Chain/confura/store" - "github.com/Conflux-Chain/go-conflux-util/dlock" "github.com/Conflux-Chain/confura/util/viper" + "github.com/Conflux-Chain/go-conflux-util/dlock" gosql "github.com/go-sql-driver/mysql" "github.com/sirupsen/logrus" "gorm.io/driver/mysql" diff --git a/sync/catchup/syncer.go b/sync/catchup/syncer.go index 02c8d850..1de71715 100644 --- a/sync/catchup/syncer.go +++ b/sync/catchup/syncer.go @@ -10,8 +10,8 @@ import ( "github.com/Conflux-Chain/confura/store/mysql" "github.com/Conflux-Chain/confura/sync/election" "github.com/Conflux-Chain/confura/sync/monitor" - sdk "github.com/Conflux-Chain/go-conflux-sdk" viperutil "github.com/Conflux-Chain/confura/util/viper" + sdk "github.com/Conflux-Chain/go-conflux-sdk" "github.com/openweb3/web3go" "github.com/pkg/errors" "github.com/sirupsen/logrus" diff --git a/sync/election/leader_manager.go b/sync/election/leader_manager.go index 07bc703f..c6fb38c3 100644 --- a/sync/election/leader_manager.go +++ b/sync/election/leader_manager.go @@ -7,9 +7,9 @@ import ( "time" "github.com/Conflux-Chain/confura/store" + "github.com/Conflux-Chain/confura/util/viper" "github.com/Conflux-Chain/go-conflux-util/dlock" "github.com/Conflux-Chain/go-conflux-util/health" - "github.com/Conflux-Chain/confura/util/viper" "github.com/google/uuid" "github.com/pkg/errors" "github.com/sirupsen/logrus" diff --git a/sync/sync_db.go b/sync/sync_db.go index 62fd2070..47906938 100644 --- a/sync/sync_db.go +++ b/sync/sync_db.go @@ -13,11 +13,11 @@ import ( "github.com/Conflux-Chain/confura/sync/monitor" citypes "github.com/Conflux-Chain/confura/types" "github.com/Conflux-Chain/confura/util/metrics" + viperutil "github.com/Conflux-Chain/confura/util/viper" sdk "github.com/Conflux-Chain/go-conflux-sdk" "github.com/Conflux-Chain/go-conflux-sdk/types" "github.com/Conflux-Chain/go-conflux-util/dlock" "github.com/Conflux-Chain/go-conflux-util/health" - viperutil "github.com/Conflux-Chain/confura/util/viper" "github.com/pkg/errors" "github.com/sirupsen/logrus" "gorm.io/gorm" diff --git a/sync/sync_eth.go b/sync/sync_eth.go index aeae8f0f..a02c1080 100644 --- a/sync/sync_eth.go +++ b/sync/sync_eth.go @@ -12,9 +12,9 @@ import ( "github.com/Conflux-Chain/confura/sync/election" "github.com/Conflux-Chain/confura/sync/monitor" "github.com/Conflux-Chain/confura/util/metrics" + viperutil "github.com/Conflux-Chain/confura/util/viper" "github.com/Conflux-Chain/go-conflux-util/dlock" "github.com/Conflux-Chain/go-conflux-util/health" - viperutil "github.com/Conflux-Chain/confura/util/viper" "github.com/openweb3/web3go" ethtypes "github.com/openweb3/web3go/types" "github.com/pkg/errors" diff --git a/sync/sync_trace_log.go b/sync/sync_trace_log.go index cd4851aa..67b256bd 100644 --- a/sync/sync_trace_log.go +++ b/sync/sync_trace_log.go @@ -7,6 +7,7 @@ import ( "github.com/Conflux-Chain/confura/store/mysql" "github.com/Conflux-Chain/confura/sync/tracelog" rpcutil "github.com/Conflux-Chain/confura/util/rpc" + viperutil "github.com/Conflux-Chain/confura/util/viper" sdk "github.com/Conflux-Chain/go-conflux-sdk" "github.com/Conflux-Chain/go-conflux-sdk/types" "github.com/Conflux-Chain/go-conflux-util/blockchain/sync" @@ -14,7 +15,6 @@ import ( "github.com/Conflux-Chain/go-conflux-util/blockchain/sync/poll" "github.com/Conflux-Chain/go-conflux-util/blockchain/sync/process/db" "github.com/Conflux-Chain/go-conflux-util/ctxutil" - viperutil "github.com/Conflux-Chain/confura/util/viper" "github.com/sirupsen/logrus" ) diff --git a/util/boundary_test.go b/util/boundary_test.go index baad7cf4..fe79db6d 100644 --- a/util/boundary_test.go +++ b/util/boundary_test.go @@ -1,9 +1,10 @@ package util import ( + "testing" + "github.com/ethereum/go-ethereum/common" web3Types "github.com/openweb3/web3go/types" - "testing" ) func TestTopicArity(t *testing.T) { diff --git a/util/rate/loader.go b/util/rate/loader.go index 2747b364..1c3227eb 100644 --- a/util/rate/loader.go +++ b/util/rate/loader.go @@ -2,12 +2,12 @@ package rate import ( "errors" - "golang.org/x/sync/singleflight" "sync" "time" "github.com/Conflux-Chain/confura/util" "github.com/sirupsen/logrus" + "golang.org/x/sync/singleflight" ) const ( diff --git a/util/rate/registry_concurrency_test.go b/util/rate/registry_concurrency_test.go index ceff67a8..23709267 100644 --- a/util/rate/registry_concurrency_test.go +++ b/util/rate/registry_concurrency_test.go @@ -3,12 +3,13 @@ package rate import ( "context" "encoding/json" - xrate "golang.org/x/time/rate" "math" "testing" "time" limiter "github.com/Conflux-Chain/go-conflux-util/rate" + httprate "github.com/Conflux-Chain/go-conflux-util/rate/http" + xrate "golang.org/x/time/rate" ) func TestCreateDoesNotAcquireStrategyLock(t *testing.T) { @@ -63,3 +64,45 @@ func TestInvalidLimiterOptionsReturnErrors(t *testing.T) { t.Fatal(err) } } + +type gatedLimiterFactory struct { + registry *Registry + entered chan struct{} + release chan struct{} +} + +func (f *gatedLimiterFactory) GetGroupAndKey(context.Context, string) (string, string, error) { + return "default", "fixture", nil +} +func (f *gatedLimiterFactory) Create(ctx context.Context, resource, group string) (limiter.Limiter, error) { + close(f.entered) + <-f.release + return f.registry.Create(ctx, resource, group) +} +func TestReloadAndDependencyCreateMakeProgress(t *testing.T) { + strategy := &Strategy{ID: 1, Name: "default", LimitOptions: map[string]interface{}{"rpc": TokenBucketOption{Rate: 1, Burst: 1}}} + r := &Registry{strategies: map[string]*Strategy{"default": strategy}, id2Strategies: map[uint32]*Strategy{1: strategy}} + r.publishLimiterStrategies() + factory := &gatedLimiterFactory{registry: r, entered: make(chan struct{}), release: make(chan struct{})} + r.Registry = httprate.NewRegistry(factory) + requestDone := make(chan error, 1) + go func() { requestDone <- r.Limit(context.Background(), "rpc") }() + <-factory.entered // dependency now owns its limiter registry lock + r.mu.Lock() // reload owns the strategy lock + defer r.mu.Unlock() + reloadDone := make(chan struct{}) + go func() { r.removeStrategy(strategy); close(reloadDone) }() + close(factory.release) + for _, done := range []<-chan struct{}{reloadDone} { + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("reload deadlocked") + } + } + select { + case <-requestDone: + case <-time.After(time.Second): + t.Fatal("request deadlocked") + } +} diff --git a/util/rate/registry_reload.go b/util/rate/registry_reload.go index 023cb77f..39f1b7eb 100644 --- a/util/rate/registry_reload.go +++ b/util/rate/registry_reload.go @@ -109,7 +109,6 @@ func (m *Registry) removeStrategy(s *Strategy) { m.Remove(resource, s.Name) logrus.WithField("resource", resource).Info("RateLimit rule deleted") } - } func (m *Registry) addStrategy(s *Strategy) { diff --git a/util/rate/util_test.go b/util/rate/util_test.go index 94a7330b..44951883 100644 --- a/util/rate/util_test.go +++ b/util/rate/util_test.go @@ -4,9 +4,10 @@ import ( "bytes" "crypto/rand" "errors" + "testing" + "github.com/btcsuite/btcutil/base58" "github.com/stretchr/testify/require" - "testing" ) type failingRandomReader struct{} diff --git a/util/relay/relay.go b/util/relay/relay.go index 202a93c9..c6e1770b 100644 --- a/util/relay/relay.go +++ b/util/relay/relay.go @@ -5,8 +5,8 @@ import ( "github.com/Conflux-Chain/confura/util" rpcutil "github.com/Conflux-Chain/confura/util/rpc" - sdk "github.com/Conflux-Chain/go-conflux-sdk" "github.com/Conflux-Chain/confura/util/viper" + sdk "github.com/Conflux-Chain/go-conflux-sdk" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/openweb3/web3go" "github.com/sirupsen/logrus" diff --git a/util/rpc/middlewares/metrics.go b/util/rpc/middlewares/metrics.go index 915414b3..0ed768bd 100644 --- a/util/rpc/middlewares/metrics.go +++ b/util/rpc/middlewares/metrics.go @@ -2,8 +2,6 @@ package middlewares import ( "context" - "fmt" - "strings" "time" "github.com/Conflux-Chain/confura/util/metrics" @@ -61,11 +59,6 @@ func getTrafficSourceFromContext(ctx context.Context) (source string) { return source } -func isMethodNotFoundByError(method string, err error) bool { - subPattern := fmt.Sprintf("the method %s does not exist/is not available", method) - return strings.Contains(err.Error(), subPattern) -} - func unwrapJsonError(jsErr *rpc.JsonError) error { if jsErr == nil { return nil diff --git a/virtualfilter/client/client.go b/virtualfilter/client/client.go index c30a31d5..265fc706 100644 --- a/virtualfilter/client/client.go +++ b/virtualfilter/client/client.go @@ -4,8 +4,8 @@ import ( "context" "time" - cfxtypes "github.com/Conflux-Chain/go-conflux-sdk/types" "github.com/Conflux-Chain/confura/util/viper" + cfxtypes "github.com/Conflux-Chain/go-conflux-sdk/types" "github.com/openweb3/go-rpc-provider" "github.com/openweb3/go-rpc-provider/interfaces" providers "github.com/openweb3/go-rpc-provider/provider_wrapper" From 7b26cb46da74fd65a0c1a07b373cae74eedc3ad4 Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 2 Nov 2026 17:35:58 +0800 Subject: [PATCH 15/26] fix(util): retain existing dependencies and configuration initialization --- cmd/rpc.go | 2 +- config/config.go | 14 +--- go.mod | 7 +- go.sum | 9 --- node/config.go | 2 +- rpc/handler/cfx_gasstation.go | 2 +- rpc/handler/cfx_logs.go | 2 +- rpc/handler/cfx_txn.go | 2 +- rpc/handler/eth_gasstation.go | 2 +- rpc/handler/eth_txn.go | 2 +- store/eth_data.go | 2 +- store/log_filter.go | 2 +- store/mysql/config.go | 2 +- store/store.go | 2 +- sync/catchup/syncer.go | 2 +- sync/election/leader_manager.go | 2 +- sync/sync_db.go | 2 +- sync/sync_eth.go | 2 +- sync/sync_trace_log.go | 2 +- util/pprof/pprof.go | 2 +- util/rate/registry.go | 7 +- util/rate/registry_concurrency_test.go | 2 +- util/relay/relay.go | 2 +- util/rpc/cache/cache_eth.go | 2 +- util/rpc/cached_client_eth.go | 9 +-- util/rpc/config.go | 2 +- util/rpc/middlewares/web3pay.go | 2 +- util/rpc/pubsub_limiter.go | 2 +- util/rpc/server.go | 2 +- util/viper/viper.go | 108 ------------------------- util/viper/viper_test.go | 48 ----------- virtualfilter/client/client.go | 2 +- virtualfilter/config.go | 2 +- 33 files changed, 38 insertions(+), 216 deletions(-) delete mode 100644 util/viper/viper.go delete mode 100644 util/viper/viper_test.go diff --git a/cmd/rpc.go b/cmd/rpc.go index f1dae508..d5b0fcf0 100644 --- a/cmd/rpc.go +++ b/cmd/rpc.go @@ -18,8 +18,8 @@ import ( "github.com/Conflux-Chain/confura/util/rate" "github.com/Conflux-Chain/confura/util/relay" rpcutil "github.com/Conflux-Chain/confura/util/rpc" - viperutil "github.com/Conflux-Chain/confura/util/viper" vfclient "github.com/Conflux-Chain/confura/virtualfilter/client" + viperutil "github.com/Conflux-Chain/go-conflux-util/viper" ) var ( diff --git a/config/config.go b/config/config.go index 657cf904..f74ad9ac 100644 --- a/config/config.go +++ b/config/config.go @@ -11,10 +11,7 @@ import ( "github.com/Conflux-Chain/confura/util/pprof" rpcutil "github.com/Conflux-Chain/confura/util/rpc" "github.com/Conflux-Chain/confura/util/rpc/cache" - "github.com/Conflux-Chain/confura/util/viper" - "github.com/Conflux-Chain/go-conflux-util/alert" - logutil "github.com/Conflux-Chain/go-conflux-util/log" - metricsutil "github.com/Conflux-Chain/go-conflux-util/metrics" + "github.com/Conflux-Chain/go-conflux-util/config" "github.com/ethereum/go-ethereum/log" slogrus "github.com/samber/slog-logrus/v2" "github.com/sirupsen/logrus" @@ -26,14 +23,7 @@ const viperEnvPrefix = "infura" func Init() { // init utilities eg., viper, alert, metrics and logging - viper.MustInit(viperEnvPrefix) - var metricConfig metricsutil.Config - viper.MustUnmarshalKey("metrics", &metricConfig) - metricsutil.Init(metricConfig) - alert.MustInitFromViper() - var logConfig logutil.LoggingConfig - viper.MustUnmarshalKey("log", &logConfig) - logutil.MustInit(logConfig) + config.MustInit(viperEnvPrefix) // init pprof pprof.MustInit() diff --git a/go.mod b/go.mod index 9e51f2df..a5a53c3a 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ toolchain go1.23.10 require ( github.com/Conflux-Chain/confura-data-cache v1.0.1-0.20260414083601-cfb736638339 github.com/Conflux-Chain/go-conflux-sdk v1.6.2-0.20260723071534-0585f1e2e56b - github.com/Conflux-Chain/go-conflux-util v0.7.12-0.20261016064239-f3074203c485 + github.com/Conflux-Chain/go-conflux-util v0.7.4 github.com/Conflux-Chain/web3pay-service v1.0.2 github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce github.com/buraksezer/consistent v0.9.0 @@ -18,7 +18,6 @@ require ( github.com/google/uuid v1.6.0 github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d github.com/mcuadros/go-defaults v1.2.0 - github.com/mitchellh/mapstructure v1.4.3 github.com/montanaflynn/stats v0.6.6 github.com/openweb3/go-rpc-provider v0.3.7 github.com/openweb3/web3go v0.3.5 @@ -39,7 +38,6 @@ require ( google.golang.org/grpc v1.75.0 google.golang.org/protobuf v1.36.6 gorm.io/driver/mysql v1.6.0 - gorm.io/driver/sqlite v1.6.0 gorm.io/gorm v1.30.0 gorm.io/hints v1.1.0 ) @@ -113,7 +111,6 @@ require ( github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect - github.com/joho/godotenv v1.5.1 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kilic/bls12-381 v0.1.0 // indirect github.com/klauspost/compress v1.16.0 // indirect @@ -126,6 +123,7 @@ require ( github.com/mattn/go-runewidth v0.0.13 // indirect github.com/mattn/go-sqlite3 v1.14.22 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect + github.com/mitchellh/mapstructure v1.4.3 // indirect github.com/mitchellh/pointerstructure v1.2.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect @@ -178,6 +176,7 @@ require ( gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + gorm.io/driver/sqlite v1.6.0 // indirect gotest.tools v2.2.0+incompatible // indirect ) diff --git a/go.sum b/go.sum index 24dab0b5..e87bb72a 100644 --- a/go.sum +++ b/go.sum @@ -46,8 +46,6 @@ github.com/Conflux-Chain/go-conflux-sdk v1.6.2-0.20260723071534-0585f1e2e56b h1: github.com/Conflux-Chain/go-conflux-sdk v1.6.2-0.20260723071534-0585f1e2e56b/go.mod h1:8jfCPUR8MgY+yvqj2rPTHAtQLtt1tvmXuto9VV2eIJg= github.com/Conflux-Chain/go-conflux-util v0.7.4 h1:v1iC+NPEmyBBZHVV3QjYuuaXmiTwMZrAip6t3t+7wOE= github.com/Conflux-Chain/go-conflux-util v0.7.4/go.mod h1:JujW3dd5moi4Vgusd3Xwqbw5sW89MuatoyUSVf/J9Yk= -github.com/Conflux-Chain/go-conflux-util v0.7.12-0.20261016064239-f3074203c485 h1:m+WfWDqKoCcUSARVoHaDxxwf0+XAbwLyrVbS2c0nFCw= -github.com/Conflux-Chain/go-conflux-util v0.7.12-0.20261016064239-f3074203c485/go.mod h1:Mv/ijdSN0UcnWAwctYAB8Y5T3VVEYNSfefK4MJ4Nrmo= github.com/Conflux-Chain/web3pay-service v1.0.2 h1:GiZhlvtF2NfQYjYsl5qNGz2sHg53mIhsAGhEaZrtC5E= github.com/Conflux-Chain/web3pay-service v1.0.2/go.mod h1:jgw6atcXYp0i8pNOs/MhJyvbXPbfDebE5TGIZMcIH/w= github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= @@ -377,11 +375,8 @@ github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= -github.com/jinzhu/now v1.1.2/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= -github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= -github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -435,7 +430,6 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mattn/go-sqlite3 v1.14.8/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -1064,11 +1058,8 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gorm.io/driver/mysql v1.6.0 h1:eNbLmNTpPpTOVZi8MMxCi2aaIm0ZpInbORNXDwyLGvg= gorm.io/driver/mysql v1.6.0/go.mod h1:D/oCC2GWK3M/dqoLxnOlaNKmXz8WNTfcS9y5ovaSqKo= -gorm.io/driver/sqlite v1.1.6/go.mod h1:W8LmC/6UvVbHKah0+QOC7Ja66EaZXHwUTjgXY8YNWX8= gorm.io/driver/sqlite v1.6.0 h1:WHRRrIiulaPiPFmDcod6prc4l2VGVWHz80KspNsxSfQ= gorm.io/driver/sqlite v1.6.0/go.mod h1:AO9V1qIQddBESngQUKWL9yoH93HIeA1X6V633rBwyT8= -gorm.io/gorm v1.21.15/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0= -gorm.io/gorm v1.22.2/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0= gorm.io/gorm v1.30.0 h1:qbT5aPv1UH8gI99OsRlvDToLxW5zR7FzS9acZDOZcgs= gorm.io/gorm v1.30.0/go.mod h1:8Z33v652h4//uMA76KjeDH8mJXPm1QNCYrMeatR0DOE= gorm.io/hints v1.1.0 h1:Lp4z3rxREufSdxn4qmkK3TLDltrM10FLTHiuqwDPvXw= diff --git a/node/config.go b/node/config.go index 0ff4b1f2..9df713a6 100644 --- a/node/config.go +++ b/node/config.go @@ -3,7 +3,7 @@ package node import ( "time" - "github.com/Conflux-Chain/confura/util/viper" + "github.com/Conflux-Chain/go-conflux-util/viper" "github.com/buraksezer/consistent" "github.com/sirupsen/logrus" ) diff --git a/rpc/handler/cfx_gasstation.go b/rpc/handler/cfx_gasstation.go index 4130137a..fc027136 100644 --- a/rpc/handler/cfx_gasstation.go +++ b/rpc/handler/cfx_gasstation.go @@ -8,9 +8,9 @@ import ( "github.com/Conflux-Chain/confura/node" "github.com/Conflux-Chain/confura/types" "github.com/Conflux-Chain/confura/util" - "github.com/Conflux-Chain/confura/util/viper" sdk "github.com/Conflux-Chain/go-conflux-sdk" cfxtypes "github.com/Conflux-Chain/go-conflux-sdk/types" + "github.com/Conflux-Chain/go-conflux-util/viper" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/openweb3/go-rpc-provider/utils" "github.com/sirupsen/logrus" diff --git a/rpc/handler/cfx_logs.go b/rpc/handler/cfx_logs.go index 9f2cbc18..c72a8310 100644 --- a/rpc/handler/cfx_logs.go +++ b/rpc/handler/cfx_logs.go @@ -9,9 +9,9 @@ import ( "github.com/Conflux-Chain/confura/store/mysql" citypes "github.com/Conflux-Chain/confura/types" "github.com/Conflux-Chain/confura/util/metrics" - viperutil "github.com/Conflux-Chain/confura/util/viper" sdk "github.com/Conflux-Chain/go-conflux-sdk" "github.com/Conflux-Chain/go-conflux-sdk/types" + viperutil "github.com/Conflux-Chain/go-conflux-util/viper" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/pkg/errors" "github.com/sirupsen/logrus" diff --git a/rpc/handler/cfx_txn.go b/rpc/handler/cfx_txn.go index 708f9eaf..e010fe0c 100644 --- a/rpc/handler/cfx_txn.go +++ b/rpc/handler/cfx_txn.go @@ -7,9 +7,9 @@ import ( "github.com/Conflux-Chain/confura/util" "github.com/Conflux-Chain/confura/util/relay" rpcutil "github.com/Conflux-Chain/confura/util/rpc" - "github.com/Conflux-Chain/confura/util/viper" sdk "github.com/Conflux-Chain/go-conflux-sdk" "github.com/Conflux-Chain/go-conflux-sdk/types" + "github.com/Conflux-Chain/go-conflux-util/viper" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/rpc" "github.com/openweb3/go-rpc-provider/utils" diff --git a/rpc/handler/eth_gasstation.go b/rpc/handler/eth_gasstation.go index 911aee4f..44e19068 100644 --- a/rpc/handler/eth_gasstation.go +++ b/rpc/handler/eth_gasstation.go @@ -7,7 +7,7 @@ import ( "github.com/Conflux-Chain/confura/node" "github.com/Conflux-Chain/confura/types" - "github.com/Conflux-Chain/confura/util/viper" + "github.com/Conflux-Chain/go-conflux-util/viper" "github.com/ethereum/go-ethereum/common" "github.com/openweb3/go-rpc-provider/utils" ethtypes "github.com/openweb3/web3go/types" diff --git a/rpc/handler/eth_txn.go b/rpc/handler/eth_txn.go index c983eac6..4980edce 100644 --- a/rpc/handler/eth_txn.go +++ b/rpc/handler/eth_txn.go @@ -7,7 +7,7 @@ import ( "github.com/Conflux-Chain/confura/util" "github.com/Conflux-Chain/confura/util/relay" rpcutil "github.com/Conflux-Chain/confura/util/rpc" - "github.com/Conflux-Chain/confura/util/viper" + "github.com/Conflux-Chain/go-conflux-util/viper" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/rpc" diff --git a/store/eth_data.go b/store/eth_data.go index 263e8593..3157e68c 100644 --- a/store/eth_data.go +++ b/store/eth_data.go @@ -8,7 +8,7 @@ import ( "time" "github.com/Conflux-Chain/confura/util/metrics" - "github.com/Conflux-Chain/confura/util/viper" + "github.com/Conflux-Chain/go-conflux-util/viper" "github.com/ethereum/go-ethereum/common" "github.com/openweb3/go-rpc-provider/utils" "github.com/openweb3/web3go" diff --git a/store/log_filter.go b/store/log_filter.go index 3dc8d042..fe2190d1 100644 --- a/store/log_filter.go +++ b/store/log_filter.go @@ -6,8 +6,8 @@ import ( "time" citypes "github.com/Conflux-Chain/confura/types" - "github.com/Conflux-Chain/confura/util/viper" "github.com/Conflux-Chain/go-conflux-sdk/types" + "github.com/Conflux-Chain/go-conflux-util/viper" web3Types "github.com/openweb3/web3go/types" "github.com/pkg/errors" diff --git a/store/mysql/config.go b/store/mysql/config.go index a1829cbc..b5d48ef9 100644 --- a/store/mysql/config.go +++ b/store/mysql/config.go @@ -7,8 +7,8 @@ import ( "time" "github.com/Conflux-Chain/confura/store" - "github.com/Conflux-Chain/confura/util/viper" "github.com/Conflux-Chain/go-conflux-util/dlock" + "github.com/Conflux-Chain/go-conflux-util/viper" gosql "github.com/go-sql-driver/mysql" "github.com/sirupsen/logrus" "gorm.io/driver/mysql" diff --git a/store/store.go b/store/store.go index 7731757a..292c695b 100644 --- a/store/store.go +++ b/store/store.go @@ -1,7 +1,7 @@ package store import ( - "github.com/Conflux-Chain/confura/util/viper" + "github.com/Conflux-Chain/go-conflux-util/viper" "github.com/sirupsen/logrus" ) diff --git a/sync/catchup/syncer.go b/sync/catchup/syncer.go index 1de71715..c8dc41f4 100644 --- a/sync/catchup/syncer.go +++ b/sync/catchup/syncer.go @@ -10,8 +10,8 @@ import ( "github.com/Conflux-Chain/confura/store/mysql" "github.com/Conflux-Chain/confura/sync/election" "github.com/Conflux-Chain/confura/sync/monitor" - viperutil "github.com/Conflux-Chain/confura/util/viper" sdk "github.com/Conflux-Chain/go-conflux-sdk" + viperutil "github.com/Conflux-Chain/go-conflux-util/viper" "github.com/openweb3/web3go" "github.com/pkg/errors" "github.com/sirupsen/logrus" diff --git a/sync/election/leader_manager.go b/sync/election/leader_manager.go index c6fb38c3..5025d21f 100644 --- a/sync/election/leader_manager.go +++ b/sync/election/leader_manager.go @@ -7,9 +7,9 @@ import ( "time" "github.com/Conflux-Chain/confura/store" - "github.com/Conflux-Chain/confura/util/viper" "github.com/Conflux-Chain/go-conflux-util/dlock" "github.com/Conflux-Chain/go-conflux-util/health" + "github.com/Conflux-Chain/go-conflux-util/viper" "github.com/google/uuid" "github.com/pkg/errors" "github.com/sirupsen/logrus" diff --git a/sync/sync_db.go b/sync/sync_db.go index 47906938..911ab7b8 100644 --- a/sync/sync_db.go +++ b/sync/sync_db.go @@ -13,11 +13,11 @@ import ( "github.com/Conflux-Chain/confura/sync/monitor" citypes "github.com/Conflux-Chain/confura/types" "github.com/Conflux-Chain/confura/util/metrics" - viperutil "github.com/Conflux-Chain/confura/util/viper" sdk "github.com/Conflux-Chain/go-conflux-sdk" "github.com/Conflux-Chain/go-conflux-sdk/types" "github.com/Conflux-Chain/go-conflux-util/dlock" "github.com/Conflux-Chain/go-conflux-util/health" + viperutil "github.com/Conflux-Chain/go-conflux-util/viper" "github.com/pkg/errors" "github.com/sirupsen/logrus" "gorm.io/gorm" diff --git a/sync/sync_eth.go b/sync/sync_eth.go index a02c1080..dc44ef7f 100644 --- a/sync/sync_eth.go +++ b/sync/sync_eth.go @@ -12,9 +12,9 @@ import ( "github.com/Conflux-Chain/confura/sync/election" "github.com/Conflux-Chain/confura/sync/monitor" "github.com/Conflux-Chain/confura/util/metrics" - viperutil "github.com/Conflux-Chain/confura/util/viper" "github.com/Conflux-Chain/go-conflux-util/dlock" "github.com/Conflux-Chain/go-conflux-util/health" + viperutil "github.com/Conflux-Chain/go-conflux-util/viper" "github.com/openweb3/web3go" ethtypes "github.com/openweb3/web3go/types" "github.com/pkg/errors" diff --git a/sync/sync_trace_log.go b/sync/sync_trace_log.go index 67b256bd..b12f1768 100644 --- a/sync/sync_trace_log.go +++ b/sync/sync_trace_log.go @@ -7,7 +7,6 @@ import ( "github.com/Conflux-Chain/confura/store/mysql" "github.com/Conflux-Chain/confura/sync/tracelog" rpcutil "github.com/Conflux-Chain/confura/util/rpc" - viperutil "github.com/Conflux-Chain/confura/util/viper" sdk "github.com/Conflux-Chain/go-conflux-sdk" "github.com/Conflux-Chain/go-conflux-sdk/types" "github.com/Conflux-Chain/go-conflux-util/blockchain/sync" @@ -15,6 +14,7 @@ import ( "github.com/Conflux-Chain/go-conflux-util/blockchain/sync/poll" "github.com/Conflux-Chain/go-conflux-util/blockchain/sync/process/db" "github.com/Conflux-Chain/go-conflux-util/ctxutil" + viperutil "github.com/Conflux-Chain/go-conflux-util/viper" "github.com/sirupsen/logrus" ) diff --git a/util/pprof/pprof.go b/util/pprof/pprof.go index 1fdc5ccf..a7f54ca4 100644 --- a/util/pprof/pprof.go +++ b/util/pprof/pprof.go @@ -5,7 +5,7 @@ import ( "net/http" _ "net/http/pprof" - "github.com/Conflux-Chain/confura/util/viper" + "github.com/Conflux-Chain/go-conflux-util/viper" "github.com/sirupsen/logrus" ) diff --git a/util/rate/registry.go b/util/rate/registry.go index 58b1fd4c..97f85aab 100644 --- a/util/rate/registry.go +++ b/util/rate/registry.go @@ -138,8 +138,11 @@ func (r *Registry) createWithOption(option interface{}) (l rate.Limiter, err err } l = rate.NewFixedWindow(opt.Interval, opt.Quota) case TokenBucketOption: - if err := rate.ValidateTokenBucketConfig(float64(opt.Rate), opt.Burst, 1); err != nil { - return nil, err + if opt.Rate <= 0 || math.IsNaN(float64(opt.Rate)) || math.IsInf(float64(opt.Rate), 0) { + return nil, errors.New("token bucket rate must be positive and finite") + } + if opt.Burst <= 0 { + return nil, errors.New("token bucket burst must be positive") } if float64(opt.Burst)/float64(opt.Rate) >= float64(math.MaxInt64/int64(time.Second)) { return nil, errors.New("token bucket refill interval is too large") diff --git a/util/rate/registry_concurrency_test.go b/util/rate/registry_concurrency_test.go index 23709267..08588a9c 100644 --- a/util/rate/registry_concurrency_test.go +++ b/util/rate/registry_concurrency_test.go @@ -51,7 +51,7 @@ func TestFractionalTokenBucket(t *testing.T) { func TestInvalidLimiterOptionsReturnErrors(t *testing.T) { r := &Registry{} - for _, opt := range []interface{}{TokenBucketOption{Rate: 0, Burst: 1}, TokenBucketOption{Rate: -1, Burst: 1}, TokenBucketOption{Rate: 1, Burst: 0}, TokenBucketOption{Rate: xrate.Limit(math.NaN()), Burst: 1}, TokenBucketOption{Rate: 1e-20, Burst: 1}, FixedWindowOption{}, FixedWindowOption{Interval: -1, Quota: 1}} { + for _, opt := range []interface{}{TokenBucketOption{Rate: 0, Burst: 1}, TokenBucketOption{Rate: -1, Burst: 1}, TokenBucketOption{Rate: 1, Burst: 0}, TokenBucketOption{Rate: xrate.Limit(math.NaN()), Burst: 1}, TokenBucketOption{Rate: xrate.Limit(math.Inf(1)), Burst: 1}, TokenBucketOption{Rate: 1e-20, Burst: 1}, FixedWindowOption{}, FixedWindowOption{Interval: -1, Quota: 1}} { if _, err := r.createWithOption(opt); err == nil { t.Errorf("accepted invalid option %+v", opt) } diff --git a/util/relay/relay.go b/util/relay/relay.go index c6e1770b..f5d69ca0 100644 --- a/util/relay/relay.go +++ b/util/relay/relay.go @@ -5,8 +5,8 @@ import ( "github.com/Conflux-Chain/confura/util" rpcutil "github.com/Conflux-Chain/confura/util/rpc" - "github.com/Conflux-Chain/confura/util/viper" sdk "github.com/Conflux-Chain/go-conflux-sdk" + "github.com/Conflux-Chain/go-conflux-util/viper" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/openweb3/web3go" "github.com/sirupsen/logrus" diff --git a/util/rpc/cache/cache_eth.go b/util/rpc/cache/cache_eth.go index 61c52e8e..e7f62b9c 100644 --- a/util/rpc/cache/cache_eth.go +++ b/util/rpc/cache/cache_eth.go @@ -9,7 +9,7 @@ import ( "sync/atomic" "time" - "github.com/Conflux-Chain/confura/util/viper" + "github.com/Conflux-Chain/go-conflux-util/viper" "github.com/ethereum/go-ethereum/common" "github.com/mcuadros/go-defaults" "github.com/openweb3/go-rpc-provider/utils" diff --git a/util/rpc/cached_client_eth.go b/util/rpc/cached_client_eth.go index 38354875..97563a58 100644 --- a/util/rpc/cached_client_eth.go +++ b/util/rpc/cached_client_eth.go @@ -13,10 +13,10 @@ import ( "github.com/Conflux-Chain/confura/util" "github.com/Conflux-Chain/confura/util/metrics" lruCache "github.com/Conflux-Chain/confura/util/rpc/cache" - viperutil "github.com/Conflux-Chain/confura/util/viper" "github.com/Conflux-Chain/go-conflux-util/blockchain/sync/evm" "github.com/Conflux-Chain/go-conflux-util/blockchain/sync/poll" "github.com/Conflux-Chain/go-conflux-util/blockchain/sync/process" + viperutil "github.com/Conflux-Chain/go-conflux-util/viper" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" providers "github.com/openweb3/go-rpc-provider/provider_wrapper" @@ -106,12 +106,7 @@ func NewEthNearHeadCacheFromViper(url string) (*EthNearHeadCache, error) { adapter.Close() return nil, err } - poller, err := poll.NewLatestPoller(adapter, finalized, poll.ReorgWindowParams{}, conf.Poller) - if err != nil { - cancel() - adapter.Close() - return nil, errors.WithMessage(err, "failed to create near head poller") - } + poller := poll.NewLatestPoller(adapter, finalized, poll.ReorgWindowParams{}, conf.Poller) cache := &EthNearHeadCache{EthCache: nearhead.NewEthCache(conf.Cache), stopSync: cancel, maxLogs: conf.Cache.MaxLogs} writer := &nearHeadWriter{cache: cache, writer: nearhead.NewWriter(cache.EthCache, conf.Writer)} cache.syncDone.Add(2) diff --git a/util/rpc/config.go b/util/rpc/config.go index 29a0d750..c7b1e9b1 100644 --- a/util/rpc/config.go +++ b/util/rpc/config.go @@ -3,7 +3,7 @@ package rpc import ( "time" - "github.com/Conflux-Chain/confura/util/viper" + "github.com/Conflux-Chain/go-conflux-util/viper" ) var ( diff --git a/util/rpc/middlewares/web3pay.go b/util/rpc/middlewares/web3pay.go index 68494648..a0425032 100644 --- a/util/rpc/middlewares/web3pay.go +++ b/util/rpc/middlewares/web3pay.go @@ -4,7 +4,7 @@ import ( "strings" "github.com/Conflux-Chain/confura/util/rpc/handlers" - "github.com/Conflux-Chain/confura/util/viper" + "github.com/Conflux-Chain/go-conflux-util/viper" web3pay "github.com/Conflux-Chain/web3pay-service/client" web3paymw "github.com/Conflux-Chain/web3pay-service/client/middleware" "github.com/openweb3/go-rpc-provider" diff --git a/util/rpc/pubsub_limiter.go b/util/rpc/pubsub_limiter.go index 35765ecd..b812f6e1 100644 --- a/util/rpc/pubsub_limiter.go +++ b/util/rpc/pubsub_limiter.go @@ -8,7 +8,7 @@ import ( "sync" "github.com/Conflux-Chain/confura/util/rpc/handlers" - "github.com/Conflux-Chain/confura/util/viper" + "github.com/Conflux-Chain/go-conflux-util/viper" "github.com/sirupsen/logrus" ) diff --git a/util/rpc/server.go b/util/rpc/server.go index fea1acd8..f675d969 100644 --- a/util/rpc/server.go +++ b/util/rpc/server.go @@ -9,7 +9,7 @@ import ( "time" "github.com/Conflux-Chain/confura/util/rpc/handlers" - viperutil "github.com/Conflux-Chain/confura/util/viper" + viperutil "github.com/Conflux-Chain/go-conflux-util/viper" "github.com/openweb3/go-rpc-provider" "github.com/sirupsen/logrus" "github.com/spf13/viper" diff --git a/util/viper/viper.go b/util/viper/viper.go deleted file mode 100644 index 25010ad3..00000000 --- a/util/viper/viper.go +++ /dev/null @@ -1,108 +0,0 @@ -// Package viper preserves Confura's configuration contract independently of -// changes to dependency initialization: require a config file, never load .env, -// and apply default tags before decoding explicit values (including 0/false). -package viper - -import ( - "os" - "sort" - "strings" - - "github.com/mcuadros/go-defaults" - "github.com/mitchellh/mapstructure" - "github.com/sirupsen/logrus" - base "github.com/spf13/viper" -) - -var envKeyPrefix string - -type ValueResolver func(string) (interface{}, bool) - -func MustInit(prefix string, paths ...string) { - if err := Init(prefix, paths...); err != nil { - logrus.WithError(err).Fatal("Failed to read configuration") - } -} - -// Init is startup-only; Viper's global state must not be changed concurrently. -func Init(prefix string, paths ...string) error { - envKeyPrefix = strings.ToUpper(prefix + "_") - base.AutomaticEnv() - base.SetEnvPrefix(prefix) - base.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) - // Bind environment-only settings so dependency decoders can discover them - // through AllKeys without running their dotenv-loading initializer. - for key, env := range environmentKeys() { - if err := base.BindEnv(key, env); err != nil { - return err - } - } - if len(paths) > 0 { - base.SetConfigFile(paths[0]) - } else { - base.SetConfigName("config") - base.AddConfigPath(".") - base.AddConfigPath("./config") - } - return base.ReadInConfig() -} - -func environmentKeys() map[string]string { - keys := make(map[string]string) - if envKeyPrefix == "" { - return keys - } - for _, entry := range os.Environ() { - name, _, ok := strings.Cut(entry, "=") - if !ok || !strings.HasPrefix(name, envKeyPrefix) { - continue - } - key := strings.ToLower(strings.ReplaceAll(strings.TrimPrefix(name, envKeyPrefix), "_", ".")) - keys[key] = name - } - return keys -} - -func MustUnmarshalKey(key string, out interface{}, resolvers ...ValueResolver) { - if err := UnmarshalKey(key, out, resolvers...); err != nil { - logrus.WithError(err).WithField("key", key).Fatal("Failed to decode configuration") - } -} - -func UnmarshalKey(key string, out interface{}, resolvers ...ValueResolver) error { - defaults.SetDefaults(out) - prefix := strings.ToLower(key) - if prefix != "" { - prefix += "." - } - keys := make(map[string]bool) - for _, k := range base.AllKeys() { - if strings.HasPrefix(k, prefix) { - keys[k] = true - } - } - for k := range environmentKeys() { - if strings.HasPrefix(k, prefix) { - keys[k] = true - } - } - ordered := make([]string, 0, len(keys)) - for k := range keys { - ordered = append(ordered, k) - } - sort.Strings(ordered) - sub := base.New() - for _, k := range ordered { - value := base.Get(k) - if len(resolvers) > 0 { - if v, ok := resolvers[0](k); ok { - value = v - } - } - if value != nil { - sub.Set(strings.TrimPrefix(k, prefix), value) - } - } - hook := mapstructure.ComposeDecodeHookFunc(mapstructure.StringToTimeDurationHookFunc(), mapstructure.StringToSliceHookFunc(","), mapstructure.TextUnmarshallerHookFunc()) - return sub.Unmarshal(out, base.DecodeHook(hook)) -} diff --git a/util/viper/viper_test.go b/util/viper/viper_test.go deleted file mode 100644 index b6ae41b5..00000000 --- a/util/viper/viper_test.go +++ /dev/null @@ -1,48 +0,0 @@ -package viper - -import ( - "os" - "path/filepath" - "testing" - "time" - - base "github.com/spf13/viper" - "github.com/stretchr/testify/require" -) - -func TestConfigurationContract(t *testing.T) { - base.Reset() - t.Cleanup(base.Reset) - dir := t.TempDir() - file := filepath.Join(dir, "config.yaml") - require.NoError(t, os.WriteFile(file, []byte("fixture:\n count: 0\n enabled: false\n names: [explicit]\n"), 0600)) - t.Setenv("UTILFIX_FIXTURE_DELAY", "2s") - require.NoError(t, Init("UTILFIX", file)) - var conf struct { - Count int `default:"3"` - Enabled bool `default:"true"` - Names []string `default:"[one,two]"` - Delay time.Duration `default:"1s"` - } - require.NoError(t, UnmarshalKey("fixture", &conf)) - require.Zero(t, conf.Count) - require.False(t, conf.Enabled) - require.Equal(t, []string{"explicit", "two"}, conf.Names) - require.Equal(t, 2*time.Second, conf.Delay) -} - -func TestMissingConfigAndDotenv(t *testing.T) { - base.Reset() - t.Cleanup(base.Reset) - dir := t.TempDir() - old, err := os.Getwd() - require.NoError(t, err) - require.NoError(t, os.Chdir(dir)) - t.Cleanup(func() { _ = os.Chdir(old) }) - require.NoError(t, os.WriteFile(".env", []byte("UTILFIX_FROM_DOTENV=unexpected\n"), 0600)) - require.Error(t, Init("UTILFIX")) - require.Empty(t, os.Getenv("UTILFIX_FROM_DOTENV")) - require.NoError(t, os.WriteFile("config.yaml", []byte("{}"), 0600)) - require.NoError(t, Init("UTILFIX")) - require.Empty(t, os.Getenv("UTILFIX_FROM_DOTENV")) -} diff --git a/virtualfilter/client/client.go b/virtualfilter/client/client.go index 265fc706..dfb2e792 100644 --- a/virtualfilter/client/client.go +++ b/virtualfilter/client/client.go @@ -4,8 +4,8 @@ import ( "context" "time" - "github.com/Conflux-Chain/confura/util/viper" cfxtypes "github.com/Conflux-Chain/go-conflux-sdk/types" + "github.com/Conflux-Chain/go-conflux-util/viper" "github.com/openweb3/go-rpc-provider" "github.com/openweb3/go-rpc-provider/interfaces" providers "github.com/openweb3/go-rpc-provider/provider_wrapper" diff --git a/virtualfilter/config.go b/virtualfilter/config.go index a4f06005..5401fd67 100644 --- a/virtualfilter/config.go +++ b/virtualfilter/config.go @@ -3,7 +3,7 @@ package virtualfilter import ( "time" - "github.com/Conflux-Chain/confura/util/viper" + "github.com/Conflux-Chain/go-conflux-util/viper" ) // ethConfig represents the configuration of the EVM space virtual filter system. From 885296cbe186b89e63d326a91872c2b00e5f32ff Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 3 Nov 2026 10:32:23 +0800 Subject: [PATCH 16/26] fix(acl): preserve filter ID read semantics --- rpc/acl_boundary_test.go | 21 ++++++++++++++ util/acl/validator.go | 8 ++---- util/acl/validator_boundary_test.go | 43 +++++++++++++++++++++++++++-- 3 files changed, 63 insertions(+), 9 deletions(-) diff --git a/rpc/acl_boundary_test.go b/rpc/acl_boundary_test.go index 40f1a4bd..714224ef 100644 --- a/rpc/acl_boundary_test.go +++ b/rpc/acl_boundary_test.go @@ -24,6 +24,12 @@ func (*ACLBoundaryAPI) GetLogs(context.Context, wt.FilterQuery) ([]wt.Log, error return []wt.Log{{Address: common.HexToAddress("0x2222222222222222222222222222222222222222")}}, nil } func (*ACLBoundaryAPI) NewFilter(context.Context, wt.FilterQuery) (string, error) { return "0x1", nil } +func (*ACLBoundaryAPI) GetFilterLogs(context.Context, rpc.ID) ([]wt.Log, error) { + return []wt.Log{{Address: common.HexToAddress("0x1111111111111111111111111111111111111111")}}, nil +} +func (api *ACLBoundaryAPI) GetFilterChanges(ctx context.Context, id rpc.ID) ([]wt.Log, error) { + return api.GetFilterLogs(ctx, id) +} func (*ACLBoundaryAPI) Echo(ctx context.Context) (string, error) { id, _ := handlers.GetAuthIdFromContext(ctx) return id, nil @@ -93,6 +99,21 @@ func TestContractACLHTTPBatchAndWebSocket(t *testing.T) { } }) } + t.Run("allowed filter remains readable", func(t *testing.T) { + created := request("/", key, `{"jsonrpc":"2.0","id":1,"method":"eth_newFilter","params":[{"address":"0x1111111111111111111111111111111111111111"}]}`) + var response struct { + Result string `json:"result"` + Error json.RawMessage `json:"error"` + } + require.NoError(t, json.Unmarshal([]byte(created), &response)) + require.Empty(t, response.Error, created) + require.Equal(t, "0x1", response.Result) + for _, method := range []string{"eth_getFilterLogs", "eth_getFilterChanges"} { + out := request("/", key, `{"jsonrpc":"2.0","id":2,"method":"`+method+`","params":["`+response.Result+`"]}`) + require.Contains(t, out, `"result":[{"address":"0x1111111111111111111111111111111111111111"`, out) + require.NotContains(t, out, `"error"`, out) + } + }) // Real WebSocket upgrade and provider subscription dispatch, with identical HTTP context injector. ws := httptest.NewServer(httpMiddleware("eth", r, nil)(h.WebsocketHandler([]string{"*"}))) defer ws.Close() diff --git a/util/acl/validator.go b/util/acl/validator.go index 6a893a32..e4e114d6 100644 --- a/util/acl/validator.go +++ b/util/acl/validator.go @@ -196,12 +196,8 @@ func (v *validatorBase) validateContractAddresses(ctx Context) error { return nil } - // A filter ID carries no verifiable contract constraint. Until filter - // ownership is tracked, do not accept IDs created by another identity. - switch ctx.RpcMethod { - case "eth_getFilterLogs", "eth_getFilterChanges", "cfx_getFilterLogs", "cfx_getFilterChanges": - return errInvalidContractAddr - } + // Check filter criteria at creation. Filter-ID reads retain their existing + // access semantics; they do not carry addresses to validate here. if ctx.RpcMethod == "eth_subscribe" || ctx.RpcMethod == "cfx_subscribe" { var args []json.RawMessage var kind string diff --git a/util/acl/validator_boundary_test.go b/util/acl/validator_boundary_test.go index 74711005..aaed18a2 100644 --- a/util/acl/validator_boundary_test.go +++ b/util/acl/validator_boundary_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/Conflux-Chain/confura/util/rpc/handlers" + cfxTypes "github.com/Conflux-Chain/go-conflux-sdk/types" "github.com/Conflux-Chain/go-conflux-sdk/types/cfxaddress" "github.com/ethereum/go-ethereum/common" web3Types "github.com/openweb3/web3go/types" @@ -34,9 +35,45 @@ func TestContractACLFilterAndSubscriptionBoundaries(t *testing.T) { err := v.Validate(ctx) require.Equal(t, tc.allowed, err == nil, tc.params) } - for _, method := range []string{"eth_getFilterLogs", "eth_getFilterChanges"} { - require.Error(t, v.Validate(Context{RpcMethod: method})) - require.NoError(t, NewEthValidator(&AllowList{}).Validate(Context{RpcMethod: method})) +} + +func TestContractACLRetainsFilterIDReads(t *testing.T) { + for _, tc := range []struct { + prefix string + factory ValidatorFactory + }{ + {"eth", NewEthValidator}, + {"cfx", NewCfxValidator}, + } { + t.Run(tc.prefix, func(t *testing.T) { + for _, suffix := range []string{"getFilterLogs", "getFilterChanges"} { + method := tc.prefix + "_" + suffix + ctx := Context{Context: context.Background(), RpcMethod: method, RawParams: json.RawMessage(`["0x1"]`)} + require.NoError(t, tc.factory(&AllowList{ContractAddresses: []string{allowedContractAddr}}).Validate(ctx)) + require.NoError(t, tc.factory(&AllowList{}).Validate(ctx)) + // Filter-ID reads still enforce method restrictions. + require.ErrorIs(t, tc.factory(&AllowList{ContractAddresses: []string{allowedContractAddr}, DisallowMethods: []string{method}}).Validate(ctx), errBadRpcMethod) + } + }) + } +} + +func TestCoreNewFilterContractACL(t *testing.T) { + allowed := cfxaddress.MustNew(allowedContractAddr, 1) + denied := cfxaddress.MustNew(notAllowedContractAddr, 1) + v := NewCfxValidator(&AllowList{ContractAddresses: []string{allowed.MustGetBase32Address()}}) + for _, tc := range []struct { + addresses []cfxaddress.Address + allowed bool + }{ + {[]cfxaddress.Address{allowed}, true}, + {[]cfxaddress.Address{denied}, false}, + {[]cfxaddress.Address{allowed, denied}, false}, + {nil, false}, + } { + ctx := cfxGetLogsContext(cfxTypes.LogFilter{Address: tc.addresses}) + ctx.RpcMethod = "cfx_newFilter" + require.Equal(t, tc.allowed, v.Validate(ctx) == nil) } } From f02527c4e82a38e163e13e9d058b818a297c5d33 Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 3 Nov 2026 11:31:36 +0800 Subject: [PATCH 17/26] fix(util): preserve Conflux wildcard topic semantics --- util/blockchain.go | 4 +--- util/boundary_test.go | 37 +++++++++++++++++++++++++++---------- util/rpc/nearhead_test.go | 19 +++++++++++++++---- 3 files changed, 43 insertions(+), 17 deletions(-) diff --git a/util/blockchain.go b/util/blockchain.go index 661d5da5..e0c897ca 100644 --- a/util/blockchain.go +++ b/util/blockchain.go @@ -194,15 +194,13 @@ func GetEthHardforkBlockNumber(chainId uint64) web3goTypes.BlockNumber { // MatchEthLogTopics checks if the eSpace event log matches with the given topics condition func MatchEthLogTopics(log *web3goTypes.Log, topics [][]common.Hash) bool { - if len(topics) > len(log.Topics) { - return false - } find := func(t common.Hash, topics []common.Hash) bool { return slices.Contains(topics, t) } for i := range topics { if len(topics[i]) == 0 { + // Conflux eSpace wildcards also match missing topic positions. continue } diff --git a/util/boundary_test.go b/util/boundary_test.go index fe79db6d..8deddd04 100644 --- a/util/boundary_test.go +++ b/util/boundary_test.go @@ -7,16 +7,33 @@ import ( web3Types "github.com/openweb3/web3go/types" ) -func TestTopicArity(t *testing.T) { - h := common.HexToHash("0x1") - log := &web3Types.Log{Topics: []common.Hash{h}} - for _, topics := range [][][]common.Hash{{{h}, {}}, {{h}, {h}}, {{}, {}}} { - if MatchEthLogTopics(log, topics) { - t.Fatal("accepted missing topic") - } - } - if !MatchEthLogTopics(log, [][]common.Hash{{}}) { - t.Fatal("rejected existing wildcard position") +// Follow Conflux eSpace's LogFilterParams::matches, including missing wildcard +// positions: https://github.com/Conflux-Chain/conflux-rust/blob/2cb16583060d7a569271adb59cc1a9cd92dfd447/crates/primitives/src/filter.rs#L317-L339 +func TestEthLogTopicsMatchConfluxSemantics(t *testing.T) { + a, b := common.HexToHash("0x1"), common.HexToHash("0x2") + for _, tc := range []struct { + name string + log []common.Hash + filter [][]common.Hash + want bool + }{ + {"no conditions", nil, nil, true}, + {"existing wildcard", []common.Hash{a}, [][]common.Hash{{}}, true}, + {"missing null position", []common.Hash{a}, [][]common.Hash{{a}, nil}, true}, + {"missing empty position", []common.Hash{a}, [][]common.Hash{{a}, {}}, true}, + {"all wildcard positions", []common.Hash{a}, [][]common.Hash{nil, nil}, true}, + {"wildcard without topics", nil, [][]common.Hash{nil}, true}, + {"missing required topic", []common.Hash{a}, [][]common.Hash{{a}, {b}}, false}, + {"wildcard then missing required topic", []common.Hash{a}, [][]common.Hash{nil, {b}}, false}, + {"wildcard hole", []common.Hash{a, b}, [][]common.Hash{nil, {b}}, true}, + {"matching alternative", []common.Hash{a}, [][]common.Hash{{b, a}}, true}, + {"nonmatching topic", []common.Hash{a}, [][]common.Hash{{b}}, false}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := MatchEthLogTopics(&web3Types.Log{Topics: tc.log}, tc.filter); got != tc.want { + t.Fatalf("matched %v, want %v", got, tc.want) + } + }) } } diff --git a/util/rpc/nearhead_test.go b/util/rpc/nearhead_test.go index 85c11921..d6968ae2 100644 --- a/util/rpc/nearhead_test.go +++ b/util/rpc/nearhead_test.go @@ -65,10 +65,21 @@ func TestNearHeadLogTopicsAndOwnership(t *testing.T) { n := fixtureNearhead() hash := common.HexToHash("0xa") require.NoError(t, n.Put(fixtureBlock(100, hash, []common.Hash{hash}))) - for _, topics := range [][][]common.Hash{{{hash}, {}}, {{hash}, {hash}}} { - logs, err := n.GetLogsByBlockHash(hash, nearhead.FilterOpt{Topics: topics}) - require.NoError(t, err) - require.Empty(t, logs.Logs) + for _, tc := range []struct { + name string + topics [][]common.Hash + count int + }{ + {"missing null position", [][]common.Hash{{hash}, nil}, 1}, + {"missing empty position", [][]common.Hash{{hash}, {}}, 1}, + {"missing required topic", [][]common.Hash{{hash}, {hash}}, 0}, + {"wildcard then missing required topic", [][]common.Hash{nil, {hash}}, 0}, + } { + t.Run(tc.name, func(t *testing.T) { + logs, err := n.GetLogsByBlockHash(hash, nearhead.FilterOpt{Topics: tc.topics}) + require.NoError(t, err) + require.Len(t, logs.Logs, tc.count) + }) } logs, err := n.GetLogsByBlockHash(hash, nearhead.FilterOpt{}) require.NoError(t, err) From b8e93f4a032096c022f4bba1fcd05da8b4e412de Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 3 Nov 2026 11:55:52 +0800 Subject: [PATCH 18/26] fix(rpc): preserve Conflux trace_get index semantics --- util/rpc/cached_client_eth.go | 21 +++---- util/rpc/cached_client_eth_test.go | 92 ++++++++++++++++++++++++++---- 2 files changed, 93 insertions(+), 20 deletions(-) diff --git a/util/rpc/cached_client_eth.go b/util/rpc/cached_client_eth.go index 97563a58..74ef0ce3 100644 --- a/util/rpc/cached_client_eth.go +++ b/util/rpc/cached_client_eth.go @@ -4,7 +4,6 @@ import ( "context" "math" "math/big" - "slices" "sync" "github.com/Conflux-Chain/confura-data-cache/nearhead" @@ -937,13 +936,18 @@ func (c traceCacheClient) LazyTrace( transactionHash common.Hash, indexes []uint, ) (res cacheTypes.Lazy[*web3Types.LocalizedTrace], _ error) { + if len(indexes) == 0 { + return res, nil + } + // Conflux eSpace uses the first index as an offset in transaction traces. + index := indexes[0] tracker := newRPCCacheTracker(rpcMethodTraceGet) defer tracker.report() return readThroughLazyCacheChain( tracker, func() (res cacheTypes.Lazy[*web3Types.LocalizedTrace], err error) { - if trace := traceAtAddress(c.nearhead.GetTransactionTraces(transactionHash), indexes); trace != nil { + if trace := traceAtIndex(c.nearhead.GetTransactionTraces(transactionHash), index); trace != nil { res, err = cacheTypes.NewLazy(trace) } return res, err @@ -957,7 +961,7 @@ func (c traceCacheClient) LazyTrace( if err != nil { return cacheTypes.Lazy[*web3Types.LocalizedTrace]{}, err } - return cacheTypes.NewLazy(traceAtAddress(loaded, indexes)) + return cacheTypes.NewLazy(traceAtIndex(loaded, index)) }, func() (cacheTypes.Lazy[*web3Types.LocalizedTrace], error) { hexIndexes := make([]hexutil.Uint, len(indexes)) @@ -1356,13 +1360,10 @@ func readThroughLazyCacheChain[T any]( ) } -// trace_get indexes are a path through the call tree, not an offset in the -// flattened trace_transaction response. An empty path selects the root. -func traceAtAddress(traces []web3Types.LocalizedTrace, path []uint) *web3Types.LocalizedTrace { - for i := range traces { - if slices.Equal(traces[i].TraceAddress, path) { - return &traces[i] - } +// Keep the index unsigned so large inputs cannot wrap into a valid offset. +func traceAtIndex(traces []web3Types.LocalizedTrace, index uint) *web3Types.LocalizedTrace { + if index < uint(len(traces)) { + return &traces[index] } return nil } diff --git a/util/rpc/cached_client_eth_test.go b/util/rpc/cached_client_eth_test.go index a8358668..67badfb4 100644 --- a/util/rpc/cached_client_eth_test.go +++ b/util/rpc/cached_client_eth_test.go @@ -74,23 +74,95 @@ func TestLargeTransactionIndexBypassesNarrowCache(t *testing.T) { type fixtureTraceCache struct{ cacheRpc.Interface } func (c *fixtureTraceCache) GetTransactionTraces(common.Hash) (ct.Lazy[[]wt.LocalizedTrace], error) { + return ct.NewLazy(fixtureTransactionTraces()) +} + +func fixtureTransactionTraces() []wt.LocalizedTrace { var traces []wt.LocalizedTrace + txHash := common.HexToHash("0x1") for _, path := range [][]uint{{}, {0}, {0, 1}, {0, 2}} { var trace wt.LocalizedTrace _ = json.Unmarshal([]byte(`{"type":"call","action":{"from":"0x0000000000000000000000000000000000000000","to":"0x0000000000000000000000000000000000000000","gas":"0x0","input":"0x","value":"0x0","callType":"call"},"traceAddress":[],"subtraces":0,"blockNumber":100,"blockHash":"0x0000000000000000000000000000000000000000000000000000000000000001"}`), &trace) trace.TraceAddress = path + trace.TransactionHash = &txHash traces = append(traces, trace) } - return ct.NewLazy(traces) + return traces } -func TestTraceGetUsesEntirePathIncludingRoot(t *testing.T) { - c := traceCacheClient{cacheClientContext: cacheClientContext{nearhead: fixtureNearhead(), dataCache: &fixtureTraceCache{cacheRpc.NotFoundImpl}}} - for _, path := range [][]uint{{}, {0, 1}, {0, 2}} { - lazy, err := c.LazyTrace(common.Hash{}, path) - require.NoError(t, err) - got, err := lazy.Load() - require.NoError(t, err) - require.NotNil(t, got) - require.Equal(t, path, got.TraceAddress) + +// Conflux eSpace selects the first flat index and returns null for empty input: +// https://github.com/Conflux-Chain/conflux-rust/blob/2cb16583060d7a569271adb59cc1a9cd92dfd447/crates/rpc/rpc-eth-impl/src/trace.rs#L305-L319 +func TestTraceGetMatchesConfluxSemantics(t *testing.T) { + traces, txHash := fixtureTransactionTraces(), common.HexToHash("0x1") + for _, source := range []string{"nearhead", "data-cache", "fullnode"} { + t.Run(source, func(t *testing.T) { + for _, tc := range []struct { + name string + indexes []uint + want int // -1 means null + }{ + {"empty", []uint{}, -1}, + {"nil", nil, -1}, + {"first", []uint{0}, 0}, + {"second", []uint{1}, 1}, + {"third", []uint{2}, 2}, + {"ignore suffix one", []uint{0, 1}, 0}, + {"ignore suffix two", []uint{0, 2}, 0}, + {"ignore suffix after second", []uint{1, 0}, 1}, + {"out of range", []uint{99}, -1}, + {"large index", []uint{uint(uint64(1) << 32)}, -1}, + {"maximum index", []uint{^uint(0)}, -1}, + } { + t.Run(tc.name, func(t *testing.T) { + n := fixtureNearhead() + var dataCache cacheRpc.Interface = cacheRpc.NotFoundImpl + switch source { + case "nearhead": + block := fixtureBlock(100, common.HexToHash("0x1"), nil) + block.Traces = traces + require.NoError(t, n.Put(block)) + case "data-cache": + dataCache = &fixtureTraceCache{cacheRpc.NotFoundImpl} + } + calls := 0 + p := &fixtureProvider{call: func(_ context.Context, out interface{}, method string, args ...interface{}) error { + calls++ + require.Equal(t, "trace_get", method) + require.Equal(t, txHash, args[0]) + indices := args[1].([]hexutil.Uint) + require.Len(t, indices, len(tc.indexes)) + for i := range indices { + require.Equal(t, hexutil.Uint(tc.indexes[i]), indices[i]) + } + // Model the pinned node implementation, independently of TraceAddress. + var trace *wt.LocalizedTrace + if len(indices) > 0 && uint(indices[0]) < uint(len(traces)) { + trace = &traces[uint(indices[0])] + } + raw, err := json.Marshal(trace) + if err != nil { + return err + } + return json.Unmarshal(raw, out) + }} + c := traceCacheClient{RpcTraceClient: client.NewRpcTraceClient(p), cacheClientContext: cacheClientContext{nearhead: n, dataCache: dataCache, resolver: fixtureResolver{}}} + lazy, err := c.LazyTrace(txHash, tc.indexes) + require.NoError(t, err) + got, err := lazy.Load() + require.NoError(t, err) + if tc.want < 0 { + require.Nil(t, got) + } else { + require.NotNil(t, got) + require.Equal(t, traces[tc.want].TraceAddress, got.TraceAddress) + } + wantCalls := 0 + if len(tc.indexes) > 0 && (source == "fullnode" || tc.want < 0) { + wantCalls = 1 + } + require.Equal(t, wantCalls, calls) + }) + } + }) } } From 6bb4d29d0302af71b9b8b6997e0dc01938ce7995 Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 3 Nov 2026 15:00:33 +0800 Subject: [PATCH 19/26] fix(rpc): restore direct trace cache reads with index guards --- util/rpc/cached_client_eth.go | 26 ++++++++-------------- util/rpc/cached_client_eth_test.go | 35 ++++++++++++++++++++++++++++-- 2 files changed, 42 insertions(+), 19 deletions(-) diff --git a/util/rpc/cached_client_eth.go b/util/rpc/cached_client_eth.go index 74ef0ce3..6ea34a18 100644 --- a/util/rpc/cached_client_eth.go +++ b/util/rpc/cached_client_eth.go @@ -947,21 +947,21 @@ func (c traceCacheClient) LazyTrace( return readThroughLazyCacheChain( tracker, func() (res cacheTypes.Lazy[*web3Types.LocalizedTrace], err error) { - if trace := traceAtIndex(c.nearhead.GetTransactionTraces(transactionHash), index); trace != nil { + // The pinned cache converts index to int for its bounds check. + if index > uint(math.MaxInt) { + return res, nil + } + if trace := c.nearhead.GetTrace(transactionHash, index); trace != nil { res, err = cacheTypes.NewLazy(trace) } return res, err }, func() (cacheTypes.Lazy[*web3Types.LocalizedTrace], error) { - traces, err := c.dataCache.GetTransactionTraces(transactionHash) - if err != nil { - return cacheTypes.Lazy[*web3Types.LocalizedTrace]{}, err - } - loaded, err := traces.Load() - if err != nil { - return cacheTypes.Lazy[*web3Types.LocalizedTrace]{}, err + // The pinned gRPC client narrows index to uint32 on the wire. + if uint64(index) > math.MaxUint32 { + return cacheTypes.Lazy[*web3Types.LocalizedTrace]{}, nil } - return cacheTypes.NewLazy(traceAtIndex(loaded, index)) + return c.dataCache.GetTrace(transactionHash, index) }, func() (cacheTypes.Lazy[*web3Types.LocalizedTrace], error) { hexIndexes := make([]hexutil.Uint, len(indexes)) @@ -1359,11 +1359,3 @@ func readThroughLazyCacheChain[T any]( }, ) } - -// Keep the index unsigned so large inputs cannot wrap into a valid offset. -func traceAtIndex(traces []web3Types.LocalizedTrace, index uint) *web3Types.LocalizedTrace { - if index < uint(len(traces)) { - return &traces[index] - } - return nil -} diff --git a/util/rpc/cached_client_eth_test.go b/util/rpc/cached_client_eth_test.go index 67badfb4..70c94e70 100644 --- a/util/rpc/cached_client_eth_test.go +++ b/util/rpc/cached_client_eth_test.go @@ -3,6 +3,7 @@ package rpc import ( "context" "encoding/json" + "math" "math/big" "testing" @@ -71,9 +72,26 @@ func TestLargeTransactionIndexBypassesNarrowCache(t *testing.T) { require.Zero(t, d.calls) } -type fixtureTraceCache struct{ cacheRpc.Interface } +type fixtureTraceCache struct { + cacheRpc.Interface + indexes []uint + bulkCalls int +} + +func (c *fixtureTraceCache) GetTrace(_ common.Hash, index uint) (ct.Lazy[*wt.LocalizedTrace], error) { + c.indexes = append(c.indexes, index) + // Model the pinned gRPC client's uint32 wire conversion. The caller must + // skip this API when the original index does not fit. + index = uint(uint32(index)) + traces := fixtureTransactionTraces() + if index >= uint(len(traces)) { + return ct.NewLazy((*wt.LocalizedTrace)(nil)) + } + return ct.NewLazy(&traces[index]) +} func (c *fixtureTraceCache) GetTransactionTraces(common.Hash) (ct.Lazy[[]wt.LocalizedTrace], error) { + c.bulkCalls++ return ct.NewLazy(fixtureTransactionTraces()) } @@ -110,19 +128,24 @@ func TestTraceGetMatchesConfluxSemantics(t *testing.T) { {"ignore suffix two", []uint{0, 2}, 0}, {"ignore suffix after second", []uint{1, 0}, 1}, {"out of range", []uint{99}, -1}, + {"maximum protocol index", []uint{math.MaxUint32}, -1}, {"large index", []uint{uint(uint64(1) << 32)}, -1}, + {"maximum signed index", []uint{uint(math.MaxInt)}, -1}, + {"signed overflow", []uint{uint(math.MaxInt) + 1}, -1}, {"maximum index", []uint{^uint(0)}, -1}, } { t.Run(tc.name, func(t *testing.T) { n := fixtureNearhead() var dataCache cacheRpc.Interface = cacheRpc.NotFoundImpl + var traceCache *fixtureTraceCache switch source { case "nearhead": block := fixtureBlock(100, common.HexToHash("0x1"), nil) block.Traces = traces require.NoError(t, n.Put(block)) case "data-cache": - dataCache = &fixtureTraceCache{cacheRpc.NotFoundImpl} + traceCache = &fixtureTraceCache{Interface: cacheRpc.NotFoundImpl} + dataCache = traceCache } calls := 0 p := &fixtureProvider{call: func(_ context.Context, out interface{}, method string, args ...interface{}) error { @@ -161,6 +184,14 @@ func TestTraceGetMatchesConfluxSemantics(t *testing.T) { wantCalls = 1 } require.Equal(t, wantCalls, calls) + if traceCache != nil { + require.Zero(t, traceCache.bulkCalls, "trace_get must not fetch all transaction traces") + if len(tc.indexes) > 0 && uint64(tc.indexes[0]) <= math.MaxUint32 { + require.Equal(t, []uint{tc.indexes[0]}, traceCache.indexes) + } else { + require.Empty(t, traceCache.indexes, "unsafe index must bypass data-cache") + } + } }) } }) From d11629946a94c5ea751d9994456f14fb74e907c7 Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 3 Nov 2026 16:30:38 +0800 Subject: [PATCH 20/26] fix(rpc): delegate near-head cache internals to data-cache --- util/rpc/cached_client_eth.go | 64 +++++++------- util/rpc/cached_client_eth_test.go | 2 +- util/rpc/nearhead.go | 121 -------------------------- util/rpc/nearhead_test.go | 134 ++++++++++++++++------------- 4 files changed, 107 insertions(+), 214 deletions(-) delete mode 100644 util/rpc/nearhead.go diff --git a/util/rpc/cached_client_eth.go b/util/rpc/cached_client_eth.go index 6ea34a18..fed8a301 100644 --- a/util/rpc/cached_client_eth.go +++ b/util/rpc/cached_client_eth.go @@ -8,13 +8,11 @@ import ( "github.com/Conflux-Chain/confura-data-cache/nearhead" cacheRpc "github.com/Conflux-Chain/confura-data-cache/rpc" + cacheSync "github.com/Conflux-Chain/confura-data-cache/sync" cacheTypes "github.com/Conflux-Chain/confura-data-cache/types" "github.com/Conflux-Chain/confura/util" "github.com/Conflux-Chain/confura/util/metrics" lruCache "github.com/Conflux-Chain/confura/util/rpc/cache" - "github.com/Conflux-Chain/go-conflux-util/blockchain/sync/evm" - "github.com/Conflux-Chain/go-conflux-util/blockchain/sync/poll" - "github.com/Conflux-Chain/go-conflux-util/blockchain/sync/process" viperutil "github.com/Conflux-Chain/go-conflux-util/viper" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" @@ -54,12 +52,12 @@ const ( rpcMethodTraceBlock = "trace_block" ) +// EthNearHeadCache uses data-cache's readers unchanged. Cache-internal locking, +// topic filtering and returned-buffer ownership require fixes in that package. type EthNearHeadCache struct { *nearhead.EthCache - stopSync context.CancelFunc - syncDone sync.WaitGroup - snapshotMu sync.RWMutex - maxLogs int + stopSync context.CancelFunc + syncDone sync.WaitGroup } func newDummyEthNearHeadCache() *EthNearHeadCache { @@ -77,11 +75,8 @@ func (c *EthNearHeadCache) Close() { func NewEthNearHeadCacheFromViper(url string) (*EthNearHeadCache, error) { var conf struct { - Enabled bool - Adapter evm.AdapterConfig - Poller poll.Option - Writer nearhead.WriteOption - Cache nearhead.Config + Enabled bool + cacheSync.NearHeadConfig `mapstructure:",squash"` } if err := viperutil.UnmarshalKey("nearhead", &conf); err != nil { return nil, errors.WithMessage(err, "failed to unmarshal near head config") @@ -94,25 +89,16 @@ func NewEthNearHeadCacheFromViper(url string) (*EthNearHeadCache, error) { conf.Adapter.URL = url - adapter, err := evm.NewAdapterWithConfig(conf.Adapter) - if err != nil { - return nil, errors.WithMessage(err, "failed to create near head adapter") - } + // Keep polling, reorg writes and adapter ownership in data-cache. Internal + // cache fixes belong there; Confura only coordinates its full-node reads. ctx, cancel := context.WithCancel(context.Background()) - finalized, err := adapter.GetFinalizedBlockNumber(ctx) + cache := &EthNearHeadCache{stopSync: cancel} + ethCache, err := cacheSync.StartNearHead(ctx, &cache.syncDone, conf.NearHeadConfig) if err != nil { cancel() - adapter.Close() - return nil, err + return nil, errors.WithMessage(err, "failed to sync near head data") } - poller := poll.NewLatestPoller(adapter, finalized, poll.ReorgWindowParams{}, conf.Poller) - cache := &EthNearHeadCache{EthCache: nearhead.NewEthCache(conf.Cache), stopSync: cancel, maxLogs: conf.Cache.MaxLogs} - writer := &nearHeadWriter{cache: cache, writer: nearhead.NewWriter(cache.EthCache, conf.Writer)} - cache.syncDone.Add(2) - go poller.Poll(ctx, &cache.syncDone) - go process.Process(ctx, &cache.syncDone, poller.DataCh(), writer) - var once sync.Once - cache.stopSync = func() { once.Do(func() { cancel(); cache.syncDone.Wait(); adapter.Close() }) } + cache.EthCache = ethCache return cache, nil } @@ -691,7 +677,17 @@ func (c ethCacheClient) logsByBlockRange(filter web3Types.FilterQuery) (res []we return nil, tracker.withError(err) } - ethLogs, cachedHash, err := c.nearhead.logsSnapshot( + // Bracket the dependency's log read with cached-boundary checks. Its public + // API cannot return logs and a hash atomically. These checks detect observed + // changes, not ABA reorgs; an atomic snapshot requires a data-cache change. + boundary := c.nearhead.GetBlock(cacheTypes.BlockHashOrNumberWithNumber(toBlock), false) + cacheEnd := toBlock + if boundary.Coverage[0] < boundary.Coverage[1] && toBlock >= boundary.Coverage[1] { + cacheEnd = boundary.Coverage[1] - 1 + boundary = c.nearhead.GetBlock(cacheTypes.BlockHashOrNumberWithNumber(cacheEnd), false) + } + cachedBoundary := boundary.Result + ethLogs, err := c.nearhead.GetLogsByBlockRange( fromBlock, toBlock, nearhead.FilterOpt{ Addresses: filter.Addresses, @@ -718,8 +714,14 @@ func (c ethCacheClient) logsByBlockRange(filter web3Types.FilterQuery) (res []we return nil, tracker.withError(err) } - // A canonical hash at the upper bound commits to the entire requested - // ancestry. Check the cache boundary and fence all full-node range reads. + boundaryAfter := c.nearhead.GetBlock(cacheTypes.BlockHashOrNumberWithNumber(ethLogs.ToBlock), false).Result + if cachedBoundary == nil || ethLogs.ToBlock != cacheEnd || boundaryAfter == nil || boundaryAfter.Hash != cachedBoundary.Hash { + return nil, tracker.withError(errors.New("near head cache changed while reading logs; retry request")) + } + + // Confura stitches cache and full-node results, so it checks their boundary + // hashes and the node's upper-bound hash before and after stitching. This is + // not an atomic node snapshot and cannot detect ABA or inconsistent backends. anchor, err := c.RpcEthClient.BlockByNumber(web3Types.BlockNumber(toBlock), false) if err != nil { return nil, tracker.withError(err) @@ -734,7 +736,7 @@ func (c ethCacheClient) logsByBlockRange(filter web3Types.FilterQuery) (res []we return nil, tracker.withError(err) } } - if cacheAnchor == nil || cacheAnchor.Hash != cachedHash { + if cacheAnchor == nil || cacheAnchor.Hash != cachedBoundary.Hash { return nil, tracker.withError(errors.New("near head logs differ from canonical chain; retry request")) } diff --git a/util/rpc/cached_client_eth_test.go b/util/rpc/cached_client_eth_test.go index 70c94e70..0d6fa9ea 100644 --- a/util/rpc/cached_client_eth_test.go +++ b/util/rpc/cached_client_eth_test.go @@ -37,7 +37,7 @@ func (fixtureResolver) Resolve(n wt.BlockNumber) (uint64, error) { return 100, nil } func fixtureNearhead() *EthNearHeadCache { - return &EthNearHeadCache{maxLogs: 100, EthCache: nearhead.NewEthCache(nearhead.Config{MaxMemory: 1 << 20, MaxLogs: 100})} + return &EthNearHeadCache{EthCache: nearhead.NewEthCache(nearhead.Config{MaxMemory: 1 << 20, MaxLogs: 100})} } func fixtureBlock(n int64, h common.Hash, topics []common.Hash) evm.BlockData { tx := common.HexToHash("0x1") diff --git a/util/rpc/nearhead.go b/util/rpc/nearhead.go deleted file mode 100644 index eaf695e3..00000000 --- a/util/rpc/nearhead.go +++ /dev/null @@ -1,121 +0,0 @@ -package rpc - -import ( - "context" - "fmt" - - "github.com/Conflux-Chain/confura-data-cache/nearhead" - ct "github.com/Conflux-Chain/confura-data-cache/types" - "github.com/Conflux-Chain/confura/util" - "github.com/Conflux-Chain/go-conflux-util/blockchain/sync/evm" - "github.com/Conflux-Chain/go-conflux-util/blockchain/sync/poll" - "github.com/ethereum/go-ethereum/common" - wt "github.com/openweb3/web3go/types" -) - -// Serialize the dependency's Pop/Put pair with multi-block log snapshots. -// Its individual methods are safe, but do not provide a compound read view. -type nearHeadWriter struct { - cache *EthNearHeadCache - writer *nearhead.Writer -} - -func (w *nearHeadWriter) Process(ctx context.Context, data poll.Revertable[evm.BlockData]) { - w.cache.snapshotMu.Lock() - defer w.cache.snapshotMu.Unlock() - w.writer.Process(ctx, data) -} - -func (c *EthNearHeadCache) Put(data evm.BlockData) error { - c.snapshotMu.Lock() - defer c.snapshotMu.Unlock() - return c.EthCache.Put(data) -} -func (c *EthNearHeadCache) Pop(number uint64) int { - c.snapshotMu.Lock() - defer c.snapshotMu.Unlock() - return c.EthCache.Pop(number) -} - -func (c *EthNearHeadCache) GetLogsByBlockHash(hash common.Hash, filter nearhead.FilterOpt) (*nearhead.EthLogs, error) { - c.snapshotMu.RLock() - defer c.snapshotMu.RUnlock() - ref := ct.BlockHashOrNumberWithHash(hash) - block := c.EthCache.GetBlock(ref, false).Result - if block == nil { - return nil, nil - } - receipts := c.EthCache.GetBlockReceipts(ref).Result - if receipts == nil { - return nil, nil - } - logs, err := c.collectLogs(nil, receipts, filter) - if err != nil { - return nil, err - } - return &nearhead.EthLogs{FromBlock: block.Number.Uint64(), ToBlock: block.Number.Uint64(), Logs: logs}, nil -} - -func (c *EthNearHeadCache) logsSnapshot(from, to uint64, filter nearhead.FilterOpt) (*nearhead.EthLogs, common.Hash, error) { - c.snapshotMu.RLock() - defer c.snapshotMu.RUnlock() - if from > to { - return nil, common.Hash{}, fmt.Errorf("invalid block range params") - } - coverage := c.EthCache.GetBlock(ct.BlockHashOrNumberWithNumber(from), false).Coverage - if coverage[0] == coverage[1] || to < coverage[0] || from >= coverage[1] { - return nil, common.Hash{}, nil - } - from, to = max(from, coverage[0]), min(to, coverage[1]-1) - logs := make([]wt.Log, 0) - for number := from; number <= to; number++ { - receipts := c.EthCache.GetBlockReceipts(ct.BlockHashOrNumberWithNumber(number)).Result - if receipts == nil { - return nil, common.Hash{}, nil - } // receipts were not collected - var err error - logs, err = c.collectLogs(logs, receipts, filter) - if err != nil { - return nil, common.Hash{}, err - } - } - last := c.EthCache.GetBlock(ct.BlockHashOrNumberWithNumber(to), false).Result - if last == nil { - return nil, common.Hash{}, nil - } - return &nearhead.EthLogs{FromBlock: from, ToBlock: to, Logs: logs}, last.Hash, nil -} - -func (c *EthNearHeadCache) collectLogs(logs []wt.Log, receipts []*wt.Receipt, filter nearhead.FilterOpt) ([]wt.Log, error) { - if logs == nil { - logs = make([]wt.Log, 0) - } - for _, receipt := range receipts { - if receipt == nil { - return nil, fmt.Errorf("incomplete cached receipt") - } - for _, entry := range receipt.Logs { - if entry == nil { - return nil, fmt.Errorf("invalid cached log") - } - if !util.IncludeEthLogAddrs(entry, filter.Addresses) || !util.MatchEthLogTopics(entry, filter.Topics) { - continue - } - copied := *entry - copied.Topics = append([]common.Hash(nil), entry.Topics...) - copied.Data = append([]byte(nil), entry.Data...) - logs = append(logs, copied) - if len(logs) > c.maxLogs { - return nil, fmt.Errorf("the result set exceeds the max limit of %v logs, please narrow down your filter conditions", c.maxLogs) - } - } - } - return logs, nil -} - -// GetLogsByBlockRange returns a snapshot without using the dependency's topic -// matcher. A caller combining this with other sources must fence the chain view. -func (c *EthNearHeadCache) GetLogsByBlockRange(from, to uint64, filter nearhead.FilterOpt) (*nearhead.EthLogs, error) { - logs, _, err := c.logsSnapshot(from, to, filter) - return logs, err -} diff --git a/util/rpc/nearhead_test.go b/util/rpc/nearhead_test.go index d6968ae2..995af297 100644 --- a/util/rpc/nearhead_test.go +++ b/util/rpc/nearhead_test.go @@ -4,11 +4,8 @@ import ( "context" "encoding/json" "fmt" - "sync" "testing" - "time" - "github.com/Conflux-Chain/confura-data-cache/nearhead" cacheRpc "github.com/Conflux-Chain/confura-data-cache/rpc" "github.com/ethereum/go-ethereum/common" "github.com/openweb3/web3go/client" @@ -61,70 +58,85 @@ func TestLogsRejectReorgDuringStitch(t *testing.T) { } } -func TestNearHeadLogTopicsAndOwnership(t *testing.T) { - n := fixtureNearhead() - hash := common.HexToHash("0xa") - require.NoError(t, n.Put(fixtureBlock(100, hash, []common.Hash{hash}))) +// These tests cover Confura's cache/full-node boundary checks. They do not +// establish snapshot atomicity or repair data-cache's filtering and locking. +func TestLogsRangeBoundaryChecks(t *testing.T) { for _, tc := range []struct { - name string - topics [][]common.Hash - count int + name string + from, to wt.BlockNumber + stale bool + empty bool + nilHeader int + failedRPC int + wantHeaders int + wantReads int + wantLogs int + wantError string }{ - {"missing null position", [][]common.Hash{{hash}, nil}, 1}, - {"missing empty position", [][]common.Hash{{hash}, {}}, 1}, - {"missing required topic", [][]common.Hash{{hash}, {hash}}, 0}, - {"wildcard then missing required topic", [][]common.Hash{nil, {hash}}, 0}, + {name: "fully cached", from: 100, to: 100, wantHeaders: 2, wantLogs: 1}, + {name: "node suffix", from: 100, to: 101, wantHeaders: 3, wantReads: 1, wantLogs: 2}, + {name: "cache miss", from: 101, to: 101, wantReads: 1, wantLogs: 1}, + {name: "stale cache", from: 100, to: 100, stale: true, wantHeaders: 1, wantError: "differ from canonical"}, + {name: "stale empty logs", from: 100, to: 100, stale: true, empty: true, wantHeaders: 1, wantError: "differ from canonical"}, + {name: "missing anchor", from: 100, to: 100, nilHeader: 1, wantHeaders: 1, wantError: "anchor unavailable"}, + {name: "missing cache boundary", from: 100, to: 101, nilHeader: 2, wantHeaders: 2, wantError: "differ from canonical"}, + {name: "anchor RPC error", from: 100, to: 100, failedRPC: 1, wantHeaders: 1, wantError: "header RPC failed"}, + {name: "final RPC error", from: 100, to: 100, failedRPC: 2, wantHeaders: 2, wantError: "header RPC failed"}, + {name: "missing final header", from: 100, to: 100, nilHeader: 2, wantHeaders: 2, wantError: "chain changed"}, } { t.Run(tc.name, func(t *testing.T) { - logs, err := n.GetLogsByBlockHash(hash, nearhead.FilterOpt{Topics: tc.topics}) - require.NoError(t, err) - require.Len(t, logs.Logs, tc.count) - }) - } - logs, err := n.GetLogsByBlockHash(hash, nearhead.FilterOpt{}) - require.NoError(t, err) - require.Len(t, logs.Logs, 1) - logs.Logs[0].Topics[0] = common.Hash{} - again, err := n.GetLogsByBlockHash(hash, nearhead.FilterOpt{}) - require.NoError(t, err) - require.Equal(t, hash, again.Logs[0].Topics[0]) - n.maxLogs = 0 - _, err = n.GetLogsByBlockHash(hash, nearhead.FilterOpt{}) - require.Error(t, err) -} - -func TestNearHeadHashReadsAndWriterProgress(t *testing.T) { - n := fixtureNearhead() - hash := common.HexToHash("0xa") - require.NoError(t, n.Put(fixtureBlock(100, hash, nil))) - var wg sync.WaitGroup - for i := 0; i < 4; i++ { - wg.Add(1) - go func() { - defer wg.Done() - for j := 0; j < 100; j++ { - _, err := n.GetLogsByBlockHash(hash, nearhead.FilterOpt{}) + n := fixtureNearhead() + cachedHash := common.HexToHash("0xa100") + block := fixtureBlock(100, cachedHash, nil) + if tc.empty { + block.Receipts[0].Logs = nil + } + require.NoError(t, n.Put(block)) + headers, reads := 0, 0 + p := &fixtureProvider{call: func(_ context.Context, out interface{}, method string, args ...interface{}) error { + var data interface{} + switch method { + case "eth_getBlockByNumber": + headers++ + if headers == tc.failedRPC { + return fmt.Errorf("header RPC failed") + } + if headers != tc.nilHeader { + number := args[0].(wt.BlockNumber) + hash := common.HexToHash("0xa101") + if number == 100 { + hash = cachedHash + if tc.stale { + hash = common.HexToHash("0xb100") + } + } + header := fixtureBlock(int64(number), hash, nil).Block + header.Transactions = *wt.NewTxOrHashListByHashes(nil) + data = header + } + case "eth_getLogs": + reads++ + data = []wt.Log{{BlockNumber: 101, BlockHash: common.HexToHash("0xa101")}} + default: + t.Fatalf("unexpected method %s", method) + } + raw, err := json.Marshal(data) if err != nil { - t.Error(err) + return err } + return json.Unmarshal(raw, out) + }} + c := ethCacheClient{RpcEthClient: client.NewRpcEthClient(p), cacheClientContext: cacheClientContext{nearhead: n, dataCache: cacheRpc.NotFoundImpl, resolver: fixtureResolver{}}} + logs, err := c.Logs(wt.FilterQuery{FromBlock: &tc.from, ToBlock: &tc.to}) + if tc.wantError != "" { + require.ErrorContains(t, err, tc.wantError) + require.Nil(t, logs, "discard results when the boundary check fails") + } else { + require.NoError(t, err) + require.Len(t, logs, tc.wantLogs) } - }() - } - wg.Add(1) - go func() { - defer wg.Done() - for j := 0; j < 100; j++ { - n.Pop(100) - if err := n.Put(fixtureBlock(100, hash, nil)); err != nil { - t.Error(err) - } - } - }() - done := make(chan struct{}) - go func() { wg.Wait(); close(done) }() - select { - case <-done: - case <-time.After(3 * time.Second): - t.Fatal("cache read/write deadlock") + require.Equal(t, tc.wantHeaders, headers) + require.Equal(t, tc.wantReads, reads) + }) } } From 142e84fa820170b80fd8ee4bb830cf7814ebf46d Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 4 Nov 2026 10:40:46 +0800 Subject: [PATCH 21/26] fix(rpc): retry mixed near-head log reads within a local budget --- util/rpc/cached_client_eth.go | 213 +++++++++------ util/rpc/nearhead_test.go | 501 +++++++++++++++++++++++++++------- 2 files changed, 541 insertions(+), 173 deletions(-) diff --git a/util/rpc/cached_client_eth.go b/util/rpc/cached_client_eth.go index fed8a301..72c08595 100644 --- a/util/rpc/cached_client_eth.go +++ b/util/rpc/cached_client_eth.go @@ -5,6 +5,7 @@ import ( "math" "math/big" "sync" + "time" "github.com/Conflux-Chain/confura-data-cache/nearhead" cacheRpc "github.com/Conflux-Chain/confura-data-cache/rpc" @@ -27,6 +28,15 @@ import ( var ( _ util.EthBlockNumberResolver = (*ethBlockTagResolver)(nil) + + // ErrInconsistentCanonicalView means near-head and node log reads could not + // be aligned within the local retry budget. No partial result is returned. + ErrInconsistentCanonicalView = errors.New("inconsistent canonical view") +) + +const ( + nearHeadLogRetryTimeout = time.Second + nearHeadLogRetryBackoff = 10 * time.Millisecond ) const ( @@ -667,122 +677,165 @@ func (c ethCacheClient) logsByBlockHash(filter web3Types.FilterQuery) ([]web3Typ return logs, nil } -func (c ethCacheClient) logsByBlockRange(filter web3Types.FilterQuery) (res []web3Types.Log, err error) { +func (c ethCacheClient) logsByBlockRange(filter web3Types.FilterQuery) ([]web3Types.Log, error) { tracker := newRPCCacheTracker(rpcMethodEthLogsByBlockRange).withNearHeadHit(false).withFullNodeHit(false) defer tracker.report() fromBlock, toBlock, err := c.resolveBlockRange(filter) if err != nil { - err = errors.WithMessage(err, "failed to resolve block range") - return nil, tracker.withError(err) + return nil, tracker.withError(errors.WithMessage(err, "failed to resolve block range")) + } + // Resolve moving tags once. Every attempt, including a cache miss after a + // conflict, must query the same heights without changing the caller's filter. + from, to := web3Types.BlockNumber(fromBlock), web3Types.BlockNumber(toBlock) + filter.FromBlock, filter.ToBlock = &from, &to + + ctx := context.Background() + var cancel context.CancelFunc + defer func() { + if cancel != nil { + cancel() + } + }() + for { + logs, err := c.logsByBlockRangeOnce(ctx, filter, fromBlock, toBlock, tracker) + if ctx.Err() != nil { + return nil, tracker.withError(errors.WithMessage(ErrInconsistentCanonicalView, "near-head log retry budget exhausted")) + } + if !errors.Is(err, ErrInconsistentCanonicalView) { + return logs, tracker.withError(err) + } + if cancel == nil { + // RPC request context is not available here. Start one local budget + // on the first conflict and bind only this call's client copy. The + // pinned provider's sleeps/cache locks are not cancellable, so this + // bounds retries, not the entire function's wall-clock duration. + ctx, cancel = context.WithTimeout(ctx, nearHeadLogRetryTimeout) + eth := *c.RpcEthClient + eth.SetContext(ctx) + c.RpcEthClient = ð + } + timer := time.NewTimer(nearHeadLogRetryBackoff) + select { + case <-ctx.Done(): + timer.Stop() + return nil, tracker.withError(errors.WithMessage(ErrInconsistentCanonicalView, "near-head log retry budget exhausted")) + case <-timer.C: + } + if ctx.Err() != nil { + return nil, tracker.withError(errors.WithMessage(ErrInconsistentCanonicalView, "near-head log retry budget exhausted")) + } } +} - // Bracket the dependency's log read with cached-boundary checks. Its public - // API cannot return logs and a hash atomically. These checks detect observed - // changes, not ABA reorgs; an atomic snapshot requires a data-cache change. - boundary := c.nearhead.GetBlock(cacheTypes.BlockHashOrNumberWithNumber(toBlock), false) - cacheEnd := toBlock - if boundary.Coverage[0] < boundary.Coverage[1] && toBlock >= boundary.Coverage[1] { - cacheEnd = boundary.Coverage[1] - 1 - boundary = c.nearhead.GetBlock(cacheTypes.BlockHashOrNumberWithNumber(cacheEnd), false) +// logsByBlockRangeOnce owns all data for one attempt. A conflict discards both +// node segments and the cache result; no logs carry over to the next attempt. +func (c ethCacheClient) logsByBlockRangeOnce( + ctx context.Context, + filter web3Types.FilterQuery, + fromBlock, toBlock uint64, + tracker *rpcCacheTracker, +) ([]web3Types.Log, error) { + if err := ctx.Err(); err != nil { + return nil, err } - cachedBoundary := boundary.Result - ethLogs, err := c.nearhead.GetLogsByBlockRange( - fromBlock, toBlock, - nearhead.FilterOpt{ - Addresses: filter.Addresses, - Topics: filter.Topics, - }, - ) + opt := nearhead.FilterOpt{Addresses: filter.Addresses, Topics: filter.Topics} + ethLogs, err := c.nearhead.GetLogsByBlockRange(fromBlock, toBlock, opt) if err != nil { - err = errors.WithMessage(err, "failed to get logs from near head cache") - return nil, tracker.withError(err) + return nil, errors.WithMessage(err, "failed to get logs from near head cache") } - if ethLogs == nil { - logs, err := c.RpcEthClient.Logs(filter) - if err != nil { - return nil, tracker.withError(err) - } - tracker.withFullNodeHit(true) - return logs, nil + return c.fetchLogsFromFullNode(ctx, filter, fromBlock, toBlock) } - - if ethLogs.FromBlock < fromBlock || ethLogs.ToBlock > toBlock { - err = errors.New("bad event logs from near head cache: unexpected block range") - return nil, tracker.withError(err) + if ethLogs.FromBlock < fromBlock || ethLogs.ToBlock > toBlock || ethLogs.FromBlock > ethLogs.ToBlock { + return nil, errors.New("bad event logs from near head cache: unexpected block range") } - - boundaryAfter := c.nearhead.GetBlock(cacheTypes.BlockHashOrNumberWithNumber(ethLogs.ToBlock), false).Result - if cachedBoundary == nil || ethLogs.ToBlock != cacheEnd || boundaryAfter == nil || boundaryAfter.Hash != cachedBoundary.Hash { - return nil, tracker.withError(errors.New("near head cache changed while reading logs; retry request")) + tracker.withNearHeadHit(true) + if ethLogs.FromBlock == fromBlock && ethLogs.ToBlock == toBlock { + // One cache read supplies the entire result. No node canonical lookup + // is needed; this returns the cache's current view, even if it lags. + return ethLogs.Logs, nil } - // Confura stitches cache and full-node results, so it checks their boundary - // hashes and the node's upper-bound hash before and after stitching. This is - // not an atomic node snapshot and cannot detect ABA or inconsistent backends. - anchor, err := c.RpcEthClient.BlockByNumber(web3Types.BlockNumber(toBlock), false) - if err != nil { - return nil, tracker.withError(err) - } - if anchor == nil { - return nil, tracker.withError(errors.New("log range anchor unavailable")) + // Only mixed reads need a boundary. Re-read logs between two cache header + // reads because data-cache cannot return logs and their end hash atomically. + // This detects observed changes, not ABA; an atomic snapshot belongs upstream. + end := ethLogs.ToBlock + ref := cacheTypes.BlockHashOrNumberWithNumber(end) + before := c.nearhead.GetBlock(ref, false).Result + if before == nil || before.Hash == (common.Hash{}) { + return nil, errors.WithMessage(ErrInconsistentCanonicalView, "near-head boundary unavailable") } - cacheAnchor := anchor - if ethLogs.ToBlock != toBlock { - cacheAnchor, err = c.RpcEthClient.BlockByNumber(web3Types.BlockNumber(ethLogs.ToBlock), false) - if err != nil { - return nil, tracker.withError(err) - } + snapshot, err := c.nearhead.GetLogsByBlockRange(fromBlock, toBlock, opt) + if err != nil { + return nil, errors.WithMessage(err, "failed to get logs from near head cache") } - if cacheAnchor == nil || cacheAnchor.Hash != cachedBoundary.Hash { - return nil, tracker.withError(errors.New("near head logs differ from canonical chain; retry request")) + after := c.nearhead.GetBlock(ref, false).Result + if snapshot == nil || snapshot.FromBlock != ethLogs.FromBlock || snapshot.ToBlock != end || + after == nil || after.Hash != before.Hash { + return nil, errors.WithMessage(ErrInconsistentCanonicalView, "near-head cache changed while reading logs") } - tracker.withNearHeadHit(true) - - // Stitches together logs from the near head cache and full node. - // Handle the block range before the near head cached range. - if fromBlock < ethLogs.FromBlock { - logs, err := c.fetchLogsFromFullNode(filter, fromBlock, ethLogs.FromBlock-1) + // A block hash commits to its ancestry. For an internally coherent cache + // and node view, matching the cached end aligns both the older node prefix + // and the newer node suffix. Check that same height around BOTH node reads; + // no start/parent-hash or query-upper-bound lookup is needed here. This is + // optimistic alignment, not a node snapshot or protection against ABA or + // independently inconsistent responses within the node's Logs RPC. + if err := c.checkNearHeadLogBoundary(ctx, end, before.Hash); err != nil { + return nil, err + } + logs := make([]web3Types.Log, 0, len(snapshot.Logs)) + if fromBlock < snapshot.FromBlock { + tracker.withFullNodeHit(true) + prefix, err := c.fetchLogsFromFullNode(ctx, filter, fromBlock, snapshot.FromBlock-1) if err != nil { - return nil, tracker.withError(err) + return nil, err } - - res = append(logs, res...) - tracker.withFullNodeHit(true) + logs = append(logs, prefix...) } - - // Append logs from the near head cache. - res = append(res, ethLogs.Logs...) - - // Handle the block range after the near head cached range. - if toBlock > ethLogs.ToBlock { - logs, err := c.fetchLogsFromFullNode(filter, ethLogs.ToBlock+1, toBlock) + logs = append(logs, snapshot.Logs...) + if toBlock > end { + tracker.withFullNodeHit(true) + suffix, err := c.fetchLogsFromFullNode(ctx, filter, end+1, toBlock) if err != nil { - return nil, tracker.withError(err) + return nil, err } - - res = append(res, logs...) - tracker.withFullNodeHit(true) + logs = append(logs, suffix...) } + if err := c.checkNearHeadLogBoundary(ctx, end, before.Hash); err != nil { + return nil, err + } + return logs, nil +} - after, err := c.RpcEthClient.BlockByNumber(web3Types.BlockNumber(toBlock), false) +func (c ethCacheClient) checkNearHeadLogBoundary(ctx context.Context, number uint64, hash common.Hash) error { + if err := ctx.Err(); err != nil { + return err + } + block, err := c.RpcEthClient.BlockByNumber(web3Types.BlockNumber(number), false) if err != nil { - return nil, tracker.withError(err) + return err } - if after == nil || after.Hash != anchor.Hash { - return nil, tracker.withError(errors.New("chain changed while reading logs; retry request")) + if err := ctx.Err(); err != nil { + return err } - - return res, nil + if block == nil || block.Number == nil || !block.Number.IsUint64() || + block.Number.Uint64() != number || block.Hash != hash { + return errors.WithMessage(ErrInconsistentCanonicalView, "node boundary differs from near-head cache") + } + return nil } func (c ethCacheClient) fetchLogsFromFullNode( + ctx context.Context, filter web3Types.FilterQuery, from, to uint64, ) ([]web3Types.Log, error) { + if err := ctx.Err(); err != nil { + return nil, err + } fromBlock := web3Types.BlockNumber(from) toBlock := web3Types.BlockNumber(to) filter.FromBlock = &fromBlock diff --git a/util/rpc/nearhead_test.go b/util/rpc/nearhead_test.go index 995af297..b2c52271 100644 --- a/util/rpc/nearhead_test.go +++ b/util/rpc/nearhead_test.go @@ -4,7 +4,9 @@ import ( "context" "encoding/json" "fmt" + "sync" "testing" + "time" cacheRpc "github.com/Conflux-Chain/confura-data-cache/rpc" "github.com/ethereum/go-ethereum/common" @@ -13,130 +15,443 @@ import ( "github.com/stretchr/testify/require" ) -func TestLogsRejectReorgDuringStitch(t *testing.T) { - for _, reorg := range []bool{false, true} { - t.Run(fmt.Sprint(reorg), func(t *testing.T) { +func nearHeadTestReply(out, value interface{}) error { + raw, err := json.Marshal(value) + if err != nil { + return err + } + return json.Unmarshal(raw, out) +} + +func nearHeadTestHeader(number int64, hash common.Hash) *wt.Block { + block := fixtureBlock(number, hash, nil).Block + block.Transactions = *wt.NewTxOrHashListByHashes(nil) + return block +} + +func nearHeadTestClient(n *EthNearHeadCache, p *fixtureProvider) ethCacheClient { + return ethCacheClient{RpcEthClient: client.NewRpcEthClient(p), cacheClientContext: cacheClientContext{nearhead: n, dataCache: cacheRpc.NotFoundImpl, resolver: fixtureResolver{}}} +} + +// A complete cache hit must not depend on a live node. Mixed reads check only +// the cached end, even when the node supplies both a prefix and a suffix. +func TestNearHeadLogsCoverage(t *testing.T) { + for _, tc := range []struct { + name string + from, to wt.BlockNumber + empty bool + events []string + blocks []uint64 + }{ + {name: "complete", from: 100, to: 100, blocks: []uint64{100}}, + {name: "complete empty", from: 100, to: 100, empty: true}, + {name: "miss", from: 101, to: 101, events: []string{"logs:101-101"}, blocks: []uint64{101}}, + {name: "left", from: 99, to: 100, events: []string{"header:100", "logs:99-99", "header:100"}, blocks: []uint64{99, 100}}, + {name: "right", from: 100, to: 101, events: []string{"header:100", "logs:101-101", "header:100"}, blocks: []uint64{100, 101}}, + {name: "both", from: 99, to: 101, events: []string{"header:100", "logs:99-99", "logs:101-101", "header:100"}, blocks: []uint64{99, 100, 101}}, + {name: "both empty", from: 99, to: 101, empty: true, events: []string{"header:100", "logs:99-99", "logs:101-101", "header:100"}}, + } { + t.Run(tc.name, func(t *testing.T) { + n := fixtureNearhead() + hash := common.HexToHash("0xa100") + block := fixtureBlock(100, hash, nil) + if tc.empty { + block.Receipts[0].Logs = nil + } + require.NoError(t, n.Put(block)) + var events []string + p := &fixtureProvider{call: func(_ context.Context, out interface{}, method string, args ...interface{}) error { + switch method { + case "eth_getBlockByNumber": + number := args[0].(wt.BlockNumber) + require.Equal(t, wt.BlockNumber(100), number, "no start, parent, or request-upper-bound lookup") + events = append(events, fmt.Sprintf("header:%d", number)) + return nearHeadTestReply(out, nearHeadTestHeader(100, hash)) + case "eth_getLogs": + fq := args[0].(wt.FilterQuery) + events = append(events, fmt.Sprintf("logs:%d-%d", *fq.FromBlock, *fq.ToBlock)) + logs := []wt.Log{} + if !tc.empty { + logs = append(logs, wt.Log{BlockNumber: uint64(*fq.FromBlock)}) + } + return nearHeadTestReply(out, logs) + default: + t.Fatalf("unexpected RPC %s", method) + return nil + } + }} + logs, err := nearHeadTestClient(n, p).Logs(wt.FilterQuery{FromBlock: &tc.from, ToBlock: &tc.to}) + require.NoError(t, err) + require.NotNil(t, logs) + var blocks []uint64 + for _, entry := range logs { + blocks = append(blocks, entry.BlockNumber) + } + require.Equal(t, tc.blocks, blocks) + require.Equal(t, tc.events, events) + }) + } +} + +// Reorg either node segment between the two boundary checks. A fresh attempt +// must replace every old log, including a prefix already read before the reorg. +func TestNearHeadLogsRetryWholeStitch(t *testing.T) { + for _, reorgAt := range []uint64{99, 101} { + t.Run(fmt.Sprint(reorgAt), func(t *testing.T) { n := fixtureNearhead() a, b := common.HexToHash("0xa100"), common.HexToHash("0xb100") require.NoError(t, n.Put(fixtureBlock(100, a, nil))) - canonical := a - p := &fixtureProvider{call: func(_ context.Context, out interface{}, method string, args ...interface{}) error { - var data interface{} + canonical, switched, headers := a, false, 0 + var reads []uint64 + p := &fixtureProvider{call: func(ctx context.Context, out interface{}, method string, args ...interface{}) error { switch method { case "eth_getBlockByNumber": - block := fixtureBlock(100, canonical, nil).Block - block.Transactions = *wt.NewTxOrHashListByHashes(nil) - data = block + headers++ + require.Equal(t, wt.BlockNumber(100), args[0]) + if headers > 2 { + _, bounded := ctx.Deadline() + require.True(t, bounded, "retry RPCs share the internal budget") + } + return nearHeadTestReply(out, nearHeadTestHeader(100, canonical)) case "eth_getLogs": - if reorg { - canonical = b + fq := args[0].(wt.FilterQuery) + number := uint64(*fq.FromBlock) + reads = append(reads, number) + servedHash := canonical + if number == reorgAt && !switched { + switched, canonical = true, b require.Equal(t, 1, n.Pop(100)) require.NoError(t, n.Put(fixtureBlock(100, b, nil))) } - data = []wt.Log{{BlockNumber: 99, BlockHash: common.HexToHash("0x99")}} + return nearHeadTestReply(out, []wt.Log{{BlockNumber: number, BlockHash: servedHash}}) default: - t.Fatalf("unexpected method %s", method) + t.Fatalf("unexpected RPC %s", method) + return nil + } + }} + from, to := wt.BlockNumber(99), wt.BlockNumber(101) + logs, err := nearHeadTestClient(n, p).Logs(wt.FilterQuery{FromBlock: &from, ToBlock: &to}) + require.NoError(t, err) + require.Len(t, logs, 3) + for i, entry := range logs { + require.Equal(t, uint64(99+i), entry.BlockNumber) + require.Equal(t, b, entry.BlockHash, "discard the failed attempt's logs") + } + require.Equal(t, 4, headers) + require.Equal(t, []uint64{99, 101, 99, 101}, reads) + }) + } +} + +func TestNearHeadLogsRetryBudget(t *testing.T) { + for _, lateSuccess := range []bool{false, true} { + t.Run(fmt.Sprint(lateSuccess), func(t *testing.T) { + n := fixtureNearhead() + hash := common.HexToHash("0xa100") + require.NoError(t, n.Put(fixtureBlock(100, hash, nil))) + var deadline time.Time + headers := 0 + p := &fixtureProvider{call: func(ctx context.Context, out interface{}, method string, args ...interface{}) error { + require.Equal(t, "eth_getBlockByNumber", method) + headers++ + if headers == 1 { + _, bounded := ctx.Deadline() + require.False(t, bounded, "budget starts only on the first conflict") + return nearHeadTestReply(out, nil) + } + current, bounded := ctx.Deadline() + require.True(t, bounded) + if deadline.IsZero() { + deadline = current + } else { + require.Equal(t, deadline, current, "conflicts must not reset the deadline") } - raw, err := json.Marshal(data) - if err != nil { - return err + if headers < 4 { + return nearHeadTestReply(out, nil) } - return json.Unmarshal(raw, out) + <-ctx.Done() + if lateSuccess { + return nearHeadTestReply(out, nearHeadTestHeader(100, hash)) + } + return ctx.Err() }} - c := ethCacheClient{RpcEthClient: client.NewRpcEthClient(p), cacheClientContext: cacheClientContext{nearhead: n, dataCache: cacheRpc.NotFoundImpl, resolver: fixtureResolver{}}} from, to := wt.BlockNumber(99), wt.BlockNumber(100) - logs, err := c.Logs(wt.FilterQuery{FromBlock: &from, ToBlock: &to}) - if reorg { - require.ErrorContains(t, err, "chain changed") - require.Nil(t, logs) + started := time.Now() + logs, err := nearHeadTestClient(n, p).Logs(wt.FilterQuery{FromBlock: &from, ToBlock: &to}) + require.ErrorIs(t, err, ErrInconsistentCanonicalView) + require.Nil(t, logs) + require.Equal(t, 4, headers) + require.WithinDuration(t, started.Add(time.Second), deadline, 100*time.Millisecond) + require.GreaterOrEqual(t, time.Since(started), time.Second) + require.Less(t, time.Since(started), 3*time.Second, "fixture RPCs obey the deadline") + }) + } +} + +func TestNearHeadLogsPreserveRPCErrors(t *testing.T) { + failure := fmt.Errorf("node unavailable") + for _, failAt := range []string{"before", "logs", "after"} { + t.Run(failAt, func(t *testing.T) { + n := fixtureNearhead() + hash := common.HexToHash("0xa100") + require.NoError(t, n.Put(fixtureBlock(100, hash, nil))) + headers, reads := 0, 0 + p := &fixtureProvider{call: func(_ context.Context, out interface{}, method string, args ...interface{}) error { + if method == "eth_getLogs" { + reads++ + if failAt == "logs" { + return failure + } + return nearHeadTestReply(out, []wt.Log{{BlockNumber: 99}}) + } + require.Equal(t, "eth_getBlockByNumber", method) + headers++ + if headers == 1 && failAt == "before" || headers == 2 && failAt == "after" { + return failure + } + return nearHeadTestReply(out, nearHeadTestHeader(100, hash)) + }} + from, to := wt.BlockNumber(99), wt.BlockNumber(100) + logs, err := nearHeadTestClient(n, p).Logs(wt.FilterQuery{FromBlock: &from, ToBlock: &to}) + require.ErrorIs(t, err, failure) + require.Nil(t, logs) + if failAt == "before" { + require.Equal(t, 1, headers) + require.Zero(t, reads) } else { - require.NoError(t, err) - require.Len(t, logs, 2) - require.Equal(t, a, logs[1].BlockHash) + require.Equal(t, 1, reads) } }) } } -// These tests cover Confura's cache/full-node boundary checks. They do not -// establish snapshot atomicity or repair data-cache's filtering and locking. -func TestLogsRangeBoundaryChecks(t *testing.T) { - for _, tc := range []struct { - name string - from, to wt.BlockNumber - stale bool - empty bool - nilHeader int - failedRPC int - wantHeaders int - wantReads int - wantLogs int - wantError string - }{ - {name: "fully cached", from: 100, to: 100, wantHeaders: 2, wantLogs: 1}, - {name: "node suffix", from: 100, to: 101, wantHeaders: 3, wantReads: 1, wantLogs: 2}, - {name: "cache miss", from: 101, to: 101, wantReads: 1, wantLogs: 1}, - {name: "stale cache", from: 100, to: 100, stale: true, wantHeaders: 1, wantError: "differ from canonical"}, - {name: "stale empty logs", from: 100, to: 100, stale: true, empty: true, wantHeaders: 1, wantError: "differ from canonical"}, - {name: "missing anchor", from: 100, to: 100, nilHeader: 1, wantHeaders: 1, wantError: "anchor unavailable"}, - {name: "missing cache boundary", from: 100, to: 101, nilHeader: 2, wantHeaders: 2, wantError: "differ from canonical"}, - {name: "anchor RPC error", from: 100, to: 100, failedRPC: 1, wantHeaders: 1, wantError: "header RPC failed"}, - {name: "final RPC error", from: 100, to: 100, failedRPC: 2, wantHeaders: 2, wantError: "header RPC failed"}, - {name: "missing final header", from: 100, to: 100, nilHeader: 2, wantHeaders: 2, wantError: "chain changed"}, - } { - t.Run(tc.name, func(t *testing.T) { - n := fixtureNearhead() - cachedHash := common.HexToHash("0xa100") - block := fixtureBlock(100, cachedHash, nil) - if tc.empty { - block.Receipts[0].Logs = nil +// Hold one request inside its retry RPC while a second request uses the same +// original client. The second request must not inherit the first's deadline. +func TestNearHeadLogsRetryContextIsolation(t *testing.T) { + n := fixtureNearhead() + hash := common.HexToHash("0xa100") + require.NoError(t, n.Put(fixtureBlock(100, hash, nil))) + entered, release := make(chan struct{}), make(chan struct{}) + var releaseOnce sync.Once + t.Cleanup(func() { releaseOnce.Do(func() { close(release) }) }) + headers := 0 + p := &fixtureProvider{call: func(ctx context.Context, out interface{}, method string, args ...interface{}) error { + if method == "eth_getLogs" { + fq := args[0].(wt.FilterQuery) + if *fq.FromBlock == 102 { + if _, bounded := ctx.Deadline(); bounded { + return fmt.Errorf("shared client context was mutated") + } + } + return nearHeadTestReply(out, []wt.Log{}) + } + headers++ + if headers == 1 { + return nearHeadTestReply(out, nil) + } + if headers == 2 { + if _, bounded := ctx.Deadline(); !bounded { + return fmt.Errorf("retry has no deadline") } + close(entered) + select { + case <-release: + case <-ctx.Done(): + return ctx.Err() + } + } + return nearHeadTestReply(out, nearHeadTestHeader(100, hash)) + }} + c := nearHeadTestClient(n, p) + done := make(chan error, 1) + go func() { + from, to := wt.BlockNumber(99), wt.BlockNumber(100) + _, err := c.Logs(wt.FilterQuery{FromBlock: &from, ToBlock: &to}) + done <- err + }() + select { + case <-entered: + case <-time.After(3 * time.Second): + t.Fatal("retry did not reach the controlled RPC") + } + from, to := wt.BlockNumber(102), wt.BlockNumber(102) + _, err := c.Logs(wt.FilterQuery{FromBlock: &from, ToBlock: &to}) + require.NoError(t, err) + releaseOnce.Do(func() { close(release) }) + select { + case err := <-done: + require.NoError(t, err) + case <-time.After(3 * time.Second): + t.Fatal("retry did not finish after release") + } +} + +func TestNearHeadLogsRetryInvalidBoundary(t *testing.T) { + for _, invalid := range []string{"missing", "hash", "number"} { + t.Run(invalid, func(t *testing.T) { + n := fixtureNearhead() + hash := common.HexToHash("0xa100") + block := fixtureBlock(100, hash, nil) + block.Receipts[0].Logs = nil // Empty cached results still need alignment. require.NoError(t, n.Put(block)) headers, reads := 0, 0 p := &fixtureProvider{call: func(_ context.Context, out interface{}, method string, args ...interface{}) error { - var data interface{} - switch method { - case "eth_getBlockByNumber": + if method == "eth_getLogs" { + reads++ + return nearHeadTestReply(out, []wt.Log{}) + } + require.Equal(t, "eth_getBlockByNumber", method) + headers++ + block := nearHeadTestHeader(100, hash) + if headers == 1 { + switch invalid { + case "missing": + block = nil + case "hash": + block.Hash = common.HexToHash("0xb100") + case "number": + block.Number.SetInt64(101) + } + } + return nearHeadTestReply(out, block) + }} + from, to := wt.BlockNumber(99), wt.BlockNumber(100) + logs, err := nearHeadTestClient(n, p).Logs(wt.FilterQuery{FromBlock: &from, ToBlock: &to}) + require.NoError(t, err) + require.Empty(t, logs) + require.NotNil(t, logs) + require.Equal(t, 3, headers) + require.Equal(t, 1, reads) + }) + } +} + +type nearHeadTestResolver struct { + calls, latest int +} + +func (r *nearHeadTestResolver) Resolve(number wt.BlockNumber) (uint64, error) { + r.calls++ + if number >= 0 { + return uint64(number), nil + } + r.latest++ + return uint64(99 + r.latest), nil +} + +func TestNearHeadLogsRetryFreezesLatest(t *testing.T) { + n := fixtureNearhead() + hash := common.HexToHash("0xa100") + require.NoError(t, n.Put(fixtureBlock(100, hash, nil))) + headers := 0 + p := &fixtureProvider{call: func(_ context.Context, out interface{}, method string, args ...interface{}) error { + if method == "eth_getBlockByNumber" { + require.Equal(t, wt.BlockNumber(100), args[0]) + headers++ + if headers == 1 { + return nearHeadTestReply(out, nil) + } + return nearHeadTestReply(out, nearHeadTestHeader(100, hash)) + } + require.Equal(t, "eth_getLogs", method) + fq := args[0].(wt.FilterQuery) + require.Equal(t, wt.BlockNumber(99), *fq.FromBlock) + require.Equal(t, wt.BlockNumber(99), *fq.ToBlock) + return nearHeadTestReply(out, []wt.Log{}) + }} + c := nearHeadTestClient(n, p) + resolver := &nearHeadTestResolver{} + c.resolver = resolver + from, to := wt.BlockNumber(99), wt.LatestBlockNumber + filter := wt.FilterQuery{FromBlock: &from, ToBlock: &to} + _, err := c.Logs(filter) + require.NoError(t, err) + require.Equal(t, 2, resolver.calls) + require.Equal(t, 1, resolver.latest) + require.Equal(t, wt.LatestBlockNumber, *filter.ToBlock) + require.Equal(t, wt.BlockNumber(99), *filter.FromBlock) +} + +func TestNearHeadLogsRetryCoverageChange(t *testing.T) { + for _, complete := range []bool{false, true} { + t.Run(fmt.Sprint(complete), func(t *testing.T) { + n := fixtureNearhead() + hash := common.HexToHash("0xa100") + require.NoError(t, n.Put(fixtureBlock(100, hash, nil))) + headers, reads := 0, 0 + p := &fixtureProvider{call: func(ctx context.Context, out interface{}, method string, args ...interface{}) error { + if method == "eth_getBlockByNumber" { headers++ - if headers == tc.failedRPC { - return fmt.Errorf("header RPC failed") + if headers != 1 { + return fmt.Errorf("new single-source attempt must not check headers") } - if headers != tc.nilHeader { - number := args[0].(wt.BlockNumber) - hash := common.HexToHash("0xa101") - if number == 100 { - hash = cachedHash - if tc.stale { - hash = common.HexToHash("0xb100") - } - } - header := fixtureBlock(int64(number), hash, nil).Block - header.Transactions = *wt.NewTxOrHashListByHashes(nil) - data = header + require.Equal(t, 1, n.Pop(100)) + if complete { + require.NoError(t, n.Put(fixtureBlock(99, hash, nil))) + require.NoError(t, n.Put(fixtureBlock(100, hash, nil))) } - case "eth_getLogs": - reads++ - data = []wt.Log{{BlockNumber: 101, BlockHash: common.HexToHash("0xa101")}} - default: - t.Fatalf("unexpected method %s", method) - } - raw, err := json.Marshal(data) - if err != nil { - return err + return nearHeadTestReply(out, nil) } - return json.Unmarshal(raw, out) + require.Equal(t, "eth_getLogs", method) + reads++ + fq := args[0].(wt.FilterQuery) + require.Equal(t, wt.BlockNumber(99), *fq.FromBlock) + require.Equal(t, wt.BlockNumber(100), *fq.ToBlock) + _, bounded := ctx.Deadline() + require.True(t, bounded) + return nearHeadTestReply(out, []wt.Log{}) }} - c := ethCacheClient{RpcEthClient: client.NewRpcEthClient(p), cacheClientContext: cacheClientContext{nearhead: n, dataCache: cacheRpc.NotFoundImpl, resolver: fixtureResolver{}}} - logs, err := c.Logs(wt.FilterQuery{FromBlock: &tc.from, ToBlock: &tc.to}) - if tc.wantError != "" { - require.ErrorContains(t, err, tc.wantError) - require.Nil(t, logs, "discard results when the boundary check fails") + from, to := wt.BlockNumber(99), wt.BlockNumber(100) + logs, err := nearHeadTestClient(n, p).Logs(wt.FilterQuery{FromBlock: &from, ToBlock: &to}) + require.NoError(t, err) + require.Equal(t, 1, headers) + if complete { + require.Len(t, logs, 2) + require.Zero(t, reads) } else { - require.NoError(t, err) - require.Len(t, logs, tc.wantLogs) + require.Empty(t, logs) + require.Equal(t, 1, reads) } - require.Equal(t, tc.wantHeaders, headers) - require.Equal(t, tc.wantReads, reads) }) } } + +func TestNearHeadLogsUsesCachedEnd(t *testing.T) { + n := fixtureNearhead() + first, last := common.HexToHash("0xa100"), common.HexToHash("0xa101") + topic := common.HexToHash("0x1234") + address := common.HexToAddress("0xabcd") + for i, hash := range []common.Hash{first, last} { + block := fixtureBlock(int64(100+i), hash, []common.Hash{topic}) + block.Block.ParentHash = common.HexToHash("0xa99") + if i > 0 { + block.Block.ParentHash = first + } + block.Receipts[0].Logs[0].Address = address + require.NoError(t, n.Put(block)) + } + headers := 0 + p := &fixtureProvider{call: func(_ context.Context, out interface{}, method string, args ...interface{}) error { + if method == "eth_getBlockByNumber" { + headers++ + require.Equal(t, wt.BlockNumber(101), args[0], "use the cached end, not its start or the requested upper bound") + return nearHeadTestReply(out, nearHeadTestHeader(101, last)) + } + require.Equal(t, "eth_getLogs", method) + fq := args[0].(wt.FilterQuery) + require.Equal(t, []common.Address{address}, fq.Addresses) + require.Equal(t, [][]common.Hash{{topic}}, fq.Topics) + return nearHeadTestReply(out, []wt.Log{{BlockNumber: uint64(*fq.FromBlock)}}) + }} + from, to := wt.BlockNumber(99), wt.BlockNumber(102) + filter := wt.FilterQuery{FromBlock: &from, ToBlock: &to, Addresses: []common.Address{address}, Topics: [][]common.Hash{{topic}}} + logs, err := nearHeadTestClient(n, p).Logs(filter) + require.NoError(t, err) + require.Equal(t, 2, headers) + require.Len(t, logs, 4) + for i, entry := range logs { + require.Equal(t, uint64(99+i), entry.BlockNumber) + } + require.Equal(t, wt.BlockNumber(99), *filter.FromBlock) + require.Equal(t, wt.BlockNumber(102), *filter.ToBlock) +} From cc69d043ecb6f6291cbfd3a93f5d554b5a15b911 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 4 Nov 2026 14:39:30 +0800 Subject: [PATCH 22/26] fix(rpc): simplify near-head log reads and retry admission --- util/rpc/cached_client_eth.go | 191 ++++++++++++++---------- util/rpc/nearhead_test.go | 267 +++++++++++++++++++++++++++++----- 2 files changed, 341 insertions(+), 117 deletions(-) diff --git a/util/rpc/cached_client_eth.go b/util/rpc/cached_client_eth.go index 72c08595..a7606b55 100644 --- a/util/rpc/cached_client_eth.go +++ b/util/rpc/cached_client_eth.go @@ -690,92 +690,133 @@ func (c ethCacheClient) logsByBlockRange(filter web3Types.FilterQuery) ([]web3Ty from, to := web3Types.BlockNumber(fromBlock), web3Types.BlockNumber(toBlock) filter.FromBlock, filter.ToBlock = &from, &to - ctx := context.Background() - var cancel context.CancelFunc - defer func() { - if cancel != nil { - cancel() - } - }() - for { - logs, err := c.logsByBlockRangeOnce(ctx, filter, fromBlock, toBlock, tracker) - if ctx.Err() != nil { - return nil, tracker.withError(errors.WithMessage(ErrInconsistentCanonicalView, "near-head log retry budget exhausted")) - } - if !errors.Is(err, ErrInconsistentCanonicalView) { + // The local window controls admission to another attempt, not RPC duration. + // Keep the configured client context/timeouts. An admitted attempt may finish + // after this window and still return a coherent result or its original error. + var deadline time.Time + started := time.Now() + for attempts := 1; ; attempts++ { + logs, err := c.logsByBlockRangeOnce(filter, fromBlock, toBlock, tracker) + conflict, ok := err.(*nearHeadLogConflict) + if !ok { + if err == nil && attempts > 1 { + logrus.WithField("attempts", attempts).Debug("Near-head log consistency recovered") + } return logs, tracker.withError(err) } - if cancel == nil { - // RPC request context is not available here. Start one local budget - // on the first conflict and bind only this call's client copy. The - // pinned provider's sleeps/cache locks are not cancellable, so this - // bounds retries, not the entire function's wall-clock duration. - ctx, cancel = context.WithTimeout(ctx, nearHeadLogRetryTimeout) - eth := *c.RpcEthClient - eth.SetContext(ctx) - c.RpcEthClient = ð + if deadline.IsZero() { + deadline = time.Now().Add(nearHeadLogRetryTimeout) } - timer := time.NewTimer(nearHeadLogRetryBackoff) - select { - case <-ctx.Done(): - timer.Stop() - return nil, tracker.withError(errors.WithMessage(ErrInconsistentCanonicalView, "near-head log retry budget exhausted")) - case <-timer.C: + logger := logrus.WithFields(logrus.Fields{ + "fromBlock": fromBlock, + "toBlock": toBlock, + "boundary": conflict.boundary, + "reason": conflict.reason, + "attempts": attempts, + "elapsed": time.Since(started), + }) + if remaining := time.Until(deadline); remaining > 0 { + logger.Debug("Retrying near-head logs after canonical view conflict") + time.Sleep(min(nearHeadLogRetryBackoff, remaining)) } - if ctx.Err() != nil { - return nil, tracker.withError(errors.WithMessage(ErrInconsistentCanonicalView, "near-head log retry budget exhausted")) + if !time.Now().Before(deadline) { + logger.WithField("elapsed", time.Since(started)).Info("Near-head log consistency retry window exhausted") + return nil, tracker.withError(ErrInconsistentCanonicalView) } } } +// nearHeadLogConflict carries fixed diagnostic reasons only. It is consumed by +// the retry loop; clients receive only ErrInconsistentCanonicalView on exhaustion. +type nearHeadLogConflict struct { + reason string + boundary uint64 +} + +func (e *nearHeadLogConflict) Error() string { return e.reason } + +// nearHeadLogReader keeps the metadata/log read sequence explicit and testable. +// Cache-internal snapshot and ownership guarantees remain data-cache's concern. +type nearHeadLogReader interface { + GetBlock(cacheTypes.BlockHashOrNumber, bool) nearhead.EthCacheResultWithCoverage[*web3Types.Block] + GetLogsByBlockRange(uint64, uint64, nearhead.FilterOpt) (*nearhead.EthLogs, error) +} + +// readNearHeadLogs scans logs at most once. A lightweight metadata lookup +// captures the cached end first, so a partial hit needs no second log scan. +func readNearHeadLogs( + cache nearHeadLogReader, + fromBlock, toBlock uint64, + opt nearhead.FilterOpt, +) (*nearhead.EthLogs, common.Hash, error) { + if fromBlock > toBlock { + return nil, common.Hash{}, errors.New("failed to get logs from near head cache: invalid block range params") + } + // In the pinned cache, true returns the cached block directly; false copies + // it and builds a transaction-hash list. We only read metadata, never mutate. + probe := cache.GetBlock(cacheTypes.BlockHashOrNumberWithNumber(toBlock), true) + start, end := probe.Coverage[0], probe.Coverage[1] // end is exclusive + if start >= end || toBlock < start || fromBlock >= end { + return nil, common.Hash{}, nil + } + end = min(toBlock, end-1) + ref := cacheTypes.BlockHashOrNumberWithNumber(end) + before := probe.Result + if end != toBlock { + before = cache.GetBlock(ref, true).Result + } + // Copy the hash before the scan; the cache returns a shared block pointer. + var hash common.Hash + if before != nil { + hash = before.Hash + } + snapshot, err := cache.GetLogsByBlockRange(fromBlock, toBlock, opt) + if err != nil { + return nil, common.Hash{}, errors.WithMessage(err, "failed to get logs from near head cache") + } + if snapshot == nil { + return nil, common.Hash{}, &nearHeadLogConflict{"near-head coverage disappeared", end} + } + if snapshot.FromBlock < fromBlock || snapshot.ToBlock > toBlock || snapshot.FromBlock > snapshot.ToBlock { + return nil, common.Hash{}, errors.New("bad event logs from near head cache: unexpected block range") + } + if snapshot.FromBlock == fromBlock && snapshot.ToBlock == toBlock { + // A complete result has one source and needs no boundary validation. + return snapshot, common.Hash{}, nil + } + if before == nil || hash == (common.Hash{}) { + return nil, common.Hash{}, &nearHeadLogConflict{"near-head boundary unavailable", end} + } + after := cache.GetBlock(ref, true).Result + if snapshot.ToBlock != end || after == nil || after.Hash != hash { + return nil, common.Hash{}, &nearHeadLogConflict{"near-head cache changed while reading logs", end} + } + // Start eviction can extend the node prefix. The actual returned range is + // authoritative as long as its end still matches the captured boundary. + return snapshot, hash, nil +} + // logsByBlockRangeOnce owns all data for one attempt. A conflict discards both // node segments and the cache result; no logs carry over to the next attempt. func (c ethCacheClient) logsByBlockRangeOnce( - ctx context.Context, filter web3Types.FilterQuery, fromBlock, toBlock uint64, tracker *rpcCacheTracker, ) ([]web3Types.Log, error) { - if err := ctx.Err(); err != nil { - return nil, err - } opt := nearhead.FilterOpt{Addresses: filter.Addresses, Topics: filter.Topics} - ethLogs, err := c.nearhead.GetLogsByBlockRange(fromBlock, toBlock, opt) + snapshot, hash, err := readNearHeadLogs(c.nearhead, fromBlock, toBlock, opt) if err != nil { - return nil, errors.WithMessage(err, "failed to get logs from near head cache") + return nil, err } - if ethLogs == nil { + if snapshot == nil { tracker.withFullNodeHit(true) - return c.fetchLogsFromFullNode(ctx, filter, fromBlock, toBlock) - } - if ethLogs.FromBlock < fromBlock || ethLogs.ToBlock > toBlock || ethLogs.FromBlock > ethLogs.ToBlock { - return nil, errors.New("bad event logs from near head cache: unexpected block range") + return c.fetchLogsFromFullNode(filter, fromBlock, toBlock) } tracker.withNearHeadHit(true) - if ethLogs.FromBlock == fromBlock && ethLogs.ToBlock == toBlock { - // One cache read supplies the entire result. No node canonical lookup - // is needed; this returns the cache's current view, even if it lags. - return ethLogs.Logs, nil - } - - // Only mixed reads need a boundary. Re-read logs between two cache header - // reads because data-cache cannot return logs and their end hash atomically. - // This detects observed changes, not ABA; an atomic snapshot belongs upstream. - end := ethLogs.ToBlock - ref := cacheTypes.BlockHashOrNumberWithNumber(end) - before := c.nearhead.GetBlock(ref, false).Result - if before == nil || before.Hash == (common.Hash{}) { - return nil, errors.WithMessage(ErrInconsistentCanonicalView, "near-head boundary unavailable") - } - snapshot, err := c.nearhead.GetLogsByBlockRange(fromBlock, toBlock, opt) - if err != nil { - return nil, errors.WithMessage(err, "failed to get logs from near head cache") - } - after := c.nearhead.GetBlock(ref, false).Result - if snapshot == nil || snapshot.FromBlock != ethLogs.FromBlock || snapshot.ToBlock != end || - after == nil || after.Hash != before.Hash { - return nil, errors.WithMessage(ErrInconsistentCanonicalView, "near-head cache changed while reading logs") + if snapshot.FromBlock == fromBlock && snapshot.ToBlock == toBlock { + return snapshot.Logs, nil } + end := snapshot.ToBlock // A block hash commits to its ancestry. For an internally coherent cache // and node view, matching the cached end aligns both the older node prefix @@ -783,13 +824,13 @@ func (c ethCacheClient) logsByBlockRangeOnce( // no start/parent-hash or query-upper-bound lookup is needed here. This is // optimistic alignment, not a node snapshot or protection against ABA or // independently inconsistent responses within the node's Logs RPC. - if err := c.checkNearHeadLogBoundary(ctx, end, before.Hash); err != nil { + if err := c.checkNearHeadLogBoundary(end, hash); err != nil { return nil, err } logs := make([]web3Types.Log, 0, len(snapshot.Logs)) if fromBlock < snapshot.FromBlock { tracker.withFullNodeHit(true) - prefix, err := c.fetchLogsFromFullNode(ctx, filter, fromBlock, snapshot.FromBlock-1) + prefix, err := c.fetchLogsFromFullNode(filter, fromBlock, snapshot.FromBlock-1) if err != nil { return nil, err } @@ -798,44 +839,34 @@ func (c ethCacheClient) logsByBlockRangeOnce( logs = append(logs, snapshot.Logs...) if toBlock > end { tracker.withFullNodeHit(true) - suffix, err := c.fetchLogsFromFullNode(ctx, filter, end+1, toBlock) + suffix, err := c.fetchLogsFromFullNode(filter, end+1, toBlock) if err != nil { return nil, err } logs = append(logs, suffix...) } - if err := c.checkNearHeadLogBoundary(ctx, end, before.Hash); err != nil { + if err := c.checkNearHeadLogBoundary(end, hash); err != nil { return nil, err } return logs, nil } -func (c ethCacheClient) checkNearHeadLogBoundary(ctx context.Context, number uint64, hash common.Hash) error { - if err := ctx.Err(); err != nil { - return err - } +func (c ethCacheClient) checkNearHeadLogBoundary(number uint64, hash common.Hash) error { block, err := c.RpcEthClient.BlockByNumber(web3Types.BlockNumber(number), false) if err != nil { return err } - if err := ctx.Err(); err != nil { - return err - } if block == nil || block.Number == nil || !block.Number.IsUint64() || block.Number.Uint64() != number || block.Hash != hash { - return errors.WithMessage(ErrInconsistentCanonicalView, "node boundary differs from near-head cache") + return &nearHeadLogConflict{"node boundary differs from near-head cache", number} } return nil } func (c ethCacheClient) fetchLogsFromFullNode( - ctx context.Context, filter web3Types.FilterQuery, from, to uint64, ) ([]web3Types.Log, error) { - if err := ctx.Err(); err != nil { - return nil, err - } fromBlock := web3Types.BlockNumber(from) toBlock := web3Types.BlockNumber(to) filter.FromBlock = &fromBlock diff --git a/util/rpc/nearhead_test.go b/util/rpc/nearhead_test.go index b2c52271..3995b3e8 100644 --- a/util/rpc/nearhead_test.go +++ b/util/rpc/nearhead_test.go @@ -1,6 +1,7 @@ package rpc import ( + "bytes" "context" "encoding/json" "fmt" @@ -8,10 +9,13 @@ import ( "testing" "time" + "github.com/Conflux-Chain/confura-data-cache/nearhead" cacheRpc "github.com/Conflux-Chain/confura-data-cache/rpc" + cacheTypes "github.com/Conflux-Chain/confura-data-cache/types" "github.com/ethereum/go-ethereum/common" "github.com/openweb3/web3go/client" wt "github.com/openweb3/web3go/types" + "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" ) @@ -108,10 +112,8 @@ func TestNearHeadLogsRetryWholeStitch(t *testing.T) { case "eth_getBlockByNumber": headers++ require.Equal(t, wt.BlockNumber(100), args[0]) - if headers > 2 { - _, bounded := ctx.Deadline() - require.True(t, bounded, "retry RPCs share the internal budget") - } + _, bounded := ctx.Deadline() + require.False(t, bounded, "keep the original client context on every attempt") return nearHeadTestReply(out, nearHeadTestHeader(100, canonical)) case "eth_getLogs": fq := args[0].(wt.FilterQuery) @@ -143,47 +145,82 @@ func TestNearHeadLogsRetryWholeStitch(t *testing.T) { } } +// The retry window admits attempts; it does not cancel an RPC or reject a +// coherent result that completes after the window. Late ordinary errors survive. func TestNearHeadLogsRetryBudget(t *testing.T) { - for _, lateSuccess := range []bool{false, true} { - t.Run(fmt.Sprint(lateSuccess), func(t *testing.T) { + for _, outcome := range []string{"conflict", "success", "RPC error"} { + t.Run(outcome, func(t *testing.T) { + logger := logrus.StandardLogger() + output, level := logger.Out, logger.GetLevel() + var diagnostics bytes.Buffer + logger.SetOutput(&diagnostics) + logger.SetLevel(logrus.InfoLevel) + t.Cleanup(func() { logger.SetOutput(output); logger.SetLevel(level) }) + n := fixtureNearhead() hash := common.HexToHash("0xa100") require.NoError(t, n.Put(fixtureBlock(100, hash, nil))) - var deadline time.Time - headers := 0 + var firstConflict time.Time + headers, reads := 0, 0 + failure := fmt.Errorf("node unavailable") p := &fixtureProvider{call: func(ctx context.Context, out interface{}, method string, args ...interface{}) error { + _, bounded := ctx.Deadline() + require.False(t, bounded, "do not replace configured RPC context/timeouts") + if method == "eth_getLogs" { + reads++ + return nearHeadTestReply(out, []wt.Log{}) + } require.Equal(t, "eth_getBlockByNumber", method) headers++ if headers == 1 { - _, bounded := ctx.Deadline() - require.False(t, bounded, "budget starts only on the first conflict") + firstConflict = time.Now() return nearHeadTestReply(out, nil) } - current, bounded := ctx.Deadline() - require.True(t, bounded) - if deadline.IsZero() { - deadline = current - } else { - require.Equal(t, deadline, current, "conflicts must not reset the deadline") - } if headers < 4 { return nearHeadTestReply(out, nil) } - <-ctx.Done() - if lateSuccess { - return nearHeadTestReply(out, nearHeadTestHeader(100, hash)) + if headers == 4 { + require.Less(t, time.Since(firstConflict), nearHeadLogRetryTimeout, "admit the round inside the window") + time.Sleep(time.Until(firstConflict.Add(nearHeadLogRetryTimeout + 50*time.Millisecond))) + switch outcome { + case "conflict": + return nearHeadTestReply(out, nil) + case "RPC error": + return failure + } } - return ctx.Err() + return nearHeadTestReply(out, nearHeadTestHeader(100, hash)) }} from, to := wt.BlockNumber(99), wt.BlockNumber(100) - started := time.Now() - logs, err := nearHeadTestClient(n, p).Logs(wt.FilterQuery{FromBlock: &from, ToBlock: &to}) - require.ErrorIs(t, err, ErrInconsistentCanonicalView) - require.Nil(t, logs) - require.Equal(t, 4, headers) - require.WithinDuration(t, started.Add(time.Second), deadline, 100*time.Millisecond) - require.GreaterOrEqual(t, time.Since(started), time.Second) - require.Less(t, time.Since(started), 3*time.Second, "fixture RPCs obey the deadline") + c := nearHeadTestClient(n, p) + c.nodeName = "https://secret@node.invalid/private-api-key" + logs, err := c.Logs(wt.FilterQuery{FromBlock: &from, ToBlock: &to}) + require.GreaterOrEqual(t, time.Since(firstConflict), nearHeadLogRetryTimeout) + switch outcome { + case "conflict": + require.Same(t, ErrInconsistentCanonicalView, err) + require.EqualError(t, err, "inconsistent canonical view") + require.Nil(t, logs) + require.Equal(t, 4, headers, "no new round after expiry") + require.Zero(t, reads) + require.Equal(t, 1, bytes.Count(diagnostics.Bytes(), []byte("level=info"))) + require.Contains(t, diagnostics.String(), "node boundary differs from near-head cache") + require.Contains(t, diagnostics.String(), "boundary=100") + case "success": + require.NoError(t, err) + require.Len(t, logs, 1) + require.Equal(t, 5, headers) + require.Equal(t, 1, reads) + require.Empty(t, diagnostics.String()) + case "RPC error": + require.ErrorIs(t, err, failure) + require.Nil(t, logs) + require.Equal(t, 4, headers) + require.Zero(t, reads) + require.Empty(t, diagnostics.String()) + } + require.NotContains(t, diagnostics.String(), "private-api-key") + require.NotContains(t, diagnostics.String(), "secret") }) } } @@ -226,7 +263,7 @@ func TestNearHeadLogsPreserveRPCErrors(t *testing.T) { } // Hold one request inside its retry RPC while a second request uses the same -// original client. The second request must not inherit the first's deadline. +// original client. Both requests must retain the configured context/deadline. func TestNearHeadLogsRetryContextIsolation(t *testing.T) { n := fixtureNearhead() hash := common.HexToHash("0xa100") @@ -235,13 +272,14 @@ func TestNearHeadLogsRetryContextIsolation(t *testing.T) { var releaseOnce sync.Once t.Cleanup(func() { releaseOnce.Do(func() { close(release) }) }) headers := 0 + configuredCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() p := &fixtureProvider{call: func(ctx context.Context, out interface{}, method string, args ...interface{}) error { + require.Equal(t, configuredCtx, ctx) if method == "eth_getLogs" { fq := args[0].(wt.FilterQuery) if *fq.FromBlock == 102 { - if _, bounded := ctx.Deadline(); bounded { - return fmt.Errorf("shared client context was mutated") - } + require.Equal(t, configuredCtx, ctx, "shared client context was mutated") } return nearHeadTestReply(out, []wt.Log{}) } @@ -250,9 +288,7 @@ func TestNearHeadLogsRetryContextIsolation(t *testing.T) { return nearHeadTestReply(out, nil) } if headers == 2 { - if _, bounded := ctx.Deadline(); !bounded { - return fmt.Errorf("retry has no deadline") - } + require.Equal(t, configuredCtx, ctx, "retry context must stay unchanged") close(entered) select { case <-release: @@ -263,6 +299,8 @@ func TestNearHeadLogsRetryContextIsolation(t *testing.T) { return nearHeadTestReply(out, nearHeadTestHeader(100, hash)) }} c := nearHeadTestClient(n, p) + c.RpcEthClient.SetContext(configuredCtx) + originalClient := c.RpcEthClient done := make(chan error, 1) go func() { from, to := wt.BlockNumber(99), wt.BlockNumber(100) @@ -277,6 +315,7 @@ func TestNearHeadLogsRetryContextIsolation(t *testing.T) { from, to := wt.BlockNumber(102), wt.BlockNumber(102) _, err := c.Logs(wt.FilterQuery{FromBlock: &from, ToBlock: &to}) require.NoError(t, err) + require.Same(t, originalClient, c.RpcEthClient) releaseOnce.Do(func() { close(release) }) select { case err := <-done: @@ -398,7 +437,7 @@ func TestNearHeadLogsRetryCoverageChange(t *testing.T) { require.Equal(t, wt.BlockNumber(99), *fq.FromBlock) require.Equal(t, wt.BlockNumber(100), *fq.ToBlock) _, bounded := ctx.Deadline() - require.True(t, bounded) + require.False(t, bounded) return nearHeadTestReply(out, []wt.Log{}) }} from, to := wt.BlockNumber(99), wt.BlockNumber(100) @@ -455,3 +494,157 @@ func TestNearHeadLogsUsesCachedEnd(t *testing.T) { require.Equal(t, wt.BlockNumber(99), *filter.FromBlock) require.Equal(t, wt.BlockNumber(102), *filter.ToBlock) } + +// Wrap the real cache only at call boundaries, so mutations occur at controlled +// points without relying on scheduling or changing the dependency's readers. +type nearHeadCountingReader struct { + *EthNearHeadCache + t *testing.T + blocks int + scans int + beforeScan func() + afterScan func(*nearhead.EthLogs) + missingAt int + scanError error + wantOpt nearhead.FilterOpt +} + +func (r *nearHeadCountingReader) GetBlock(ref cacheTypes.BlockHashOrNumber, full bool) nearhead.EthCacheResultWithCoverage[*wt.Block] { + r.blocks++ + require.True(r.t, full, "metadata lookup must not build transaction hash lists") + res := r.EthNearHeadCache.GetBlock(ref, full) + if r.blocks == r.missingAt { + res.Result = nil + } + return res +} + +func (r *nearHeadCountingReader) GetLogsByBlockRange(from, to uint64, opt nearhead.FilterOpt) (*nearhead.EthLogs, error) { + r.scans++ + require.Equal(r.t, r.wantOpt, opt) + if r.beforeScan != nil { + r.beforeScan() + } + if r.scanError != nil { + return nil, r.scanError + } + res, err := r.EthNearHeadCache.GetLogsByBlockRange(from, to, opt) + if r.afterScan != nil { + r.afterScan(res) + } + return res, err +} + +func TestReadNearHeadLogsSingleScan(t *testing.T) { + a, b := common.HexToHash("0xa101"), common.HexToHash("0xb101") + for _, tc := range []struct { + name string + from, to uint64 + wantBlocks int + wantScans int + wantFrom uint64 + wantTo uint64 + conflict bool + }{ + {name: "complete", from: 100, to: 101, wantBlocks: 1, wantScans: 1, wantFrom: 100, wantTo: 101}, + {name: "empty complete", from: 100, to: 101, wantBlocks: 1, wantScans: 1, wantFrom: 100, wantTo: 101}, + {name: "miss", from: 102, to: 103, wantBlocks: 1}, + {name: "left", from: 99, to: 101, wantBlocks: 2, wantScans: 1, wantFrom: 100, wantTo: 101}, + {name: "right", from: 100, to: 102, wantBlocks: 3, wantScans: 1, wantFrom: 100, wantTo: 101}, + {name: "both", from: 99, to: 102, wantBlocks: 3, wantScans: 1, wantFrom: 100, wantTo: 101}, + {name: "disappeared", from: 99, to: 101, wantBlocks: 1, wantScans: 1, conflict: true}, + {name: "reorg before scan", from: 99, to: 101, wantBlocks: 2, wantScans: 1, conflict: true}, + {name: "reorg after scan", from: 99, to: 101, wantBlocks: 2, wantScans: 1, conflict: true}, + {name: "end expanded", from: 99, to: 102, wantBlocks: 3, wantScans: 1, conflict: true}, + {name: "expanded to complete", from: 100, to: 102, wantBlocks: 2, wantScans: 1, wantFrom: 100, wantTo: 102}, + {name: "start evicted", from: 99, to: 101, wantBlocks: 2, wantScans: 1, wantFrom: 101, wantTo: 101}, + {name: "missing before", from: 99, to: 101, wantBlocks: 1, wantScans: 1, conflict: true}, + {name: "missing after", from: 99, to: 101, wantBlocks: 2, wantScans: 1, conflict: true}, + {name: "complete missing metadata", from: 100, to: 101, wantBlocks: 1, wantScans: 1, wantFrom: 100, wantTo: 101}, + } { + t.Run(tc.name, func(t *testing.T) { + n := fixtureNearhead() + first, last := fixtureBlock(100, common.HexToHash("0xa100"), nil), fixtureBlock(101, a, nil) + if tc.name == "start evicted" { + n.EthCache = nearhead.NewEthCache(nearhead.Config{MaxMemory: uint64(first.Size() + last.Size()), MaxLogs: 100}) + } + if tc.name == "empty complete" { + first.Receipts[0].Logs, last.Receipts[0].Logs = nil, nil + } + require.NoError(t, n.Put(first)) + require.NoError(t, n.Put(last)) + r := &nearHeadCountingReader{EthNearHeadCache: n, t: t} + reorg := func() { + require.Equal(t, 1, n.Pop(101)) + require.NoError(t, n.Put(fixtureBlock(101, b, nil))) + } + switch tc.name { + case "disappeared": + r.beforeScan = func() { require.Equal(t, 2, n.Pop(100)) } + case "reorg before scan": + r.beforeScan = reorg + case "reorg after scan": + r.afterScan = func(*nearhead.EthLogs) { reorg() } + case "end expanded", "expanded to complete", "start evicted": + r.beforeScan = func() { require.NoError(t, n.Put(fixtureBlock(102, common.HexToHash("0xa102"), nil))) } + case "missing before", "complete missing metadata": + r.missingAt = 1 + case "missing after": + r.missingAt = 2 + } + snapshot, hash, err := readNearHeadLogs(r, tc.from, tc.to, nearhead.FilterOpt{}) + require.Equal(t, tc.wantBlocks, r.blocks) + require.Equal(t, tc.wantScans, r.scans) + if tc.conflict { + require.IsType(t, &nearHeadLogConflict{}, err) + require.Nil(t, snapshot, "discard the scanned logs on conflict") + return + } + require.NoError(t, err) + if tc.wantScans == 0 { + require.Nil(t, snapshot) + return + } + require.Equal(t, tc.wantFrom, snapshot.FromBlock) + require.Equal(t, tc.wantTo, snapshot.ToBlock) + if tc.name == "empty complete" { + require.Empty(t, snapshot.Logs) + } + if tc.wantFrom != tc.from || tc.wantTo != tc.to { + require.Equal(t, a, hash) + } else { + require.Zero(t, hash, "complete cache result needs no boundary") + } + }) + } +} + +func TestReadNearHeadLogsErrorsAndFilter(t *testing.T) { + n := fixtureNearhead() + topic := common.HexToHash("0x1234") + address := common.HexToAddress("0xabcd") + block := fixtureBlock(100, common.HexToHash("0xa100"), []common.Hash{topic}) + block.Receipts[0].Logs[0].Address = address + require.NoError(t, n.Put(block)) + opt := nearhead.FilterOpt{Addresses: []common.Address{address}, Topics: [][]common.Hash{{topic}}} + r := &nearHeadCountingReader{EthNearHeadCache: n, t: t, wantOpt: opt} + snapshot, _, err := readNearHeadLogs(r, 99, 100, opt) + require.NoError(t, err) + require.Len(t, snapshot.Logs, 1) + require.Equal(t, 1, r.scans) + + failure := fmt.Errorf("cache unavailable") + r.scanError = failure + snapshot, _, err = readNearHeadLogs(r, 99, 100, opt) + require.ErrorIs(t, err, failure) + require.Nil(t, snapshot) + r.scanError = nil + r.afterScan = func(res *nearhead.EthLogs) { res.FromBlock = 98 } + snapshot, _, err = readNearHeadLogs(r, 99, 100, opt) + require.EqualError(t, err, "bad event logs from near head cache: unexpected block range") + require.Nil(t, snapshot) + scans := r.scans + _, _, err = readNearHeadLogs(r, 101, 100, opt) + require.EqualError(t, err, "failed to get logs from near head cache: invalid block range params") + require.Equal(t, scans, r.scans) +} From 774862b24f3a8241eab2316a3017ee4dcb4cbfe2 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 4 Nov 2026 16:07:59 +0800 Subject: [PATCH 23/26] perf(metrics): classify methods without decoding request payloads --- util/rpc/middlewares/metrics.go | 8 +- util/rpc/middlewares/metrics_test.go | 164 +++++++++++++++++++++++++++ 2 files changed, 170 insertions(+), 2 deletions(-) diff --git a/util/rpc/middlewares/metrics.go b/util/rpc/middlewares/metrics.go index 0ed768bd..aa351a55 100644 --- a/util/rpc/middlewares/metrics.go +++ b/util/rpc/middlewares/metrics.go @@ -26,10 +26,10 @@ func Metrics(next rpc.HandleCallMsgFunc) rpc.HandleCallMsgFunc { start := time.Now() resp := next(ctx, msg) - metricMethod := registeredMetricMethod(ctx, msg) if resp == nil { return nil } + metricMethod := registeredMetricMethod(ctx, msg) // collect rpc QPS/latency, response size etc. space, _ := handlers.GetNamespaceFromContext(ctx) @@ -79,7 +79,11 @@ func registeredMetricMethod(ctx context.Context, msg *rpc.JsonRpcMessage) string return msg.Method } if h, ok := ctx.Value("handler").(*rpc.ConnHandler); ok { - _, err := h.ParseCallArguments(msg) + // The provider checks registration before parsing arguments. Probe with + // no params so classification never decodes the request payload again. + probe := *msg + probe.Params = nil + _, err := h.ParseCallArguments(&probe) if err == nil { return msg.Method } diff --git a/util/rpc/middlewares/metrics_test.go b/util/rpc/middlewares/metrics_test.go index e53dc46d..263bd71f 100644 --- a/util/rpc/middlewares/metrics_test.go +++ b/util/rpc/middlewares/metrics_test.go @@ -1,19 +1,71 @@ package middlewares import ( + "bytes" "context" + "encoding/json" + "errors" "fmt" "net/http/httptest" "strings" + "sync/atomic" "testing" + "github.com/ethereum/go-ethereum/common/hexutil" rpc "github.com/openweb3/go-rpc-provider" + web3Types "github.com/openweb3/web3go/types" + "github.com/stretchr/testify/require" ) type MetricsFixture struct{} func (*MetricsFixture) Ping() bool { return true } +var metricArgumentDecodes atomic.Int32 + +type MetricArgument string + +func (a *MetricArgument) UnmarshalJSON(data []byte) error { + metricArgumentDecodes.Add(1) + return json.Unmarshal(data, (*string)(a)) +} + +func (*MetricsFixture) Required(MetricArgument) bool { return true } +func (*MetricsFixture) Optional(*MetricArgument) bool { return true } +func (*MetricsFixture) RawTransaction(hexutil.Bytes) bool { return true } +func (*MetricsFixture) Call(web3Types.CallRequest) bool { return true } + +func newMetricsFixtureServer(tb testing.TB) *rpc.Server { + tb.Helper() + server := rpc.NewServer() + tb.Cleanup(server.Stop) + if err := server.RegisterName("fixture", &MetricsFixture{}); err != nil { + tb.Fatal(err) + } + return server +} + +// Capture a real provider handler without opening a listening socket. Each +// global hook is gated by a unique context key and ignores subsequent tests. +func metricsFixtureContext(tb testing.TB) context.Context { + tb.Helper() + key := new(int) + captured := make(chan context.Context, 1) + rpc.HookHandleCallMsg(func(next rpc.HandleCallMsgFunc) rpc.HandleCallMsgFunc { + return func(ctx context.Context, msg *rpc.JsonRpcMessage) *rpc.JsonRpcMessage { + if ctx.Value(key) != nil { + captured <- ctx + } + return next(ctx, msg) + } + }) + server := newMetricsFixtureServer(tb) + req := httptest.NewRequest("POST", "/", strings.NewReader(`{"jsonrpc":"2.0","id":1,"method":"fixture_ping"}`)) + req.Header.Set("Content-Type", "application/json") + server.ServeHTTP(httptest.NewRecorder(), req.WithContext(context.WithValue(req.Context(), key, true))) + return <-captured +} + func TestMetricMethodUsesRegisteredCallbacks(t *testing.T) { // The provider exposes global hooks; scope the test hook to a context key. type fixtureKey struct{} @@ -43,3 +95,115 @@ func TestMetricMethodUsesRegisteredCallbacks(t *testing.T) { server.ServeHTTP(httptest.NewRecorder(), req) } } + +func TestMetricMethodDoesNotDecodeParameters(t *testing.T) { + ctx := metricsFixtureContext(t) + for _, tc := range []struct { + method, params, want string + }{ + {"fixture_required", `["payload"]`, "fixture_required"}, + {"fixture_required", `[123]`, "fixture_required"}, + {"fixture_required", `not JSON`, "fixture_required"}, + {"fixture_required", `[]`, "fixture_required"}, + {"fixture_optional", `["payload"]`, "fixture_optional"}, + {"fixture_optional", `[]`, "fixture_optional"}, + {"fixture_ping", `[]`, "fixture_ping"}, + {"fixture_unknown", `["payload"]`, "method_not_found"}, + {"eth_subscribe", `["logs",{}]`, "eth_subscribe"}, + {"eth_unsubscribe", `["0x1"]`, "eth_unsubscribe"}, + {"cfx_subscribe", `["logs",{}]`, "cfx_subscribe"}, + {"cfx_unsubscribe", `["0x1"]`, "cfx_unsubscribe"}, + } { + t.Run(tc.method+"/"+tc.params, func(t *testing.T) { + msg := &rpc.JsonRpcMessage{Version: "2.0", ID: json.RawMessage(`1`), Method: tc.method, Params: json.RawMessage(tc.params)} + original := *msg + original.Params = bytes.Clone(msg.Params) + metricArgumentDecodes.Store(0) + require.Equal(t, tc.want, registeredMetricMethod(ctx, msg)) + require.Zero(t, metricArgumentDecodes.Load(), "classification must not invoke argument decoders") + require.Equal(t, original, *msg, "classification must leave the message unchanged") + }) + } +} + +func TestMetricsDecodesOnlyForDispatch(t *testing.T) { + key := new(int) + rpc.HookHandleCallMsg(func(next rpc.HandleCallMsgFunc) rpc.HandleCallMsgFunc { + return func(ctx context.Context, msg *rpc.JsonRpcMessage) *rpc.JsonRpcMessage { + mode, active := ctx.Value(key).(string) + if !active { + return next(ctx, msg) + } + callNext := next + if mode == "routing failure" { + callNext = func(_ context.Context, msg *rpc.JsonRpcMessage) *rpc.JsonRpcMessage { + return msg.ErrorResponse(errors.New("fixture route unavailable")) + } + } + return Metrics(callNext)(ctx, msg) + } + }) + server := newMetricsFixtureServer(t) + for _, mode := range []string{"success", "routing failure"} { + t.Run(mode, func(t *testing.T) { + metricArgumentDecodes.Store(0) + payload := strings.Repeat("a", 128<<10) + body := `{"jsonrpc":"2.0","id":1,"method":"fixture_required","params":["` + payload + `"]}` + req := httptest.NewRequest("POST", "/", strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + resp := httptest.NewRecorder() + server.ServeHTTP(resp, req.WithContext(context.WithValue(req.Context(), key, mode))) + var msg rpc.JsonRpcMessage + require.NoError(t, json.Unmarshal(resp.Body.Bytes(), &msg)) + if mode == "success" { + require.Nil(t, msg.Error) + require.JSONEq(t, `true`, string(msg.Result)) + require.EqualValues(t, 1, metricArgumentDecodes.Load()) + } else { + require.EqualError(t, msg.Error, "fixture route unavailable") + require.Zero(t, metricArgumentDecodes.Load()) + } + }) + } +} + +type metricLookupContext struct { + context.Context + lookups int +} + +func (c *metricLookupContext) Value(key interface{}) interface{} { + if key == "handler" { + c.lookups++ + } + return c.Context.Value(key) +} + +func TestMetricsSkipsClassificationForNilResponse(t *testing.T) { + ctx := &metricLookupContext{Context: context.Background()} + next := func(context.Context, *rpc.JsonRpcMessage) *rpc.JsonRpcMessage { return nil } + require.Nil(t, Metrics(next)(ctx, &rpc.JsonRpcMessage{Method: "fixture_required"})) + require.Zero(t, ctx.lookups) +} + +func BenchmarkRegisteredMetricMethod(b *testing.B) { + ctx := metricsFixtureContext(b) + for _, size := range []int{32, 128 << 10} { + hex := "0x" + strings.Repeat("ab", size) + for _, tc := range []struct{ name, method, params string }{ + {"raw", "fixture_rawTransaction", `["` + hex + `"]`}, + {"call", "fixture_call", `[{"data":"` + hex + `"}]`}, + } { + b.Run(fmt.Sprintf("%s/%d", tc.name, size), func(b *testing.B) { + msg := &rpc.JsonRpcMessage{Method: tc.method, Params: json.RawMessage(tc.params)} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if registeredMetricMethod(ctx, msg) != tc.method { + b.Fatal("registered method misclassified") + } + } + }) + } + } +} From 5da9f8763f89c75f251a455b49d32f6121e52530 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 4 Nov 2026 16:09:37 +0800 Subject: [PATCH 24/26] perf(acl): compile wildcard rules when validators are created --- util/acl/validator.go | 52 ++++++------- util/acl/validator_rules_test.go | 128 +++++++++++++++++++++++++++++++ 2 files changed, 151 insertions(+), 29 deletions(-) create mode 100644 util/acl/validator_rules_test.go diff --git a/util/acl/validator.go b/util/acl/validator.go index e4e114d6..6a72e4f2 100644 --- a/util/acl/validator.go +++ b/util/acl/validator.go @@ -52,9 +52,9 @@ type validatorBase struct { *AllowList // wildchar pattern matching - allowMethodRules []string // allow methods - disallowMethodRules []string // disallow methods - originRules []string // request origins + allowMethodRules []*regexp.Regexp // allow methods + disallowMethodRules []*regexp.Regexp // disallow methods + originRules []*regexp.Regexp // request origins // contract addresses mapset contractAddressRules map[string]bool @@ -64,25 +64,22 @@ type validatorBase struct { contractAddressParsers map[string]contractAddressParser } -func newValidatorBase(al *AllowList) *validatorBase { - var originRules []string - for _, r := range al.Origins { - regp := util.WildCardToRegexp(strings.ToLower(r)) - originRules = append(originRules, regp) - } - - var allowMethodRules []string - for _, r := range al.AllowMethods { - regp := util.WildCardToRegexp(r) - allowMethodRules = append(allowMethodRules, regp) - } - - var disallowMethodRules []string - for _, r := range al.DisallowMethods { - regp := util.WildCardToRegexp(r) - disallowMethodRules = append(disallowMethodRules, regp) +func compileWildcardRules(patterns []string, lowercase bool) []*regexp.Regexp { + rules := make([]*regexp.Regexp, 0, len(patterns)) + for _, pattern := range patterns { + if lowercase { + pattern = strings.ToLower(pattern) + } + // Invalid patterns never matched in the per-request regexp path. Skip + // them, but retain the original lists for the empty-policy checks below. + if rule, err := regexp.Compile(util.WildCardToRegexp(pattern)); err == nil { + rules = append(rules, rule) + } } + return rules +} +func newValidatorBase(al *AllowList) *validatorBase { contractAddrRules := make(map[string]bool) for _, r := range al.ContractAddresses { contractAddrRules[strings.ToLower(r)] = true @@ -90,9 +87,9 @@ func newValidatorBase(al *AllowList) *validatorBase { return &validatorBase{ AllowList: al, - originRules: originRules, - allowMethodRules: allowMethodRules, - disallowMethodRules: disallowMethodRules, + originRules: compileWildcardRules(al.Origins, true), + allowMethodRules: compileWildcardRules(al.AllowMethods, false), + disallowMethodRules: compileWildcardRules(al.DisallowMethods, false), contractAddressRules: contractAddrRules, } } @@ -131,8 +128,7 @@ func (v *validatorBase) validateOrigin(ctx Context) error { reqOrigin = strings.ToLower(reqOrigin) for _, r := range v.originRules { - matched, err := regexp.MatchString(r, reqOrigin) - if err == nil && matched { + if r.MatchString(reqOrigin) { return nil } } @@ -169,8 +165,7 @@ func (v *validatorBase) validateAllowMethods(ctx Context) error { } for _, r := range v.allowMethodRules { - matched, err := regexp.MatchString(r, ctx.RpcMethod) - if err == nil && matched { + if r.MatchString(ctx.RpcMethod) { return nil } } @@ -180,8 +175,7 @@ func (v *validatorBase) validateAllowMethods(ctx Context) error { } for _, r := range v.disallowMethodRules { - matched, err := regexp.MatchString(r, ctx.RpcMethod) - if err == nil && matched { + if r.MatchString(ctx.RpcMethod) { return errBadRpcMethod } } diff --git a/util/acl/validator_rules_test.go b/util/acl/validator_rules_test.go new file mode 100644 index 00000000..239428a1 --- /dev/null +++ b/util/acl/validator_rules_test.go @@ -0,0 +1,128 @@ +package acl + +import ( + "context" + "fmt" + "regexp" + "strings" + "sync" + "testing" + + "github.com/Conflux-Chain/confura/util" + "github.com/Conflux-Chain/confura/util/rpc/handlers" + "github.com/stretchr/testify/require" +) + +func TestValidatorRuleSemantics(t *testing.T) { + for _, tc := range []struct { + name, origin, method string + list AllowList + want error + }{ + {name: "empty rules", method: "eth_getLogs"}, + {name: "wildcards", list: AllowList{Origins: []string{"HTTPS://*.EXAMPLE.COM"}, AllowMethods: []string{"eth_*"}}, origin: "https://app.example.com", method: "eth_getLogs"}, + {name: "literal origin", list: AllowList{Origins: []string{"https://app.example.com"}}, origin: "https://appXexample.com", want: errInvalidOrigin}, + {name: "literal method", list: AllowList{AllowMethods: []string{"eth_read.v1"}}, method: "eth_readXv1", want: errBadRpcMethod}, + {name: "regexp characters", list: AllowList{AllowMethods: []string{"eth_[read]+(v1)?$"}}, method: "eth_[read]+(v1)?$"}, + {name: "case sensitive method", list: AllowList{AllowMethods: []string{"eth_*"}}, method: "ETH_getLogs", want: errBadRpcMethod}, + {name: "missing origin", list: AllowList{Origins: []string{"*"}}, want: errInvalidOrigin}, + {name: "allow takes priority", list: AllowList{AllowMethods: []string{"eth_getLogs"}, DisallowMethods: []string{"eth_*"}}, method: "eth_getLogs"}, + {name: "allow miss denies", list: AllowList{AllowMethods: []string{"eth_getLogs"}, DisallowMethods: []string{"cfx_*"}}, method: "eth_call", want: errBadRpcMethod}, + {name: "disallow hit", list: AllowList{DisallowMethods: []string{"eth_*"}}, method: "eth_getLogs", want: errBadRpcMethod}, + {name: "disallow miss", list: AllowList{DisallowMethods: []string{"cfx_*"}}, method: "eth_getLogs"}, + {name: "invalid allow denies", list: AllowList{AllowMethods: []string{"eth_\xff"}}, method: "eth_getLogs", want: errBadRpcMethod}, + {name: "invalid disallow ignored", list: AllowList{DisallowMethods: []string{"eth_\xff"}}, method: "eth_getLogs"}, + {name: "invalid followed by valid allow", list: AllowList{AllowMethods: []string{"eth_\xff", "eth_*"}}, method: "eth_getLogs"}, + {name: "invalid followed by valid disallow", list: AllowList{DisallowMethods: []string{"eth_\xff", "eth_*"}}, method: "eth_getLogs", want: errBadRpcMethod}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + if tc.origin != "" { + ctx = context.WithValue(ctx, handlers.CtxKeyReqOrigin, tc.origin) + } + err := NewEthValidator(&tc.list).Validate(Context{Context: ctx, RpcMethod: tc.method}) + require.ErrorIs(t, err, tc.want) + }) + } +} + +// Compare decisions with the prior per-request regexp path, including invalid +// UTF-8 and empty-string patterns, whose behavior must not change on compile. +func TestValidatorRulesMatchOriginalDecisions(t *testing.T) { + ruleSets := [][]string{nil, {""}, {"*"}, {"eth_*", "cfx_*"}, {"eth_\xff"}, {"eth_\xff", "eth_*"}, {"https://*.example.com"}, {"https://\xff.example.com"}, {"[a]+(b)?."}} + inputs := []string{"", "eth_getLogs", "ETH_getLogs", "eth_\xff", "cfx_call", "https://app.example.com", "https://\xff.example.com", "[a]+(b)?."} + matches := func(rules []string, input string, lower bool) bool { + for _, rule := range rules { + if lower { + rule, input = strings.ToLower(rule), strings.ToLower(input) + } + if ok, err := regexp.MatchString(util.WildCardToRegexp(rule), input); err == nil && ok { + return true + } + } + return false + } + for _, rules := range ruleSets { + for _, input := range inputs { + ctx := Context{Context: context.WithValue(context.Background(), handlers.CtxKeyReqOrigin, input), RpcMethod: input} + for _, kind := range []string{"origin", "allow", "disallow"} { + al := &AllowList{} + allowed := true + switch kind { + case "origin": + al.Origins = rules + allowed = len(rules) == 0 || matches(rules, input, true) + case "allow": + al.AllowMethods = rules + allowed = len(rules) == 0 || matches(rules, input, false) + case "disallow": + al.DisallowMethods = rules + allowed = !matches(rules, input, false) + } + require.Equal(t, allowed, NewEthValidator(al).Validate(ctx) == nil, "%s rules=%q input=%q", kind, rules, input) + } + } + } +} + +func TestValidatorConcurrentRuleReads(t *testing.T) { + v := NewEthValidator(&AllowList{Origins: []string{"https://*.example.com"}, AllowMethods: []string{"eth_*"}}) + ctx := Context{Context: context.WithValue(context.Background(), handlers.CtxKeyReqOrigin, "https://app.example.com"), RpcMethod: "eth_getLogs"} + var wg sync.WaitGroup + for i := 0; i < 16; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < 32; j++ { + if err := v.Validate(ctx); err != nil { + t.Error(err) + } + } + }() + } + wg.Wait() +} + +func BenchmarkACLValidation(b *testing.B) { + for _, count := range []int{1, 10} { + b.Run(fmt.Sprint(count), func(b *testing.B) { + al := &AllowList{} + for i := 0; i < count; i++ { + al.Origins = append(al.Origins, fmt.Sprintf("https://app%d.*.example.com", i)) + al.AllowMethods = append(al.AllowMethods, fmt.Sprintf("fixture%d_*", i)) + } + v := NewEthValidator(al) + ctx := Context{ + Context: context.WithValue(context.Background(), handlers.CtxKeyReqOrigin, fmt.Sprintf("https://app%d.rpc.example.com", count-1)), + RpcMethod: fmt.Sprintf("fixture%d_read", count-1), + } + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if err := v.Validate(ctx); err != nil { + b.Fatal(err) + } + } + }) + } +} From e40922417a6c2a2349e2fc37cdd762fbfd44de86 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 4 Nov 2026 16:11:31 +0800 Subject: [PATCH 25/26] perf(relay): avoid transaction copies when the buffered queue is full --- util/relay/relay.go | 5 +++ util/relay/relay_test.go | 72 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+) diff --git a/util/relay/relay.go b/util/relay/relay.go index f5d69ca0..79d8258c 100644 --- a/util/relay/relay.go +++ b/util/relay/relay.go @@ -61,6 +61,11 @@ func newTxnRelayer(relayConf *TxnRelayerConfig) *txnRelayer { // Relay relays raw transaction broadcasting asynchronously. func (relayer *txnRelayer) Relay(signedTx hexutil.Bytes) bool { + // Avoid copying a transaction while a buffered queue is already full. + // This is only an early rejection: select still decides admission atomically. + if capacity := cap(relayer.txnQueue); capacity > 0 && len(relayer.txnQueue) == capacity { + return false + } // Own queued bytes; callers may reuse their request buffer after return. raw := append(hexutil.Bytes(nil), signedTx...) select { diff --git a/util/relay/relay_test.go b/util/relay/relay_test.go index baf3c682..70d5359a 100644 --- a/util/relay/relay_test.go +++ b/util/relay/relay_test.go @@ -1,6 +1,8 @@ package relay import ( + "fmt" + "runtime" "sync" "sync/atomic" "testing" @@ -41,3 +43,73 @@ func TestRelayConcurrentProducersNeverWaitForConsumer(t *testing.T) { t.Fatal("caller mutated queued bytes") } } + +func TestRelayFullQueueDoesNotCopy(t *testing.T) { + r := newTxnRelayer(&TxnRelayerConfig{BufferSize: 1}) + if !r.Relay([]byte{1}) { + t.Fatal("initial admission failed") + } + raw := make([]byte, 128<<10) + if allocs := testing.AllocsPerRun(100, func() { + if r.Relay(raw) { + t.Fatal("accepted into a full queue") + } + }); allocs != 0 { + t.Fatalf("full queue rejection allocated %v times", allocs) + } +} + +func TestRelayUnbufferedAdmission(t *testing.T) { + r := newTxnRelayer(&TxnRelayerConfig{BufferSize: 0}) + if r.Relay([]byte{1}) { + t.Fatal("unbuffered queue accepted without a receiver") + } + received := make(chan []byte, 1) + stop := make(chan struct{}) + done := make(chan struct{}) + go func() { + defer close(done) + select { + case raw := <-r.txnQueue: + received <- raw + case <-stop: + } + }() + defer func() { close(stop); <-done }() + raw := []byte{2} + deadline := time.Now().Add(time.Second) + for !r.Relay(raw) { + if time.Now().After(deadline) { + t.Fatal("unbuffered queue never admitted with a waiting receiver") + } + runtime.Gosched() + } + raw[0] = 9 + if got := <-received; got[0] != 2 { + t.Fatal("caller mutated unbuffered transaction bytes") + } +} + +func BenchmarkRelayAdmission(b *testing.B) { + for _, full := range []bool{false, true} { + for _, size := range []int{32, 128 << 10} { + b.Run(fmt.Sprintf("full=%t/%d", full, size), func(b *testing.B) { + r := newTxnRelayer(&TxnRelayerConfig{BufferSize: 1}) + raw := make([]byte, size) + if full && !r.Relay(raw) { + b.Fatal("initial admission failed") + } + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if r.Relay(raw) == full { + b.Fatal("unexpected admission result") + } + if !full { + <-r.txnQueue + } + } + }) + } + } +} From 7a6a8a74322d736858fb48e110ba363b20145561 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 4 Nov 2026 16:13:18 +0800 Subject: [PATCH 26/26] perf(rpc): build near-head retry diagnostics only when enabled --- util/rpc/cached_client_eth.go | 27 +++++++++++++--------- util/rpc/nearhead_test.go | 42 +++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 11 deletions(-) diff --git a/util/rpc/cached_client_eth.go b/util/rpc/cached_client_eth.go index a7606b55..d6e01cc7 100644 --- a/util/rpc/cached_client_eth.go +++ b/util/rpc/cached_client_eth.go @@ -699,7 +699,7 @@ func (c ethCacheClient) logsByBlockRange(filter web3Types.FilterQuery) ([]web3Ty logs, err := c.logsByBlockRangeOnce(filter, fromBlock, toBlock, tracker) conflict, ok := err.(*nearHeadLogConflict) if !ok { - if err == nil && attempts > 1 { + if err == nil && attempts > 1 && logrus.IsLevelEnabled(logrus.DebugLevel) { logrus.WithField("attempts", attempts).Debug("Near-head log consistency recovered") } return logs, tracker.withError(err) @@ -707,20 +707,25 @@ func (c ethCacheClient) logsByBlockRange(filter web3Types.FilterQuery) ([]web3Ty if deadline.IsZero() { deadline = time.Now().Add(nearHeadLogRetryTimeout) } - logger := logrus.WithFields(logrus.Fields{ - "fromBlock": fromBlock, - "toBlock": toBlock, - "boundary": conflict.boundary, - "reason": conflict.reason, - "attempts": attempts, - "elapsed": time.Since(started), - }) + logConflict := func(level logrus.Level, message string) { + if !logrus.IsLevelEnabled(level) { + return + } + logrus.WithFields(logrus.Fields{ + "fromBlock": fromBlock, + "toBlock": toBlock, + "boundary": conflict.boundary, + "reason": conflict.reason, + "attempts": attempts, + "elapsed": time.Since(started), + }).Log(level, message) + } if remaining := time.Until(deadline); remaining > 0 { - logger.Debug("Retrying near-head logs after canonical view conflict") + logConflict(logrus.DebugLevel, "Retrying near-head logs after canonical view conflict") time.Sleep(min(nearHeadLogRetryBackoff, remaining)) } if !time.Now().Before(deadline) { - logger.WithField("elapsed", time.Since(started)).Info("Near-head log consistency retry window exhausted") + logConflict(logrus.InfoLevel, "Near-head log consistency retry window exhausted") return nil, tracker.withError(ErrInconsistentCanonicalView) } } diff --git a/util/rpc/nearhead_test.go b/util/rpc/nearhead_test.go index 3995b3e8..bf23ad8c 100644 --- a/util/rpc/nearhead_test.go +++ b/util/rpc/nearhead_test.go @@ -648,3 +648,45 @@ func TestReadNearHeadLogsErrorsAndFilter(t *testing.T) { require.EqualError(t, err, "failed to get logs from near head cache: invalid block range params") require.Equal(t, scans, r.scans) } + +func TestNearHeadLogRetryDiagnosticLevels(t *testing.T) { + for _, level := range []logrus.Level{logrus.DebugLevel, logrus.InfoLevel, logrus.WarnLevel} { + t.Run(level.String(), func(t *testing.T) { + logger := logrus.StandardLogger() + output, originalLevel := logger.Out, logger.GetLevel() + var diagnostics bytes.Buffer + logger.SetOutput(&diagnostics) + logger.SetLevel(level) + t.Cleanup(func() { logger.SetOutput(output); logger.SetLevel(originalLevel) }) + + n := fixtureNearhead() + hash := common.HexToHash("0xa100") + require.NoError(t, n.Put(fixtureBlock(100, hash, nil))) + headers := 0 + p := &fixtureProvider{call: func(_ context.Context, out interface{}, method string, args ...interface{}) error { + if method == "eth_getLogs" { + return nearHeadTestReply(out, []wt.Log{}) + } + require.Equal(t, "eth_getBlockByNumber", method) + headers++ + if headers == 1 { + return nearHeadTestReply(out, nil) + } + return nearHeadTestReply(out, nearHeadTestHeader(100, hash)) + }} + from, to := wt.BlockNumber(99), wt.BlockNumber(100) + _, err := nearHeadTestClient(n, p).Logs(wt.FilterQuery{FromBlock: &from, ToBlock: &to}) + require.NoError(t, err) + require.Equal(t, 3, headers) + if level == logrus.DebugLevel { + require.Equal(t, 2, bytes.Count(diagnostics.Bytes(), []byte("level=debug"))) + require.Contains(t, diagnostics.String(), "Retrying near-head logs after canonical view conflict") + require.Contains(t, diagnostics.String(), "Near-head log consistency recovered") + require.Contains(t, diagnostics.String(), "boundary=100") + require.NotContains(t, diagnostics.String(), "level=info") + } else { + require.Empty(t, diagnostics.String()) + } + }) + } +}