Merge pull request #7 from zen-eth/optimize

chore: do some optimization

Author: fearlessfe

Makefile

@@ -8,7 +8,7 @@ GOBIN = ./build/bin
 GO ?= latest
 GORUN = go run
 
-GIT_COMMIT := $(shell git rev-parse HEAD)
+GIT_COMMIT := $(shell git rev-parse --short=8 HEAD)
 GIT_DATE := $(shell git log -1 --format=%ci | cut -d ' ' -f 1)
 
 #? shisui: Build shisui
@@ -21,7 +21,7 @@ shisui:
 
 #? shisui-image: Build shisui image
 shisui-image:
-	docker build -t ghcr.io/optimism-java/shisui:latest -f Dockerfile .
+	docker build -t ghcr.io/zen-eth/shisui:latest -f Dockerfile .
 
 #? fmt: Ensure consistent code formatting.
 fmt:

cmd/shisui/main.go

@@ -42,6 +42,7 @@ import (
 	"github.com/optimism-java/shisui2/web3"
 	"github.com/protolambda/zrnt/eth2/configs"
 	"github.com/urfave/cli/v2"
+	"go.uber.org/automaxprocs/maxprocs"
 )
 
 var (
@@ -111,6 +112,10 @@ func init() {
 	flags.AutoEnvVars(app.Flags, "SHISUI")
 
 	app.Before = func(ctx *cli.Context) error {
+		_, err := maxprocs.Set() // Automatically set GOMAXPROCS to match Linux container CPU quota.
+		if err != nil {
+			return err
+		}
 		flags.MigrateGlobalFlags(ctx)
 		if err := debug.Setup(ctx); err != nil {
 			return err

go.mod

@@ -23,6 +23,7 @@ require (
 	github.com/stretchr/testify v1.10.0
 	github.com/tetratelabs/wabin v0.0.0-20230304001439-f6f874872834
 	github.com/urfave/cli/v2 v2.27.5
+	go.uber.org/automaxprocs v1.6.0
 	go.uber.org/zap v1.27.0
 	golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f
 	golang.org/x/text v0.21.0

go.sum

@@ -193,6 +193,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g=
+github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U=
 github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y=
 github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
 github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
@@ -254,6 +256,8 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
 github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
+go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs=
+go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=

history/history_network.go

@@ -148,7 +148,7 @@ func (h *Network) GetBlockHeader(blockHash []byte) (*types.Header, error) {
 		}
 		err = h.portalProtocol.Put(contentKey, contentId, content)
 		if err != nil {
-			h.log.Error("failed to store content in getBlockHeader", "contentKey", hexutil.Encode(contentKey), "content", hexutil.Encode(content))
+			h.log.Error("failed to store content in getBlockHeader", "contentKey", hexutil.Encode(contentKey), "err", err)
 		}
 		return header, nil
 	}
@@ -199,7 +199,7 @@ func (h *Network) GetBlockBody(blockHash []byte) (*types.Body, error) {
 		}
 		err = h.portalProtocol.Put(contentKey, contentId, content)
 		if err != nil {
-			h.log.Error("failed to store content in getBlockBody", "contentKey", hexutil.Encode(contentKey), "content", hexutil.Encode(content))
+			h.log.Error("failed to store content in getBlockBody", "contentKey", hexutil.Encode(contentKey), "err", err)
 		}
 		return body, nil
 	}
@@ -248,7 +248,7 @@ func (h *Network) GetReceipts(blockHash []byte) ([]*types.Receipt, error) {
 		}
 		err = h.portalProtocol.Put(contentKey, contentId, content)
 		if err != nil {
-			h.log.Error("failed to store content in getReceipts", "contentKey", hexutil.Encode(contentKey), "content", hexutil.Encode(content))
+			h.log.Error("failed to store content in getReceipts", "contentKey", hexutil.Encode(contentKey), "err", err)
 		}
 		return receipts, nil
 	}
@@ -564,8 +564,7 @@ func (h *Network) validateContents(contentKeys [][]byte, contents [][]byte) erro
 		contentKey := contentKeys[i]
 		err := h.validateContent(contentKey, content)
 		if err != nil {
-			h.log.Error("content validate failed", "contentKey", hexutil.Encode(contentKey), "content", hexutil.Encode(content), "err", err)
-			return fmt.Errorf("content validate failed with content key %x and content %x", contentKey, content)
+			return fmt.Errorf("content validate failed with content key %x and content %x, err is %v", contentKey, content, err)
 		}
 		contentId := h.portalProtocol.ToContentId(contentKey)
 		_ = h.portalProtocol.Put(contentKey, contentId, content)

storage/content_storage.go

@@ -10,8 +10,8 @@ var ErrContentNotFound = fmt.Errorf("content not found")
 var ErrInsufficientRadius = fmt.Errorf("insufficient radius")
 
 var MaxDistance = uint256.MustFromHex("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")
-
-var SizeKey = []byte("size")
+var smallestDistance = uint256.NewInt(0).Bytes32()
+var SizeKey = smallestDistance[:] // smallest key, make sure the key will not be prund
 
 type ContentType byte
 

storage/pebble/storage.go

@@ -3,6 +3,7 @@ package pebble
 import (
 	"bytes"
 	"encoding/binary"
+	"fmt"
 	"runtime"
 	"sync"
 	"sync/atomic"
@@ -237,25 +238,28 @@ func (c *ContentStorage) Close() error {
 
 func (c *ContentStorage) prune() error {
 	expectSize := uint64(float64(c.storageCapacityInBytes) * contentDeletionFraction)
-	var curentSize uint64 = 0
+	var currentSize uint64 = 0
+	c.log.Debug("start pruning", "expectSize", expectSize)
 
 	// get the keys to be deleted order by distance desc
 	iter, err := c.db.NewIter(nil)
 	if err != nil {
 		return err
 	}
 
+	pruneCount := 0
 	batch := c.db.NewBatch()
 	for iter.Last(); iter.Valid(); iter.Prev() {
 		if bytes.Equal(iter.Key(), storage.SizeKey) {
 			continue
 		}
-		if curentSize < expectSize {
+		if currentSize < expectSize {
 			err := batch.Delete(iter.Key(), nil)
 			if err != nil {
 				return err
 			}
-			curentSize += uint64(len(iter.Key())) + uint64(len(iter.Value()))
+			currentSize += uint64(len(iter.Key())) + uint64(len(iter.Value()))
+			pruneCount++
 		} else {
 			distance := iter.Key()
 			dis := uint256.NewInt(0)
@@ -267,7 +271,12 @@ func (c *ContentStorage) prune() error {
 			break
 		}
 	}
-	newSize := c.size.Add(-curentSize)
+	size := c.size.Load()
+	if size < currentSize {
+		return fmt.Errorf("prune error, size < currentSize, got size %d, currentSize %d", size, currentSize)
+	}
+	newSize := size - currentSize
+	c.size.Store(newSize)
 	buf := c.bytePool.Get().(*[]byte)
 	defer c.bytePool.Put(buf)
 	binary.BigEndian.PutUint64(*buf, newSize)
@@ -279,6 +288,8 @@ func (c *ContentStorage) prune() error {
 	if err != nil {
 		return err
 	}
+
+	c.log.Debug("prune finished", "pruneCount", pruneCount, "pruneSize", currentSize, "current storage", newSize)
 	go func() {
 		start := uint256.NewInt(0).Bytes32()
 		end := storage.MaxDistance.Bytes32()