Merge branch 'firehose-fh2.3' into release/bsc-1.x-fh2.5

# Conflicts:
#	cmd/devp2p/main.go
#	cmd/geth/main.go
#	core/blockchain.go
#	core/blockchain_test.go
#	core/genesis.go
#	core/state_processor.go
#	core/types.go
#	eth/state_accessor.go
#	firehose/context.go
#	internal/debug/flags.go
#	internal/ethapi/api.go
#	params/version.go

.github/workflows/docker.yml

@@ -8,6 +8,10 @@ on:
       - "*/feature/*"
     tags:
       - "[a-z0-9]+-v*"
+    branches:
+      # Expected to be <network>/{fix,feature}/<name> where <network> is one of {geth,bsc,polygon,etc.}
+      - "*/fix/*"
+      - "*/feature/*"
 
 env:
   REGISTRY: ghcr.io

README.fh.md

@@ -2,14 +2,16 @@
 
 This is our Firehose instrumented fork of [ethereum/go-ethereum](https://github.com/ethereum/go-ethereum) repository. In this README, you will find instructions about how to work with this repository.
 
-### Versions
+### Protocols
 
-The Firehose instrumentation have a major version for the messages exchanges with Firehose on Ethereum binary (`fheth`). The
-versions we currently supported:
+The Firehose instrumentation have a protocol version for the messages exchanges with Firehose on Ethereum binary (`fireeth`). The
+protocols we currently develop are:
 
-- Version 2 using the `firehose-v2` branch and `fh2` tag(s) suffix
+- Protocol `fh2.3` using the `firehose-fh2.3` branch and `fh2.3` tag(s) suffix
+- Protocol `fh3.0` using the `firehose-fh3.0` branch and `fh3.0` tag(s) suffix
 
-> **Warning** Version 1 using the `firehose-v1` branch and `fh1` tag(s) suffix is not supported anymore, branches and tags exist for historical access.
+  > [!NOTE]
+  > This is an on-going development switching Firehose implementation to use Geth Native Tracer that is going to be merged upstream at some point
 
 Read [Branches & Workflow](#branches-&-workflow) section for more details about how we handle branching model and versions.
 
@@ -21,13 +23,12 @@ the same names and settings.
 
 ```
 cd ~/work
-git clone --branch="firehose-v1" [email protected]:streamingfast/go-ethereum.git
+git clone --branch="firehose-fh2.3" [email protected]:streamingfast/go-ethereum.git
 cd go-ethereum
 
 git remote rename origin sf
 
-git checkout firehose-v2
-git checkout firehose-v2-v1.10.1
+git checkout firehose-fh2.3
 
 git remote add origin https://github.com/ethereum/go-ethereum.git
 git remote add polygon https://github.com/maticnetwork/bor.git
@@ -37,27 +38,11 @@ git fetch origin
 git fetch polygon
 git fetch bsc
 
-git checkout release/geth-1.12.x-fh2
-git checkout release/bsc-1.2.x-fh2
-git checkout release/polygon-0.x-fh2
+git checkout release/geth-1.x-fh2.3
+git checkout release/bsc-1.x-fh2.2
+git checkout release/polygon-0.x-fh2.3
 ```
 
-If you want to work on one of the unmaintained branches:
-
-```
-git remote add heco https://github.com/HuobiGroup/huobi-eco-chain.git
-git remote add optimism https://github.com/ethereum-optimism/go-ethereum.git
-git remote add fantom-geth https://github.com/Fantom-foundation/go-ethereum.git
-git fetch heco
-git fetch optimism
-git fetch fantom-geth
-git checkout release/heco-1.0.x-fh1
-git checkout release/optimism-0.1.x-fh1
-git checkout release/fantom-geth-1.9.x-fh1
-```
-
-> **Warning** Those are unmaintained since a long time and have no `fh2` version, so bringing in `fh2` changes is the first step that would be required.
-
 ##### Assumptions
 
 For the best result when working with this repository and the scripts it contains:
@@ -75,62 +60,52 @@ for Firehose consumption.
 
 We use merging of the branches into one another to make that work manageable.
 The first and foremost important rule is that we always put new development
-in the `firehose-v2` branch.
+in the `firehose-fh2.3` branch.
 
 This branch must always be tracking the lowest supported version of all. Indeed,
 this is our "work" branch for our patches, **new development must go there**. If you
 perform our work with newer code, the problem that will arise is that this new
 firehose work will not be mergeable into forks or older release that we still
 support!
 
-We also support and intermediate branch for version v1.10.1 of Geth. The reason for this
-is that this version added refactoring to support multiple transactions types which needs to be
-properly handled (1.10.0 more specifically did the refactoring, but 1.10.1 was added almost at
-the same moment as 1.10.0).
-
-`firehose-v2` should be merged always in `firehose-v2-v1.10.1`.
+The lowest supported Geth version today is `1.10.1`.
 
 We then create `release/<identifier>` branch that tracks the version of interest
 for us, versions that we will manages and deploy.
 
 Currently supported forks & version and the release branch
 
-- `firehose-v2` - Default branch with all Firehose commits in it, based on Geth `1.9.10` (Ethereum Block Version 2).
-- [release/geth-1.10.x-fh2](https://github.com/streamingfast/go-ethereum/tree/release/geth-1.10.x-fh2) - Ethereum Geth, latest update for this branch is `1.10.7` ([https://github.com/ethereum/go-ethereum](https://github.com/ethereum/go-ethereum)).
-- [release/polygon-0.x-fh2](https://github.com/streamingfast/go-ethereum/tree/release/polygon-0.x-fh2) - Polygon fork (a.k.a Matic), based on Geth `1.10.3`, latest update for this branch is `v0.3.2` ([https://github.com/maticnetwork/bor](https://github.com/maticnetwork/bor)).
-- [release/bsc-1.1.x-fh2](https://github.com/streamingfast/go-ethereum/tree/release/bsc-1.1.x-fh2) - BSC fork (Binance), based on Geth `1.10.22`, latest update for this branch is `v1.1.18` ([https://github.com/binance-chain/bsc](https://github.com/binance-chain/bsc)).
-
-> **Note** To find on which Geth version a particular fork is, you can do `git merge-base sf/release/heco-v1.0.x-fh1 origin/master` where `origin/master` is the `master` branch of the original Geth repository (https://github.com/ethereum/go-ethereum).
-
-#### Unsupported
+- `firehose-fh2.3` - Default branch with all Firehose commits in it, based on Geth `1.10.1`.
+- [release/geth-1.x-fh2.3](https://github.com/streamingfast/go-ethereum/tree/release/geth-1.x-fh2.3) - Ethereum Geth, latest update for this branch is `1.10.7` ([https://github.com/ethereum/go-ethereum](https://github.com/ethereum/go-ethereum)).
+- [release/polygon-0.x-fh2.3](https://github.com/streamingfast/go-ethereum/tree/release/polygon-0.x-fh2.3) - Polygon fork (a.k.a Matic), based on Geth `1.10.3`, latest update for this branch is `v0.3.2` ([https://github.com/maticnetwork/bor](https://github.com/maticnetwork/bor)).
+- [release/bsc-1.x-fh2.2](https://github.com/streamingfast/go-ethereum/tree/release/bsc-1.x-fh2.2) - BSC fork (Binance), based on Geth `1.10.22`, latest update for this branch is `v1.1.18` ([https://github.com/binance-chain/bsc](https://github.com/binance-chain/bsc)).
 
-- [release/heco-1.0.x-fh1](https://github.com/streamingfast/go-ethereum/tree/release/heco-1.0.x-fh1) - HECO fork (a.k.a Huobi Eco Chain), based on Geth `1.10.1`, latest update for this branch is `v1.1.1` ([https://github.com/HuobiGroup/huobi-eco-chain](https://github.com/HuobiGroup/huobi-eco-chain)).
-- [release/optimism-0.1.x-fh1](https://github.com/streamingfast/go-ethereum/tree/release/optimism-0.1.x-fh1) - Optimism fork, based on Geth `1.9.10`, latest update for this branch is `v0.1.4` ([https://github.com/ethereum-optimism/go-ethereum](https://github.com/ethereum-optimism/go-ethereum)).
-- [release/fantom-geth-1.9.x-fh1](https://github.com/streamingfast/go-ethereum/tree/release/fantom-geth-1.9.x-fh1) - Fantom Geth fork, based on Geth `1.9.22`, latest update for this branch is `v1.9.22-ftm-0.5` (a branch) ([https://github.com/Fantom-foundation/go-ethereum](https://github.com/Fantom-foundation/go-ethereum)).
+> **Note** To find on which Geth version a particular fork is, you can do `git merge-base sf/release/geth-1.x-fh2.3 origin/master` where `origin/master` is the `master` branch of the original Geth repository (https://github.com/ethereum/go-ethereum).
 
 #### Making New Firehose Changes
 
-Making new changes should be performed on the `firehose-v2-v1.10.1` branch. When happy
-with the changes, simply merge the `firehose-v2-v1.10.1` branch in all the release branches we track
+Making new changes should be performed on the `firehose-fh2.3` branch. When happy
+with the changes, simply merge the `firehose-fh2.3` branch in all the release branches we track
 and support.
 
-    git checkout firehose-v2-v1.10.1
+    git checkout firehose-fh2.3
     git pull -p
 
     # Perform necessary changes, tests and commit(s)
 
-    git checkout release/geth-1.10.x-fh2
+    git checkout release/geth-1.x-fh2.3
     git pull -p
-    git merge firehose-v2-v1.10.1
+    git merge firehose-fh2.3
 
-    git checkout release/polygon-0.x-fh2
+    git checkout release/polygon-0.x-fh2.3
     git pull -p
-    git merge firehose-v2-v1.10.1
+    git merge firehose-fh2.3
 
-    git push sf firehose-v2-v1.10.1 release/geth-1.10.x-fh2 release/polygon-0.x-fh2
+    git checkout release/bsc-1.x-fh2.3
+    git pull -p
+    git merge firehose-fh2.3
 
-**Note** On newer fork, you must merge the right version, for example if the fork never merged 1.10.1 from the upstream Geth, you must **not** use
-`firehose-v2-v1.10.1` but instead use `firehose-v2` which is based on 1.9.10.
+    git push sf firehose-fh2.3 release/geth-1.x-fh2.3 release/polygon-0.x-fh2.3 release/bsc-1.x-fh2.3
 
 ### Update to New Upstream Version
 
@@ -145,7 +120,7 @@ those with your own values.
 First step is to checkout the release branch of the series you are currently
 updating to:
 
-    git checkout release/geth-1.10.x-fh2
+    git checkout release/geth-1.x-fh2.3
     git pull -p
 
 You first fetch the origin repository new data from Git:
@@ -159,15 +134,18 @@ Then apply the update
 Solve conflicts if any. Once all conflicts have been resolved, commit then
 create a tag with release
 
-    git tag geth-v1.10.18-fh2
+    git tag geth-v1.10.18-fh2.3
 
 Then push all that to the repository:
 
-    git push sf release/geth-1.10.x-fh2 geth-v1.10.18-fh2
+    git push sf release/geth-1.x-fh2.3 geth-v1.10.18-fh2.3
+
+> [!NOTE]
+> If you need to issue a Firehose bug fix for an existing version of upstream, for example a Firehose fix on `v1.10.8`, you append `-N` at the end where `N` is 1 then increments further is newer revisions are needed, so you would got tag `geth-v1.10.18-fh2.3-1`
 
 ### Development
 
-All the *new* development should happen in the `firehose-v2` branch, this is our own branch
+All the *new* development should happen in the `firehose-fh2.3` branch, this is our own branch
 containing our commits.
 
 ##### Build Locally
@@ -181,9 +159,9 @@ containing our commits.
 ### View only our commits
 
 **Important** To correctly work, you need to use the right base branch, otherwise, it will be screwed up. The `firehose-v2`
-branch was based on `v1.9.23` at time of writing.
+branch was based on `v1.10.1` at time of writing.
 
-* From `gitk`: `gitk --no-merges --first-parent v1.9.23..firehose-v2`
-* From terminal: `git log --decorate --pretty=oneline --abbrev-commit --no-merges --first-parent v1.9.23..firehose-v2`
-* From `GitHub`: [https://github.com/streamingfast/go-ethereum/compare/v1.9.23...firehose-v1](https://github.com/streamingfast/go-ethereum/compare/v1.9.23...firehose-v2)
-* Modified files in our fork: `git diff --name-status v1.9.23..firehose-v2 | grep -E "^M" | cut -d $'\t' -f 2`
+* From `gitk`: `gitk --first-parent v1.10.1..firehose-fh2.3`
+* From terminal: `git log --decorate --pretty=oneline --abbrev-commit --first-parent=v1.10.1..firehose-fh2.3`
+* From `GitHub`: [https://github.com/streamingfast/go-ethereum/compare/v1.10.1...firehose-fh2.3](https://github.com/streamingfast/go-ethereum/compare/v1.9.23...firehose-fh2.3)
+* Modified files in our fork: `git diff --name-status v1.10.1..firehose-fh2.3 | grep -E "^M" | cut -d $'\t' -f 2`

core/blockchain.go

@@ -547,12 +547,12 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis
 		}
 
 		if firehose.GenesisConfig == nil {
-			panic(fmt.Errorf("genesis config is not set, there is something weird as all code path should generate the correct genesis config"))
+			panic(firehose.MissingGenesisPanicMessage)
 		}
 
 		genesis := firehose.GenesisConfig.(*Genesis)
 		if genesis == nil {
-			panic(fmt.Errorf("genesis config is not set, there is something weird as all code path should generate the correct genesis config"))
+			panic(firehose.MissingGenesisPanicMessage)
 		}
 
 		// As far as I can tell, the block's hash comes from the keccak hash of the rlp encoding
@@ -2221,6 +2221,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error)
 				td := new(big.Int).Add(block.Difficulty(), ptd)
 				finalBlock := getFinalBlockForFirehose(bc, block)
 				firehoseContext.EndBlock(block, finalBlock, td)
+				firehoseContext.FlushBlock()
 			}
 
 			stats.processed++
@@ -2267,14 +2268,17 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error)
 		}
 		statedb.SetExpectedStateRoot(block.Root())
 		pstart := time.Now()
-		statedb, receipts, logs, usedGas, err := bc.processor.Process(block, statedb, bc.vmConfig)
+
+		firehoseContext := firehose.NoOpContext
+		if firehose.Enabled {
+			firehoseContext = firehose.NewSpeculativeExecutionContextWithBuffer(firehose.BlockSyncBuffer)
+		}
+
+		statedb, receipts, logs, usedGas, err := bc.processor.Process(block, statedb, bc.vmConfig, firehoseContext)
 		close(interruptCh) // state prefetch can be stopped
 		if err != nil {
 			bc.reportBlock(block, receipts, err)
 			statedb.StopPrefetcher()
-			if firehoseContext := firehose.MaybeSyncContext(); firehoseContext.Enabled() {
-				firehoseContext.CancelBlock(block, err)
-			}
 			return it.index, err
 		}
 		ptime := time.Since(pstart)
@@ -2285,15 +2289,12 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error)
 			log.Error("validate state failed", "error", err)
 			bc.reportBlock(block, receipts, err)
 			statedb.StopPrefetcher()
-			if firehoseContext := firehose.MaybeSyncContext(); firehoseContext.Enabled() {
-				firehoseContext.CancelBlock(block, err)
-			}
 			return it.index, err
 		}
 		vtime := time.Since(vstart)
 		proctime := time.Since(start) // processing + validation
 
-		if firehoseContext := firehose.MaybeSyncContext(); firehoseContext.Enabled() {
+		if firehoseContext.Enabled() {
 			// Calculate the total difficulty of the block
 			ptd := bc.GetTd(block.ParentHash(), block.NumberU64()-1)
 			td := new(big.Int).Add(block.Difficulty(), ptd)
@@ -2337,6 +2338,11 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error)
 
 		bc.cacheReceipts(block.Hash(), receipts, block)
 
+		if firehoseContext.Enabled() {
+			// This is last point where there is no more an early return due to an error, we flush here
+			firehoseContext.FlushBlock()
+		}
+
 		// Update the metrics touched during block commit
 		accountCommitTimer.Update(statedb.AccountCommits)   // Account commits are complete, we can mark them
 		storageCommitTimer.Update(statedb.StorageCommits)   // Storage commits are complete, we can mark them

core/blockchain_test.go

@@ -29,7 +29,6 @@ import (
 
 	"github.com/ethereum/go-ethereum/consensus/misc/eip4844"
 	"github.com/ethereum/go-ethereum/crypto/kzg4844"
-	"github.com/ethereum/go-ethereum/firehose"
 	"github.com/stretchr/testify/require"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -44,6 +43,7 @@ import (
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/eth/tracers/logger"
 	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/firehose"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/trie"
 	"github.com/holiman/uint256"
@@ -224,7 +224,7 @@ func testBlockChainImport(chain types.Blocks, pipelineCommit bool, blockchain *B
 		if pipelineCommit {
 			statedb.EnablePipeCommit()
 		}
-		statedb, receipts, _, usedGas, err := blockchain.processor.Process(block, statedb, vm.Config{})
+		statedb, receipts, _, usedGas, err := blockchain.processor.Process(block, statedb, vm.Config{}, firehose.NoOpContext)
 		if err != nil {
 			blockchain.reportBlock(block, receipts, err)
 			return err

core/state_processor.go

@@ -59,15 +59,14 @@ func NewStateProcessor(config *params.ChainConfig, bc *BlockChain, engine consen
 // Process returns the receipts and logs accumulated during the process and
 // returns the amount of gas that was used in the process. If any of the
 // transactions failed to execute due to insufficient gas it will return an error.
-func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg vm.Config) (*state.StateDB, types.Receipts, []*types.Log, uint64, error) {
+func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg vm.Config, firehoseContext *firehose.Context) (*state.StateDB, types.Receipts, []*types.Log, uint64, error) {
 	var (
-		usedGas         = new(uint64)
-		header          = block.Header()
-		blockHash       = block.Hash()
-		blockNumber     = block.Number()
-		allLogs         []*types.Log
-		gp              = new(GasPool).AddGas(block.GasLimit())
-		firehoseContext = firehose.MaybeSyncContext()
+		usedGas     = new(uint64)
+		header      = block.Header()
+		blockHash   = block.Hash()
+		blockNumber = block.Number()
+		allLogs     []*types.Log
+		gp          = new(GasPool).AddGas(block.GasLimit())
 	)
 
 	if firehoseContext.Enabled() {

core/types.go

@@ -20,6 +20,7 @@ import (
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/core/vm"
+	"github.com/ethereum/go-ethereum/firehose"
 )
 
 // Validator is an interface which defines the standard for block validation. It
@@ -58,5 +59,5 @@ type Processor interface {
 	// Process processes the state changes according to the Ethereum rules by running
 	// the transaction messages using the statedb and applying any rewards to both
 	// the processor (coinbase) and any included uncles.
-	Process(block *types.Block, statedb *state.StateDB, cfg vm.Config) (*state.StateDB, types.Receipts, []*types.Log, uint64, error)
+	Process(block *types.Block, statedb *state.StateDB, cfg vm.Config, firehoseContext *firehose.Context) (*state.StateDB, types.Receipts, []*types.Log, uint64, error)
 }

eth/state_accessor.go

@@ -153,7 +153,7 @@ func (eth *Ethereum) hashState(ctx context.Context, block *types.Block, reexec u
 		if current = eth.blockchain.GetBlockByNumber(next); current == nil {
 			return nil, nil, fmt.Errorf("block #%d not found", next)
 		}
-		statedb, _, _, _, err := eth.blockchain.Processor().Process(current, statedb, vm.Config{})
+		statedb, _, _, _, err := eth.blockchain.Processor().Process(current, statedb, vm.Config{}, firehose.NoOpContext)
 		if err != nil {
 			return nil, nil, fmt.Errorf("processing block %d failed: %v", current.NumberU64(), err)
 		}

firehose/context.go

@@ -116,8 +116,9 @@ func NewSpeculativeExecutionContext(initialAllocationInBytes int) *Context {
 func NewSpeculativeExecutionContextWithBuffer(buffer *bytes.Buffer) *Context {
 	return NewContext(NewToBufferPrinterWithBuffer(buffer), true)
 }
+
 func (ctx *Context) Enabled() bool {
-	return ctx != nil
+	return ctx != nil && Enabled
 }
 
 func (ctx *Context) FirehoseLog() []byte {
@@ -186,6 +187,20 @@ func (ctx *Context) EndBlock(block *types.Block, finalBlockHeader *types.Header,
 		Uint64(uint64(block.Size())),
 		JSON(endData),
 	)
+}
+
+// FlushBlock flushes the accumulated context's printer to "stdout" and reset's the
+// context. If the printer is not a ToBufferPrinter, this is a no-op.
+func (ctx *Context) FlushBlock() {
+	if ctx == nil || !Enabled {
+		return
+	}
+
+	// We flush to stdout only if the received `ctx` accumulated all the Firehose
+	// logs in a buffer. Other context already flushed to stdout.
+	if v, ok := ctx.printer.(*ToBufferPrinter); ok {
+		syncContext.printer.Write(v.buffer.Bytes())
+	}
 
 	ctx.exitBlock()
 }
@@ -507,7 +522,6 @@ func (ctx *Context) StartCall(callType string) {
 		ctx.openCall(),
 		Uint64(ctx.totalOrderingCounter.Inc()),
 	)
-
 }
 
 func (ctx *Context) openCall() string {