一.tendermint檔案結構

• abci-client:Tendermint充當有關一個應用的ABCI客戶端，並且維護3個連線：mempool，consensu和query。
• blockchain：提供儲存，pool（一組peers）以及在peers之間儲存以及交換區塊的reactor。
• consensus：Tendermint core的核心，實現了共識演算法。包括兩個“子模組”：wal（write-ahead logging,預寫式日誌）用於保證資料完整性和replay：宕機恢復後replay區塊和訊息。
• mempool:記憶體池模組會處理所有流入系統的交易，不論它們是來自peers（即peers也可以產生交易）還是應用（應用也可以產生交易，當然主要是應用產生）。
• p2p：圍繞點對點通訊提供了抽象。
• rpc：Tendermint的RPC服務。
• state：表示最近的狀態和執行子模組（根據應用執行區塊)。
• types：一組公開暴露的型別和方法。
• node:tendermint節點的定義

二.tendermint/node分析：

• id.go
  依賴的包:time,crypto。通過crypto生成節點的公鑰和私鑰。id中定義了NodeID，PrivNodeID，NodeGreeting, SignedNodegreeting 四個結構體。

• node.go

1.tendermint/node啟動流程分析

node --proxy_app=dummy --home "/Users/zcy/go/src/github.com/tendermint/tendermint"

2.啟動流程原始碼分析

if c.RunE != nil {
    if err := c.RunE(c, argWoFlags); err != nil {
        return err
    }
} else {
    c.Run(c, argWoFlags) 
}
 

c.RunE(c, argWoFlags)是呼叫匿名函式RunE，而RunE的賦值則是在NewRunNodeCmd函式(/cmd/tendermints/commands/run_node.go)：

// NewRunNodeCmd returns the command that allows the CLI to start a
// node. It can be used with a custom PrivValidator and in-process ABCI application.
func NewRunNodeCmd(nodeProvider nm.NodeProvider) *cobra.Command {
    cmd := &cobra.Command{
        Use:   "node",
        Short: "Run the tendermint node",
        RunE: func(cmd *cobra.Command, args []string) error {   //匿名函式
            // Create & start node
            n, err := nodeProvider(config, logger)
            if err != nil {
                return fmt.Errorf("Failed to create node: %v", err)
            }

        if err := n.Start(); err != nil {
            return fmt.Errorf("Failed to start node: %v", err)
        } else {
            logger.Info("Started node", "nodeInfo", n.Switch().NodeInfo())  //節點啟動完畢列印日誌Started node
        }

        // Trap signal, run forever.
        n.RunForever()

        return nil
    },
}

    AddNodeFlags(cmd)
    return cmd
}

這個匿名函式有三個功能：
1、建立node（nodeProvider(config, logger)）
2、啟動node（n.Start()）
3、監聽停止node的訊號（n.RunForever()）

• 建立node
  nodeProvider(config,logger)具體執行DefaultNewNode函式（/node/node.go#DefaultNewNode）。DefaultNewNode函式會返回一個Node全域性變數。

// DefaultNewNode returns a Tendermint node with default settings for the
// PrivValidator, ClientCreator, GenesisDoc, and DBProvider.
// It implements NodeProvider.
func DefaultNewNode(config *cfg.Config, logger log.Logger) (*Node, error) {
    return NewNode(config,
        types.LoadOrGenPrivValidatorFS(config.PrivValidatorFile()),
        proxy.DefaultClientCreator(config.ProxyApp, config.ABCI, config.DBDir()),
        DefaultGenesisDocProviderFunc(config),
        DefaultDBProvider,
        logger)
}

DefaultNewNode的第一個實參為config。config的定義在/cmd/tendermint/commands/root.go：

var (
    config = cfg.DefaultConfig()
)

注意：使用者在命令列傳入的引數，那只是啟動節點所需的非常小的一部分引數。大多數引數都是需要從預設配置中載入的。config全域性變數載入了node所需的所有預設引數（主要是預設基礎配置、預設RPC配置、預設P2P配置、記憶體池配置、共識配置和交易索引配置）
DefaultNewNode的第二個實參為types.LoadOrGenPrivValidatorFS(config.PrivValidatorFile())，該函式會返回一個PrivValidatorFS例項。

DefaultNewNode的第三個實參為proxy.DefaultClientCreator(config.ProxyApp,config.ABCI,config.DBDir())，其中DefaultClientCreator函式（/proxy/client.go#DefaultClientCreator）的第一個實參config.ProxyApp=“dummy”；
第二個實參config.ABCI=“socket”；第三個實參config.DBDir()

但是具體負責建立這個全域性變數的函式走的是NewNode函式。程式碼在/node/node.go#NewNode。

// NewNode returns a new, ready to go, Tendermint Node.
func NewNode(config *cfg.Config,
    privValidator types.PrivValidator,
    clientCreator proxy.ClientCreator,
    genesisDocProvider GenesisDocProvider,
    dbProvider DBProvider,
    logger log.Logger) (*Node, error) {

    // Get BlockStore
    //初始化blockstore資料庫
    blockStoreDB, err := dbProvider(&DBContext{"blockstore", config})
    if err != nil {
        return nil, err
    }
    blockStore := bc.NewBlockStore(blockStoreDB)

    // Get State
    //初始化state資料庫
    stateDB, err := dbProvider(&DBContext{"state", config})
    if err != nil {
        return nil, err
    }

    // Get genesis doc
    // TODO: move to state package?
    //從硬碟上讀取創世檔案
    genDoc, err := loadGenesisDoc(stateDB)
    if err != nil {
        genDoc, err = genesisDocProvider()
        if err != nil {
            return nil, err
        }
        // save genesis doc to prevent a certain class of user errors (e.g. when it
        // was changed, accidentally or not). Also good for audit trail.
        saveGenesisDoc(stateDB, genDoc)
    }

    state, err := sm.LoadStateFromDBOrGenesisDoc(stateDB, genDoc)
    if err != nil {
        return nil, err
    }

    // Create the proxyApp, which manages connections (consensus, mempool, query)
    // and sync tendermint and the app by performing a handshake
    // and replaying any necessary blocks
    consensusLogger := logger.With("module", "consensus")
    handshaker := cs.NewHandshaker(stateDB, state, blockStore)
    handshaker.SetLogger(consensusLogger)
    proxyApp := proxy.NewAppConns(clientCreator, handshaker)
    proxyApp.SetLogger(logger.With("module", "proxy"))
    //Start()
    if err := proxyApp.Start(); err != nil {
        return nil, fmt.Errorf("Error starting proxy app connections: %v", err)
    }

    // reload the state (it may have been updated by the handshake)
    state = sm.LoadState(stateDB)

    // Decide whether to fast-sync or not
    // We don't fast-sync when the only validator is us.
    fastSync := config.FastSync         //預設開啟快速同步
    if state.Validators.Size() == 1 {
        addr, _ := state.Validators.GetByIndex(0)   //返回驗證人的地址
        if bytes.Equal(privValidator.GetAddress(), addr) {
            fastSync = false         //如果只有一個驗證者，禁用快速同步
        }
    }

    // Log（列印日誌） whether this node is a validator or an observer（觀察者）
    if state.Validators.HasAddress(privValidator.GetAddress()) {
        consensusLogger.Info("This node is a validator", "addr", privValidator.GetAddress(), "pubKey", privValidator.GetPubKey())
    } else {
        consensusLogger.Info("This node is not a validator", "addr", privValidator.GetAddress(), "pubKey", privValidator.GetPubKey())
    }

    // Make MempoolReactor
    mempoolLogger := logger.With("module", "mempool")
    //建立交易池
    mempool := mempl.NewMempool(config.Mempool, proxyApp.Mempool(), state.LastBlockHeight)
    mempool.InitWAL() // no need to have the mempool wal during tests
    mempool.SetLogger(mempoolLogger)
    mempoolReactor := mempl.NewMempoolReactor(config.Mempool, mempool)
    mempoolReactor.SetLogger(mempoolLogger)

    if config.Consensus.WaitForTxs() {
        mempool.EnableTxsAvailable()
    }

    // Make Evidence Reactor
    evidenceDB, err := dbProvider(&DBContext{"evidence", config})
    if err != nil {
        return nil, err
    }
    evidenceLogger := logger.With("module", "evidence")
    evidenceStore := evidence.NewEvidenceStore(evidenceDB)
    evidencePool := evidence.NewEvidencePool(stateDB, evidenceStore)
    evidencePool.SetLogger(evidenceLogger)
    evidenceReactor := evidence.NewEvidenceReactor(evidencePool)
    evidenceReactor.SetLogger(evidenceLogger)

    blockExecLogger := logger.With("module", "state")
    // make block executor for consensus and blockchain reactors to execute blocks
    blockExec := sm.NewBlockExecutor(stateDB, blockExecLogger, proxyApp.Consensus(), mempool, evidencePool)

    // Make BlockchainReactor
    bcReactor := bc.NewBlockchainReactor(state.Copy(), blockExec, blockStore, fastSync)
    bcReactor.SetLogger(logger.With("module", "blockchain"))

    // Make ConsensusReactor
    consensusState := cs.NewConsensusState(config.Consensus, state.Copy(),
        blockExec, blockStore, mempool, evidencePool)
    consensusState.SetLogger(consensusLogger)
    if privValidator != nil {
        consensusState.SetPrivValidator(privValidator)
    }
    consensusReactor := cs.NewConsensusReactor(consensusState, fastSync)
    consensusReactor.SetLogger(consensusLogger)

    p2pLogger := logger.With("module", "p2p")

    sw := p2p.NewSwitch(config.P2P)
    sw.SetLogger(p2pLogger)
    sw.AddReactor("MEMPOOL", mempoolReactor)
    sw.AddReactor("BLOCKCHAIN", bcReactor)
    sw.AddReactor("CONSENSUS", consensusReactor)
    sw.AddReactor("EVIDENCE", evidenceReactor)

    // Optionally, start the pex reactor
    var addrBook pex.AddrBook
    var trustMetricStore *trust.TrustMetricStore
    if config.P2P.PexReactor {
        addrBook = pex.NewAddrBook(config.P2P.AddrBookFile(), config.P2P.AddrBookStrict)
        addrBook.SetLogger(p2pLogger.With("book", config.P2P.AddrBookFile()))

        // Get the trust metric history data
        trustHistoryDB, err := dbProvider(&DBContext{"trusthistory", config})
        if err != nil {
            return nil, err
        }
        trustMetricStore = trust.NewTrustMetricStore(trustHistoryDB, trust.DefaultConfig())
        trustMetricStore.SetLogger(p2pLogger)

        var seeds []string
        if config.P2P.Seeds != "" {
            seeds = strings.Split(config.P2P.Seeds, ",")
        }
        pexReactor := pex.NewPEXReactor(addrBook,
            &pex.PEXReactorConfig{Seeds: seeds, SeedMode: config.P2P.SeedMode})
        pexReactor.SetLogger(p2pLogger)
        sw.AddReactor("PEX", pexReactor)
    }

    // Filter peers by addr or pubkey with an ABCI query.
    // If the query return code is OK, add peer.
    // XXX: Query format subject to change
    if config.FilterPeers {
        // NOTE: addr is ip:port
        sw.SetAddrFilter(func(addr net.Addr) error {
            resQuery, err := proxyApp.Query().QuerySync(abci.RequestQuery{Path: cmn.Fmt("/p2p/filter/addr/%s", addr.String())})
            if err != nil {
                return err
            }
            if resQuery.IsErr() {
                return fmt.Errorf("Error querying abci app: %v", resQuery)
            }
            return nil
        })
        sw.SetPubKeyFilter(func(pubkey crypto.PubKey) error {
            resQuery, err := proxyApp.Query().QuerySync(abci.RequestQuery{Path: cmn.Fmt("/p2p/filter/pubkey/%X", pubkey.Bytes())})
            if err != nil {
                return err
            }
            if resQuery.IsErr() {
                return fmt.Errorf("Error querying abci app: %v", resQuery)
            }
            return nil
        })
    }

    eventBus := types.NewEventBus()
    eventBus.SetLogger(logger.With("module", "events"))

    // services which will be publishing and/or subscribing for messages (events)
    // consensusReactor will set it on consensusState and blockExecutor
    consensusReactor.SetEventBus(eventBus)

    // Transaction indexing
    var txIndexer txindex.TxIndexer
    switch config.TxIndex.Indexer {
    case "kv":
        store, err := dbProvider(&DBContext{"tx_index", config})
        if err != nil {
            return nil, err
        }
        if config.TxIndex.IndexTags != "" {
            txIndexer = kv.NewTxIndex(store, kv.IndexTags(strings.Split(config.TxIndex.IndexTags, ",")))
        } else if config.TxIndex.IndexAllTags {
            txIndexer = kv.NewTxIndex(store, kv.IndexAllTags())
        } else {
            txIndexer = kv.NewTxIndex(store)
        }
    default:
        txIndexer = &null.TxIndex{}
    }

    indexerService := txindex.NewIndexerService(txIndexer, eventBus)

    // run the profile server
    profileHost := config.ProfListenAddress
    if profileHost != "" {
        go func() {
            logger.Error("Profile server", "err", http.ListenAndServe(profileHost, nil))
        }()
    }
    //建立node，並給成員賦值
    node := &Node{
        config:        config,
        genesisDoc:    genDoc,
        privValidator: privValidator,

        sw:               sw,
        addrBook:         addrBook,
        trustMetricStore: trustMetricStore,

        stateDB:          stateDB,
        blockStore:       blockStore,
        bcReactor:        bcReactor,
        mempoolReactor:   mempoolReactor,
        consensusState:   consensusState,
        consensusReactor: consensusReactor,
        evidencePool:     evidencePool,
        proxyApp:         proxyApp,
        txIndexer:        txIndexer,
        indexerService:   indexerService,
        eventBus:         eventBus,
    }
    node.BaseService = *cmn.NewBaseService(logger, "Node", node)
    return node, nil
}

這裡，我們有必要看看Node的定義：

// Node is the highest level interface to a full Tendermint node.
// It includes all configuration information and running services.
type Node struct {
    cmn.BaseService   //內部型別

    // config
    config        *cfg.Config
    genesisDoc    *types.GenesisDoc   // initial validator set
    privValidator types.PrivValidator // local node's validator key

    // network
    sw               *p2p.Switch             // p2p connections
    addrBook         pex.AddrBook            // known peers 已知的peer
    trustMetricStore *trust.TrustMetricStore // trust metrics for all peers

    // services
    eventBus         *types.EventBus // pub/sub for services
    stateDB          dbm.DB
    blockStore       *bc.BlockStore         // store the blockchain to disk
    bcReactor        *bc.BlockchainReactor  // for fast-syncing
    mempoolReactor   *mempl.MempoolReactor  // for gossipping transactions
    consensusState   *cs.ConsensusState     // latest consensus state
    consensusReactor *cs.ConsensusReactor   // for participating in the consensus
    evidencePool     *evidence.EvidencePool // tracking evidence
    proxyApp         proxy.AppConns         // connection to the application
    rpcListeners     []net.Listener         // rpc servers
    txIndexer        txindex.TxIndexer
    indexerService   *txindex.IndexerService
}

• 啟動node
  負責啟動的函式是OnStart，定義在/node/node.go#OnStart()

// OnStart starts the Node. It implements cmn.Service.
func (n *Node) OnStart() error {
    err := n.eventBus.Start()   //複用了BaseService的(bs *BaseService) Start()方法
    if err != nil {
        return err
    }

// Run the RPC server first
// so we can eg. receive txs for the first block
if n.config.RPC.ListenAddress != "" {
    listeners, err := n.startRPC()    //啟動RPC
    if err != nil {
        return err
    }
    n.rpcListeners = listeners
}

// Create & add listener
protocol, address := cmn.ProtocolAndAddress(n.config.P2P.ListenAddress)
l := p2p.NewDefaultListener(protocol, address, n.config.P2P.SkipUPNP, n.Logger.With("module", "p2p"))
n.sw.AddListener(l)

// Generate node PrivKey
// TODO: pass in like priv_val
nodeKey, err := p2p.LoadOrGenNodeKey(n.config.NodeKeyFile())
if err != nil {
    return err
}
n.Logger.Info("P2P Node ID", "ID", nodeKey.ID(), "file", n.config.NodeKeyFile())

// Start the switch
n.sw.SetNodeInfo(n.makeNodeInfo(nodeKey.PubKey()))
n.sw.SetNodeKey(nodeKey)
err = n.sw.Start()
if err != nil {
    return err
}

// Always connect to persistent peers
if n.config.P2P.PersistentPeers != "" {
    err = n.sw.DialPeersAsync(n.addrBook, strings.Split(n.config.P2P.PersistentPeers, ","), true)
    if err != nil {
        return err
    }
}

// start tx indexer
return n.indexerService.Start()
}

• 停止node

TM中node啟動時執行了n.RunForever()，它負責監聽中斷訊號，然後停掉node。

// RunForever waits for an interrupt signal and stops the node.
func (n *Node) RunForever() {
    // Sleep forever and then...
    cmn.TrapSignal(func() {
        n.Stop()              //呼叫BaseService的(bs *BaseService) Stop方法
    })
}

具體負責中斷訊號的是TrapSignal函式(vendor/github.com/tendermint/tmlibs/common/os.go)：

// TrapSignal catches the SIGTERM and executes cb function. After that it exits
// with code 1.
func TrapSignal(cb func()) {
   c := make(chan os.Signal, 1)
   signal.Notify(c, os.Interrupt, syscall.SIGTERM)
   go func() {
      for sig := range c {
         fmt.Printf("captured %v, exiting...\n", sig)
         if cb != nil {
            cb()
         }
         os.Exit(1)
      }
   }()
   select {}
}

TrapSignal函式監聽了SIGTERM訊號。當用戶觸發了ctrl+c才終止node。

具體負責node的停止操作的是OnStop函式（/node/node.go#OnStop()）：

// OnStop stops the Node. It implements cmn.Service.
func (n *Node) OnStop() {
    n.BaseService.OnStop()

    n.Logger.Info("Stopping Node")
    // TODO: gracefully disconnect from peers.
    n.sw.Stop()

    for _, l := range n.rpcListeners {
        n.Logger.Info("Closing rpc listener", "listener", l)
        if err := l.Close(); err != nil {
            n.Logger.Error("Error closing listener", "listener", l, "err", err)
        }
    }

    n.eventBus.Stop()

    n.indexerService.Stop()
}

3.啟動案例分析:
首先，建立了多個應用連線（multiAppConn），具體是三個連線（query，mempool和consensu）。注意：這裡的應用是指在本地以dummy執行。

I[03-25|04:40:48.219] Starting multiAppConn                        module=proxy impl=multiAppConn
I[03-25|04:40:48.219] Starting localClient                         module=abci-client connection=query impl=localClient
I[03-25|04:40:48.219] Starting localClient                         module=abci-client connection=mempool impl=localClient
I[03-25|04:40:48.219] Starting localClient                         module=abci-client connection=consensus impl=localClient

接著Tendermint Core和應用完成了一次握手。

I[03-25|04:40:48.219] ABCI Handshake                               module=consensus appHeight=0 appHash=
I[03-25|04:40:48.219] ABCI Replay Blocks                           module=consensus appHeight=0 storeHeight=0 stateHeight=0
I[03-25|04:40:48.219] Completed ABCI Handshake - Tendermint and App are synced module=consensus appHeight=0 appHash=

之後，開啟了一系列的服務（為正式啟動node做準備），例如事件開關（event switch），reactor以及為了檢測IP地址，完成了UPNP發現。
“Started node”訊息預示著node啟動完畢，隨時準備區塊建立。

I[03-25|04:40:51.248] Started node                                module=main nodeInfo="NodeInfo{pk: {PubKeyEd25519{54E665EFA8EF7AD5763C421129F66FBD367B253788543C0E1B9CF047C45771D3}}, moniker: bootnode-hongkong, network: test-chain-MErM3n [listen 172.31.22.212:46656], version: 0.16.0 ([wire_version=0.7.2 p2p_version=0.5.0 consensus_version=v1/0.2.2 rpc_version=0.7.0/3 tx_index=on rpc_addr=tcp://0.0.0.0:46657])}"

接著是一個標準的區塊建立過程，即進入到新的一輪中，提案一個區塊，接收超過2/3的預投票（prevote），接著預提交（precommit）並且最終提交一個區塊(finalise)。

I[03-25|04:40:51.250] enterNewRound(1/0). Current: 1/0/RoundStepNewHeight module=consensus
I[03-25|04:40:51.250] enterPropose(1/0). Current: 1/0/RoundStepNewRound module=consensus
I[03-25|04:40:51.250] enterPropose: Our turn to propose            module=consensus proposer=BD8F0160796AEC1E6454C55CCA6AF15EACF94A2D privValidator="PrivValidator{BD8F0160796AEC1E6454C55CCA6AF15EACF94A2D LH:0, LR:0, LS:0}"
I[03-25|04:40:51.253] Signed proposal                              module=consensus height=1 round=0 proposal="Proposal{1/0 1:57D9BC19067F (-1,:0:000000000000) {/3FB2AD239E88.../} @ 2018-03-25T04:40:51.250Z}"
I[03-25|04:40:51.259] Received complete proposal block             module=consensus height=1 hash=2CFA5BC56A2E5D0BB94B84EB35AD05CC8F47CC49
I[03-25|04:40:51.259] enterPrevote(1/0). Current: 1/0/RoundStepPropose module=consensus
I[03-25|04:40:51.259] enterPrevote: ProposalBlock is valid         module=consensus height=1 round=0
I[03-25|04:40:51.261] Signed and pushed vote                       module=consensus height=1 round=0 vote="Vote{0:BD8F0160796A 1/00/1(Prevote) 2CFA5BC56A2E {/347B4DC16274.../} @ 2018-03-25T04:40:51.259Z}" err=null
I[03-25|04:40:51.265] Added to prevote                             module=consensus vote="Vote{0:BD8F0160796A 1/00/1(Prevote) 2CFA5BC56A2E {/347B4DC16274.../} @ 2018-03-25T04:40:51.259Z}" prevotes="VoteSet{H:1 R:0 T:1 +2/3:2CFA5BC56A2E5D0BB94B84EB35AD05CC8F47CC49:1:57D9BC19067F BA{1:X} map[]}"
I[03-25|04:40:51.265] enterPrecommit(1/0). Current: 1/0/RoundStepPrevote module=consensus
I[03-25|04:40:51.265] enterPrecommit: +2/3 prevoted proposal block. Locking module=consensus hash=2CFA5BC56A2E5D0BB94B84EB35AD05CC8F47CC49
I[03-25|04:40:51.267] Signed and pushed vote                       module=consensus height=1 round=0 vote="Vote{0:BD8F0160796A 1/00/2(Precommit) 2CFA5BC56A2E {/1AB356C41018.../} @ 2018-03-25T04:40:51.265Z}" err=null
I[03-25|04:40:51.271] Added to precommit                           module=consensus vote="Vote{0:BD8F0160796A 1/00/2(Precommit) 2CFA5BC56A2E {/1AB356C41018.../} @ 2018-03-25T04:40:51.265Z}" precommits="VoteSet{H:1 R:0 T:2 +2/3:2CFA5BC56A2E5D0BB94B84EB35AD05CC8F47CC49:1:57D9BC19067F BA{1:X} map[]}"
I[03-25|04:40:51.271] enterCommit(1/0). Current: 1/0/RoundStepPrecommit module=consensus
I[03-25|04:40:51.274] Finalizing commit of block with 0 txs        module=consensus height=1 hash=2CFA5BC56A2E5D0BB94B84EB35AD05CC8F47CC49 root=
I[03-25|04:40:51.274] Block{
  Header{
    ChainID:        test-chain-MErM3n
    Height:         1
    Time:           2018-03-25 12:40:51.25 +0800 CST
    NumTxs:         0
    TotalTxs:       0
    LastBlockID:    :0:000000000000
    LastCommit:
    Data:
    Validators:     7DBC0AC152D72CA2DA37CAB8D6D5A73B8DA9DC43
    App:
    Conensus:       0B8CEF95EC57AC2D96038FD0AE3901C14FAE8E73
    Results:
    Evidence:
  }#2CFA5BC56A2E5D0BB94B84EB35AD05CC8F47CC49
  Data{
  }#
  Data{
  }#
  Commit{
    BlockID:    :0:000000000000
    Precommits:
  }#
}#2CFA5BC56A2E5D0BB94B84EB35AD05CC8F47CC49 module=consensus
I[03-25|04:40:51.279] Executed block                               module=state height=1 validTxs=0 invalidTxs=0
I[03-25|04:40:51.281] Committed state                              module=state height=1 txs=0 appHash=0000000000000000
I[03-25|04:40:51.281] Recheck txs                                  module=mempool numtxs=0 height=1