Docker CE 18.03原始碼分析

摘要： Docker更新的速度太快了。網上很多分析已經過時了。今天我翻了一下Docker的原始碼，發現其實Docker的原始碼挺 簡單易懂的。當然前提是，你已經熟練掌握了Docker，經常用，知道有哪些概念，瞭解其原理。有了這些做基礎， 就很容易讀了。 建議先讀一下這篇文章：自己寫一個容器 和 ...

Docker更新的速度太快了。網上很多分析已經過時了。今天我翻了一下Docker的原始碼，發現其實Docker的原始碼挺 簡單易懂的。當然前提是，你已經熟練掌握了Docker，經常用，知道有哪些概念，瞭解其原理。有了這些做基礎， 就很容易讀了。

建議先讀一下這篇文章：自己寫一個容器 和ofollow,noindex" target="_blank">這個專案 的原始碼。

目錄結構

$ tree -d -L 1
.
├── api
├── builder
├── cli
├── client
├── cmd# 命令列的入口，我們就要從這裡跳進去看
├── container# 容器的抽象
├── contrib# 雜物堆，啥雜七雜八的都往這裡丟
├── daemon# 今天的主角，dockerd這個daemon
├── distribution# 看起來發布相關的
├── dockerversion
├── docs# 文件
├── errdefs# 一些常見的錯誤
├── hack
├── image# 映象的抽象概念
├── integration
├── integration-cli
├── internal
├── layer# 層。怎麼說呢，就是對各種layer fs的抽象
├── libcontainerd
├── migrate
├── oci# https://blog.docker.com/2017/07/demystifying-open-container-initiative-oci-specifications/ 容器標準庫相關
├── opts# 一些配置和配置校驗相關的
├── pkg# 類似於我們平時寫的utils或者helpers等
├── plugin# 外掛相關的東西
├── profiles
├── project
├── reference
├── registry
├── reports
├── restartmanager# 負責容器的重啟，比如是否設定了"always"呀
├── runconfig
├── vendor# go的vendor機制
└── volume# volume相關

• 負責儲存配置的一般都叫xx Store
• Docker的設計是單機的，不是分散式的
• Docker的設計是Client-Server模式的，平時我們用的docker這個命令被分散到https://github.com/docker/cli 這個倉庫去了

從命令列進入

入口在cmd/dockerd/docker.go :

func newDaemonCommand() *cobra.Command {
opts := newDaemonOptions(config.New())

cmd := &cobra.Command{
Use:"dockerd [OPTIONS]",
Short:"A self-sufficient runtime for containers.",
SilenceUsage:true,
SilenceErrors: true,
Args:cli.NoArgs,
RunE: func(cmd *cobra.Command, args []string) error {
if opts.version {
showVersion()
return nil
}
opts.flags = cmd.Flags()
return runDaemon(opts) // 真正的入口
},
}
cli.SetupRootCommand(cmd)

flags := cmd.Flags()
flags.BoolVarP(&opts.version, "version", "v", false, "Print version information and quit")
flags.StringVar(&opts.configFile, "config-file", defaultDaemonConfigFile, "Daemon configuration file")
opts.InstallFlags(flags)
installConfigFlags(opts.daemonConfig, flags)
installServiceFlags(flags)

return cmd
}

然後跳到cmd/dockerd/daemon.go ：

func (cli *DaemonCli) start(opts *daemonOptions) (err error) {
stopc := make(chan bool)
defer close(stopc)

// warn from uuid package when running the daemon
uuid.Loggerf = logrus.Warnf

// 設定一些預設配置例如TLS啊blabla
opts.SetDefaultOptions(opts.flags)

// 載入配置
if cli.Config, err = loadDaemonCliConfig(opts); err != nil {
return err
}
cli.configFile = &opts.configFile
cli.flags = opts.flags

if cli.Config.Debug {
debug.Enable()
}

if cli.Config.Experimental {
logrus.Warn("Running experimental build")
}

logrus.SetFormatter(&logrus.TextFormatter{
TimestampFormat: jsonmessage.RFC3339NanoFixed,
DisableColors:cli.Config.RawLogs,
FullTimestamp:true,
})

// LCOW: Linux Containers On Windows. ref: https://blog.docker.com/2017/09/preview-linux-containers-on-windows/
system.InitLCOW(cli.Config.Experimental)

// 設定預設的umask。022。也就是說，建立的檔案許可權都是755
if err := setDefaultUmask(); err != nil {
return fmt.Errorf("Failed to set umask: %v", err)
}

// Create the daemon root before we create ANY other files (PID, or migrate keys)
// to ensure the appropriate ACL is set (particularly relevant on Windows)
// 設定daemon的執行時候的根目錄
if err := daemon.CreateDaemonRoot(cli.Config); err != nil {
return err
}

if cli.Pidfile != "" {
pf, err := pidfile.New(cli.Pidfile)
if err != nil {
return fmt.Errorf("Error starting daemon: %v", err)
}
defer func() {
if err := pf.Remove(); err != nil {
logrus.Error(err)
}
}()
}

// TODO: extract to newApiServerConfig()
serverConfig := &apiserver.Config{
Logging:true,
SocketGroup: cli.Config.SocketGroup,
Version:dockerversion.Version,
CorsHeaders: cli.Config.CorsHeaders,
}

// 是否走TLS
if cli.Config.TLS {
tlsOptions := tlsconfig.Options{
CAFile:cli.Config.CommonTLSOptions.CAFile,
CertFile:cli.Config.CommonTLSOptions.CertFile,
KeyFile:cli.Config.CommonTLSOptions.KeyFile,
ExclusiveRootPools: true,
}

if cli.Config.TLSVerify {
// server requires and verifies client's certificate
tlsOptions.ClientAuth = tls.RequireAndVerifyClientCert
}
tlsConfig, err := tlsconfig.Server(tlsOptions)
if err != nil {
return err
}
serverConfig.TLSConfig = tlsConfig
}

if len(cli.Config.Hosts) == 0 {
cli.Config.Hosts = make([]string, 1)
}

cli.api = apiserver.New(serverConfig)

var hosts []string

// 設定API server
for i := 0; i < len(cli.Config.Hosts); i++ {
var err error
if cli.Config.Hosts[i], err = dopts.ParseHost(cli.Config.TLS, cli.Config.Hosts[i]); err != nil {
return fmt.Errorf("error parsing -H %s : %v", cli.Config.Hosts[i], err)
}

protoAddr := cli.Config.Hosts[i]
protoAddrParts := strings.SplitN(protoAddr, "://", 2)
if len(protoAddrParts) != 2 {
return fmt.Errorf("bad format %s, expected PROTO://ADDR", protoAddr)
}

proto := protoAddrParts[0]
addr := protoAddrParts[1]

// It's a bad idea to bind to TCP without tlsverify.
if proto == "tcp" && (serverConfig.TLSConfig == nil || serverConfig.TLSConfig.ClientAuth != tls.RequireAndVerifyClientCert) {
logrus.Warn("[!] DON'T BIND ON ANY IP ADDRESS WITHOUT setting --tlsverify IF YOU DON'T KNOW WHAT YOU'RE DOING [!]")
}
ls, err := listeners.Init(proto, addr, serverConfig.SocketGroup, serverConfig.TLSConfig)
if err != nil {
return err
}
ls = wrapListeners(proto, ls)
// If we're binding to a TCP port, make sure that a container doesn't try to use it.
if proto == "tcp" {
if err := allocateDaemonPort(addr); err != nil {
return err
}
}
logrus.Debugf("Listener created for HTTP on %s (%s)", proto, addr)
hosts = append(hosts, protoAddrParts[1])
cli.api.Accept(addr, ls...)
}

registryService, err := registry.NewService(cli.Config.ServiceOptions)
if err != nil {
return err
}

rOpts, err := cli.getRemoteOptions()
if err != nil {
return fmt.Errorf("Failed to generate containerd options: %s", err)
}
containerdRemote, err := libcontainerd.New(filepath.Join(cli.Config.Root, "containerd"), filepath.Join(cli.Config.ExecRoot, "containerd"), rOpts...)
if err != nil {
return err
}
signal.Trap(func() {
cli.stop()
<-stopc // wait for daemonCli.start() to return
}, logrus.StandardLogger())

// Notify that the API is active, but before daemon is set up.
preNotifySystem()

pluginStore := plugin.NewStore()

// 初始化中介軟體
if err := cli.initMiddlewares(cli.api, serverConfig, pluginStore); err != nil {
logrus.Fatalf("Error creating middlewares: %v", err)
}

// 例項化daemon
d, err := daemon.NewDaemon(cli.Config, registryService, containerdRemote, pluginStore)
if err != nil {
return fmt.Errorf("Error starting daemon: %v", err)
}

d.StoreHosts(hosts)

// validate after NewDaemon has restored enabled plugins. Dont change order.
if err := validateAuthzPlugins(cli.Config.AuthorizationPlugins, pluginStore); err != nil {
return fmt.Errorf("Error validating authorization plugin: %v", err)
}

// TODO: move into startMetricsServer()
if cli.Config.MetricsAddress != "" {
if !d.HasExperimental() {
return fmt.Errorf("metrics-addr is only supported when experimental is enabled")
}
if err := startMetricsServer(cli.Config.MetricsAddress); err != nil {
return err
}
}

// TODO: createAndStartCluster()
name, _ := os.Hostname()

// Use a buffered channel to pass changes from store watch API to daemon
// A buffer allows store watch API and daemon processing to not wait for each other
watchStream := make(chan *swarmapi.WatchMessage, 32)

// 預設就初始化Docker Swarm？
c, err := cluster.New(cluster.Config{
Root:cli.Config.Root,
Name:name,
Backend:d,
ImageBackend:d.ImageService(),
PluginBackend:d.PluginManager(),
NetworkSubnetsProvider: d,
DefaultAdvertiseAddr:cli.Config.SwarmDefaultAdvertiseAddr,
RaftHeartbeatTick:cli.Config.SwarmRaftHeartbeatTick,
RaftElectionTick:cli.Config.SwarmRaftElectionTick,
RuntimeRoot:cli.getSwarmRunRoot(),
WatchStream:watchStream,
})
if err != nil {
logrus.Fatalf("Error creating cluster component: %v", err)
}
d.SetCluster(c)
err = c.Start()
if err != nil {
logrus.Fatalf("Error starting cluster component: %v", err)
}

// Restart all autostart containers which has a swarm endpoint
// and is not yet running now that we have successfully
// initialized the cluster.
d.RestartSwarmContainers()

logrus.Info("Daemon has completed initialization")

cli.d = d

// 註冊handler到router
routerOptions, err := newRouterOptions(cli.Config, d)
if err != nil {
return err
}
routerOptions.api = cli.api
routerOptions.cluster = c

// 初始化router
initRouter(routerOptions)

// process cluster change notifications
watchCtx, cancel := context.WithCancel(context.Background())
defer cancel()
go d.ProcessClusterNotifications(watchCtx, watchStream)

cli.setupConfigReloadTrap()

// The serve API routine never exits unless an error occurs
// We need to start it as a goroutine and wait on it so
// daemon doesn't exit
serveAPIWait := make(chan error)
// 開始執行
go cli.api.Wait(serveAPIWait)

// after the daemon is done setting up we can notify systemd api
// 還要通知systemd？
notifySystem()

// Daemon is fully initialized and handling API traffic
// Wait for serve API to complete
// api server停了，daemon就跟著退出
errAPI := <-serveAPIWait
c.Cleanup()
shutdownDaemon(d)
containerdRemote.Cleanup()
if errAPI != nil {
return fmt.Errorf("Shutting down due to ServeAPI error: %v", errAPI)
}

return nil
}

這裡做的事情可就多了，載入配置，設定相關的變數和配置，監聽埠，設定訊號handler， 起了API server，等待接受請求。

建立容器

上面我們看到了daemon的啟動過程。啟動完之後就等待請求了。那我們要找一個新的入口點去跟蹤 程式碼，所以我選擇docker run 。從docker/cli 庫翻了翻，發現最後是呼叫containers/create 這樣一個介面。然後就在本倉庫裡搜尋，我們的目標是找到處理這個請求的mux所在地，然後翻出對應的 handlerapi/server/router/container/container.go ，然後跳到api/server/router/container/container_routes.go :

ccr, err := s.backend.ContainerCreate(types.ContainerCreateCo
nfig{
Name:name,
Config:config,
HostConfig:hostConfig,
NetworkingConfig: networkingConfig,
AdjustCPUShares:adjustCPUShares,
})

跳進去，發現是個介面。而s.backend 其實就是daemon。具體程式碼在上一節就可以跟到。所以我開啟fzf進行 泛搜尋func daemon ContainerCreate ，找到了daemon/create.go ：

// ContainerCreate creates a regular container
// 建立一個容器
func (daemon *Daemon) ContainerCreate(params types.ContainerCreateConfig) (containertypes.ContainerCreateCreatedBody, error) {
return daemon.containerCreate(params, false)
}

func (daemon *Daemon) containerCreate(params types.ContainerCreateConfig, managed bool) (containertypes.ContainerCreateCreatedBody, error) {
start := time.Now()
if params.Config == nil {
return containertypes.ContainerCreateCreatedBody{}, errdefs.InvalidParameter(errors.New("Config cannot be empty in order to create a container"))
}

os := runtime.GOOS // 不知道這個是幹啥的。。。Windows和Linux怎麼混來混去的。。。
if params.Config.Image != "" {
// 拉取映象
img, err := daemon.imageService.GetImage(params.Config.Image)
if err == nil {
os = img.OS
}
} else {
// This mean scratch. On Windows, we can safely assume that this is a linux
// container. On other platforms, it's the host OS (which it already is)
if runtime.GOOS == "windows" && system.LCOWSupported() {
os = "linux"
}
}

// 驗證容器的配置，如果有問題，就不能建立
warnings, err := daemon.verifyContainerSettings(os, params.HostConfig, params.Config, false)
if err != nil {
return containertypes.ContainerCreateCreatedBody{Warnings: warnings}, errdefs.InvalidParameter(err)
}

// 驗證網路配置
err = verifyNetworkingConfig(params.NetworkingConfig)
if err != nil {
return containertypes.ContainerCreateCreatedBody{Warnings: warnings}, errdefs.InvalidParameter(err)
}

if params.HostConfig == nil {
params.HostConfig = &containertypes.HostConfig{}
}
// 調整一些配置，例如CPU如果超量了，就設定成系統允許的最大的。等。
err = daemon.adaptContainerSettings(params.HostConfig, params.AdjustCPUShares)
if err != nil {
return containertypes.ContainerCreateCreatedBody{Warnings: warnings}, errdefs.InvalidParameter(err)
}

// 建立容器
container, err := daemon.create(params, managed)
if err != nil {
return containertypes.ContainerCreateCreatedBody{Warnings: warnings}, err
}
containerActions.WithValues("create").UpdateSince(start)

return containertypes.ContainerCreateCreatedBody{ID: container.ID, Warnings: warnings}, nil
}

// Create creates a new container from the given configuration with a given name.
// 建立容器
func (daemon *Daemon) create(params types.ContainerCreateConfig, managed bool) (retC *container.Container, retErr error) {
var (
container *container.Container
img*image.Image
imgIDimage.ID
errerror
)

os := runtime.GOOS
if params.Config.Image != "" {
img, err = daemon.imageService.GetImage(params.Config.Image)
if err != nil {
return nil, err
}
if img.OS != "" {
os = img.OS
} else {
// default to the host OS except on Windows with LCOW
if runtime.GOOS == "windows" && system.LCOWSupported() {
os = "linux"
}
}
imgID = img.ID()

if runtime.GOOS == "windows" && img.OS == "linux" && !system.LCOWSupported() {
return nil, errors.New("operating system on which parent image was created is not Windows")
}
} else {
// 沒搞懂這個分支在這幹啥。。。
if runtime.GOOS == "windows" {
os = "linux" // 'scratch' case.
}
}

// 再次檢查配置
if err := daemon.mergeAndVerifyConfig(params.Config, img); err != nil {
return nil, errdefs.InvalidParameter(err)
}

if err := daemon.mergeAndVerifyLogConfig(¶ms.HostConfig.LogConfig); err != nil {
return nil, errdefs.InvalidParameter(err)
}

// 建立容器。。。。這個巢狀的有點多。。。此處返回的是記憶體中對容器的一個抽象 `Container`
if container, err = daemon.newContainer(params.Name, os, params.Config, params.HostConfig, imgID, managed); err != nil {
return nil, err
}
defer func() {
if retErr != nil {
if err := daemon.cleanupContainer(container, true, true); err != nil {
logrus.Errorf("failed to cleanup container on create error: %v", err)
}
}
}()

if err := daemon.setSecurityOptions(container, params.HostConfig); err != nil {
return nil, err
}

container.HostConfig.StorageOpt = params.HostConfig.StorageOpt

// Fixes: https://github.com/moby/moby/issues/34074 and
// https://github.com/docker/for-win/issues/999.
// Merge the daemon's storage options if they aren't already present. We only
// do this on Windows as there's no effective sandbox size limit other than
// physical on Linux.
if runtime.GOOS == "windows" {
if container.HostConfig.StorageOpt == nil {
container.HostConfig.StorageOpt = make(map[string]string)
}
for _, v := range daemon.configStore.GraphOptions {
opt := strings.SplitN(v, "=", 2)
if _, ok := container.HostConfig.StorageOpt[opt[0]]; !ok {
container.HostConfig.StorageOpt[opt[0]] = opt[1]
}
}
}

// Set RWLayer for container after mount labels have been set
// 要建立一個RWLayer，這樣容器裡面才可以讀寫。前面的layer都包含在image裡。
rwLayer, err := daemon.imageService.CreateLayer(container, setupInitLayer(daemon.idMappings))
if err != nil {
return nil, errdefs.System(err)
}
container.RWLayer = rwLayer

rootIDs := daemon.idMappings.RootPair()
if err := idtools.MkdirAndChown(container.Root, 0700, rootIDs); err != nil {
return nil, err
}
if err := idtools.MkdirAndChown(container.CheckpointDir(), 0700, rootIDs); err != nil {
return nil, err
}

if err := daemon.setHostConfig(container, params.HostConfig); err != nil {
return nil, err
}

if err := daemon.createContainerOSSpecificSettings(container, params.Config, params.HostConfig); err != nil {
return nil, err
}

var endpointsConfigs map[string]*networktypes.EndpointSettings
if params.NetworkingConfig != nil {
endpointsConfigs = params.NetworkingConfig.EndpointsConfig
}
// Make sure NetworkMode has an acceptable value. We do this to ensure
// backwards API compatibility.
runconfig.SetDefaultNetModeIfBlank(container.HostConfig)

daemon.updateContainerNetworkSettings(container, endpointsConfigs)
if err := daemon.Register(container); err != nil {
return nil, err
}
stateCtr.set(container.ID, "stopped")
daemon.LogContainerEvent(container, "create")
return container, nil
}

然後跟到daemon.create ，仔細看，然後繼續跟到了daemon.newContainer ：

func (daemon *Daemon) newContainer(name string, operatingSystem string, config *containertypes.Config, hostConfig *containertypes.HostConfig, imgID image.ID, managed bool) (*container.Container, error) {
var (
idstring
errerror
noExplicitName = name == ""
)
id, name, err = daemon.generateIDAndName(name)
if err != nil {
return nil, err
}

if hostConfig.NetworkMode.IsHost() {
if config.Hostname == "" {
config.Hostname, err = os.Hostname()
if err != nil {
return nil, errdefs.System(err)
}
}
} else {
// 原來容器的hostname預設是容器id的前12位
daemon.generateHostname(id, config)
}
entrypoint, args := daemon.getEntrypointAndArgs(config.Entrypoint, config.Cmd)

base := daemon.newBaseContainer(id) // 瞧。。。又一個建立容器
base.Created = time.Now().UTC()
base.Managed = managed
base.Path = entrypoint
base.Args = args //FIXME: de-duplicate from config
base.Config = config
base.HostConfig = &containertypes.HostConfig{}
base.ImageID = imgID
base.NetworkSettings = &network.Settings{IsAnonymousEndpoint: noExplicitName}
base.Name = name
base.Driver = daemon.imageService.GraphDriverForOS(operatingSystem)
base.OS = operatingSystem
return base, err
}

整個流程就跟完了。