// kubelet 启动
// cmd/kubelet/kubelet.go:34
func main() {
	command := app.NewKubeletCommand()
	code := cli.Run(command)
	os.Exit(code)
}

↓↓↓
// cmd/kubelet/app/server.go:264
// run the kubelet
return Run(ctx, kubeletServer, kubeletDeps, utilfeature.DefaultFeatureGate)

↓↓↓
// cmd/kubelet/app/server.go:419
if err := run(ctx, s, kubeDeps, featureGate); err != nil {
		return fmt.Errorf("failed to run Kubelet: %w", err)
	}

↓↓↓
// cmd/kubelet/app/server.go:783
if err := RunKubelet(s, kubeDeps, s.RunOnce); err != nil {
		return err
	}

↓↓↓
// 此处最终 kubelet 实例 得到 statsProvider，在 1185 行 的 startKubelet 启动 kubelet server（包含 /stats/summary api）
// 点进去createAndInitKubelet 函数，可以看到 kubelet.NewMainKubelet() 函数，在这个函数内可以获取到 kubelet 最终选择的 StatsProvider
// cmd/kubelet/app/server.go:1157
k, err := createAndInitKubelet(kubeServer,
		kubeDeps,
		hostname,
		hostnameOverridden,
		nodeName,
		nodeIPs)
	if err != nil {
		return fmt.Errorf("failed to create kubelet: %w", err)
	}


↓↓↓
// 
// cmd/kubelet/app/server.go:1207
k, err = kubelet.NewMainKubelet(&kubeServer.KubeletConfiguration,
		kubeDeps,
		&kubeServer.ContainerRuntimeOptions,
		hostname,
		hostnameOverridden,
		nodeName,
		nodeIPs,
		kubeServer.ProviderID,
		kubeServer.CloudProvider,
		kubeServer.CertDirectory,
		kubeServer.RootDirectory,
		kubeServer.ImageCredentialProviderConfigFile,
		kubeServer.ImageCredentialProviderBinDir,
		kubeServer.RegisterNode,
		kubeServer.RegisterWithTaints,
		kubeServer.AllowedUnsafeSysctls,
		kubeServer.ExperimentalMounterPath,
		kubeServer.KernelMemcgNotification,
		kubeServer.ExperimentalNodeAllocatableIgnoreEvictionThreshold,
		kubeServer.MinimumGCAge,
		kubeServer.MaxPerPodContainerCount,
		kubeServer.MaxContainerCount,
		kubeServer.MasterServiceNamespace,
		kubeServer.RegisterSchedulable,
		kubeServer.KeepTerminatedPodVolumes,
		kubeServer.NodeLabels,
		kubeServer.NodeStatusMaxImages,
		kubeServer.KubeletFlags.SeccompDefault || kubeServer.KubeletConfiguration.SeccompDefault)
	if err != nil {
		return nil, err
	}
...

// 此处给 kubelet 赋值 statsProvider，选择 cAdvisor 还是 CRI 作为 StatsProvider
// 如果 runtime 是 CRI-O，那么使用 CadvisorStatsProvider
// 否则使用 CRIStatsProvider （因为现在  从CRI获取所有的 stats还没有完全支持，所以其实使用CRIStatsProvider 就是使用 CRI 和 cadvisor 混杂数据的模式。具体可看 KEP）
// 如果kubelet 又启用了 PodAndContainerStatsFromCRI 特性门控，那么就不使用cAdvisor stats 填充 CRI 不存在的数据
// pkg/kubelet/kubelet.go:697
if kubeDeps.useLegacyCadvisorStats {
		klet.StatsProvider = stats.NewCadvisorStatsProvider(
			klet.cadvisor,
			klet.resourceAnalyzer,
			klet.podManager,
			klet.runtimeCache,
			klet.containerRuntime,
			klet.statusManager,
			hostStatsProvider)
	} else {
		klet.StatsProvider = stats.NewCRIStatsProvider(
			klet.cadvisor,
			klet.resourceAnalyzer,
			klet.podManager,
			klet.runtimeCache,
			kubeDeps.RemoteRuntimeService,
			kubeDeps.RemoteImageService,
			hostStatsProvider,
			utilfeature.DefaultFeatureGate.Enabled(features.PodAndContainerStatsFromCRI))
	}


↓↓↓
// cmd/kubelet/app/server.go:1185
// k 变量包含 kubelet stats provider 参数
startKubelet(k, podCfg, &kubeServer.KubeletConfiguration, kubeDeps, kubeServer.EnableServer)
		klog.InfoS("Started kubelet")

// k 变量包含 kubelet stats provider 参数
// 在1157 行运行createAndInitKubelet函数后 kubelet已经得到 statsProvider
// cmd/kubelet/app/server.go:1185
startKubelet(k, podCfg, &kubeServer.KubeletConfiguration, kubeDeps, kubeServer.EnableServer)

↓↓↓
// cmd/kubelet/app/server.go:1195
// start the kubelet server
	if enableServer {
		go k.ListenAndServe(kubeCfg, kubeDeps.TLSOptions, kubeDeps.Auth, kubeDeps.TracerProvider)
	}

↓↓↓
// pkg/kubelet/kubelet.go:2513
server.ListenAndServeKubeletServer(kl, kl.resourceAnalyzer, kubeCfg, tlsOptions, auth, tp)

↓↓↓
// pkg/kubelet/server/server.go:162
handler := NewServer(host, resourceAnalyzer, auth, tp, kubeCfg)

↓↓↓
// pkg/kubelet/server/server.go:279
server.InstallDefaultHandlers()

↓↓↓
// pkg/kubelet/server/server.go:382
s.addMetricsBucketMatcher("stats")
	s.restfulCont.Add(stats.CreateHandlers(statsPath, s.host, s.resourceAnalyzer))

↓↓↓
// pkg/kubelet/server/stats/handler.go:122
endpoints := []struct {
		path    string
		handler restful.RouteFunction
	}{
		{"/summary", h.handleSummary},
	}

↓↓↓
// pkg/kubelet/server/stats/handler.go:143
// Handles stats summary requests to /stats/summary
// If "only_cpu_and_memory" GET param is true then only cpu and memory is returned in response.
func (h *handler) handleSummary(request *restful.Request, response *restful.Response) {
	ctx := request.Request.Context()
	onlyCPUAndMemory := false
	err := request.Request.ParseForm()
	if err != nil {
		handleError(response, "/stats/summary", fmt.Errorf("parse form failed: %w", err))
		return
	}
	if onlyCluAndMemoryParam, found := request.Request.Form["only_cpu_and_memory"]; found &&
		len(onlyCluAndMemoryParam) == 1 && onlyCluAndMemoryParam[0] == "true" {
		onlyCPUAndMemory = true
	}
	var summary *statsapi.Summary
	if onlyCPUAndMemory {
    //在此处summaryProvider调用会得到 cpu 和 memory 统计数据
		summary, err = h.summaryProvider.GetCPUAndMemoryStats(ctx)
	} else {
		// external calls to the summary API use cached stats
		forceStatsUpdate := false
    // 在此处summaryProvider调用会得到所有统计数据
		summary, err = h.summaryProvider.Get(ctx, forceStatsUpdate)
	}
	if err != nil {
		handleError(response, "/stats/summary", err)
	} else {
		writeResponse(response, summary)
	}
}

// pkg/kubelet/server/stats/summary.go:32
// SummaryProvider provides summaries of the stats from Kubelet.
type SummaryProvider interface {
	// Get provides a new Summary with the stats from Kubelet,
	// and will update some stats if updateStats is true
	Get(ctx context.Context, updateStats bool) (*statsapi.Summary, error)
	// GetCPUAndMemoryStats provides a new Summary with the CPU and memory stats from Kubelet,
	GetCPUAndMemoryStats(ctx context.Context) (*statsapi.Summary, error)
}


↓↓↓
// 在这里会得到 nodeStats 和 podStats
// pkg/kubelet/server/stats/summary.go:122
func (sp *summaryProviderImpl) GetCPUAndMemoryStats(ctx context.Context) (*statsapi.Summary, error) {
	// TODO(timstclair): Consider returning a best-effort response if any of
	// the following errors occur.
	node, err := sp.provider.GetNode()
	if err != nil {
		return nil, fmt.Errorf("failed to get node info: %v", err)
	}
	nodeConfig := sp.provider.GetNodeConfig()
	rootStats, err := sp.provider.GetCgroupCPUAndMemoryStats("/", false)
	if err != nil {
		return nil, fmt.Errorf("failed to get root cgroup stats: %v", err)
	}

  // 得到 podStats
	podStats, err := sp.provider.ListPodCPUAndMemoryStats(ctx)
	if err != nil {
		return nil, fmt.Errorf("failed to list pod stats: %v", err)
	}

	nodeStats := statsapi.NodeStats{
		NodeName:         node.Name,
		CPU:              rootStats.CPU,
		Memory:           rootStats.Memory,
		StartTime:        rootStats.StartTime,
		SystemContainers: sp.GetSystemContainersCPUAndMemoryStats(nodeConfig, podStats, false),
	}
	summary := statsapi.Summary{
		Node: nodeStats,
		Pods: podStats,
	}
	return &summary, nil
}

↓↓↓↓↓↓
// Provider 接口会定义所有的 stats 处理方法，其中 pod stats 是调用 kubelet StatsProvider结构体（pkg/kubelet/kubelet.go:1232）的方法来实现的，可看下面的 pkg/kubelet/kubelet.go:1262 行
// pkg/kubelet/server/stats/handler.go:39
// Provider hosts methods required by stats handlers.
type Provider interface {
	// The following stats are provided by either CRI or cAdvisor.
	//
	// ListPodStats returns the stats of all the containers managed by pods.
	ListPodStats(ctx context.Context) ([]statsapi.PodStats, error)
	// ListPodStatsAndUpdateCPUNanoCoreUsage updates the cpu nano core usage for
	// the containers and returns the stats for all the pod-managed containers.
	ListPodCPUAndMemoryStats(ctx context.Context) ([]statsapi.PodStats, error)
	// ListPodStatsAndUpdateCPUNanoCoreUsage returns the stats of all the
	// containers managed by pods and force update the cpu usageNanoCores.
	// This is a workaround for CRI runtimes that do not integrate with
	// cadvisor. See https://github.com/kubernetes/kubernetes/issues/72788
	// for more details.
  ...
}

↓↓↓
// stats provider
// Provider provides the stats of the node and the pod-managed containers.
type Provider struct {
	cadvisor     cadvisor.Interface
	podManager   kubepod.Manager
	runtimeCache kubecontainer.RuntimeCache
	containerStatsProvider
	rlimitStatsProvider
}

// containerStatsProvider is an interface that provides the stats of the
// containers managed by pods.
type containerStatsProvider interface {
	ListPodStats(ctx context.Context) ([]statsapi.PodStats, error)
	ListPodStatsAndUpdateCPUNanoCoreUsage(ctx context.Context) ([]statsapi.PodStats, error)
	ListPodCPUAndMemoryStats(ctx context.Context) ([]statsapi.PodStats, error)
	ImageFsStats(ctx context.Context) (*statsapi.FsStats, error)
	ImageFsDevice(ctx context.Context) (string, error)
}


↓↓↓
//pkg/kubelet/kubelet.go:1262
// ListPodCPUAndMemoryStats is delegated to StatsProvider, which implements stats.Provider interface
func (kl *Kubelet) ListPodCPUAndMemoryStats(ctx context.Context) ([]statsapi.PodStats, error) {
	return kl.StatsProvider.ListPodCPUAndMemoryStats(ctx)
}


↓↓↓
分为 cadvisorStatsProvider 和criStatsProvider 两种实现

// criStatsProvider 具体实现
// 此部分代码是 cAdvisor stats 实现，跟下文 CRI 实现类似，或者说 CRI 实现现在包含 cAdvisor 的数据，这部分代码就不再详细列出
// 感兴趣的同学可以自行查看
// pkg/kubelet/stats/cadvisor_stats_provider.go:178
// ListPodCPUAndMemoryStats returns the cpu and memory stats of all the pod-managed containers.
func (p *cadvisorStatsProvider) ListPodCPUAndMemoryStats(_ context.Context) ([]statsapi.PodStats, error) {
...
}

↓↓↓
// CRI 具体实现
// 这一段基本就是找到 pod 管理的容器，移除掉终止的，将其从 CRI 获取的数据重新整合为符合`/stats/summary` 的数据格式
// 如果从不能从 CRI获取到 stats 数据，那就使用 cAdvisor 数据填充。
// pkg/kubelet/stats/cri_stats_provider.go:270
// ListPodCPUAndMemoryStats returns the CPU and Memory stats of all the pod-managed containers.
func (p *criStatsProvider) ListPodCPUAndMemoryStats(ctx context.Context) ([]statsapi.PodStats, error) {
	// sandboxIDToPodStats is a temporary map from sandbox ID to its pod stats.
	sandboxIDToPodStats := make(map[string]*statsapi.PodStats)
	containerMap, podSandboxMap, err := p.getPodAndContainerMaps(ctx)
	if err != nil {
		return nil, fmt.Errorf("failed to get pod or container map: %v", err)
	}

	result := make([]statsapi.PodStats, 0, len(podSandboxMap))
	if p.podAndContainerStatsFromCRI {
		criSandboxStats, err := p.runtimeService.ListPodSandboxStats(ctx, &runtimeapi.PodSandboxStatsFilter{})
		// Call succeeded
		if err == nil {
			for _, criSandboxStat := range criSandboxStats {
				podSandbox, found := podSandboxMap[criSandboxStat.Attributes.Id]
				if !found {
					continue
				}
				ps := buildPodStats(podSandbox)
				addCRIPodCPUStats(ps, criSandboxStat)
				addCRIPodMemoryStats(ps, criSandboxStat)
				result = append(result, *ps)
			}
			return result, err
		}
		// Call failed, why?
		s, ok := status.FromError(err)
		// Legitimate failure, rather than the CRI implementation does not support ListPodSandboxStats.
		if !ok || s.Code() != codes.Unimplemented {
			return nil, err
		}
		// CRI implementation doesn't support ListPodSandboxStats, warn and fallback.
		klog.ErrorS(err,
			"CRI implementation must be updated to support ListPodSandboxStats if PodAndContainerStatsFromCRI feature gate is enabled. Falling back to populating with cAdvisor; this call will fail in the future.",
		)
	}

	resp, err := p.runtimeService.ListContainerStats(ctx, &runtimeapi.ContainerStatsFilter{})
	if err != nil {
		return nil, fmt.Errorf("failed to list all container stats: %v", err)
	}

	allInfos, err := getCadvisorContainerInfo(p.cadvisor)
	if err != nil {
		return nil, fmt.Errorf("failed to fetch cadvisor stats: %v", err)
	}
	caInfos, allInfos := getCRICadvisorStats(allInfos)

	for _, stats := range resp {
		containerID := stats.Attributes.Id
		container, found := containerMap[containerID]
		if !found {
			continue
		}

		podSandboxID := container.PodSandboxId
		podSandbox, found := podSandboxMap[podSandboxID]
		if !found {
			continue
		}

		// Creates the stats of the pod (if not created yet) which the
		// container belongs to.
		ps, found := sandboxIDToPodStats[podSandboxID]
		if !found {
			ps = buildPodStats(podSandbox)
			sandboxIDToPodStats[podSandboxID] = ps
		}

		// 这部分实际效果是 cAdvisor stats 覆盖了 CRI CPU 和 Memory Stats，是个 bug，并不符合 KEP 设计。
		// Issue： https://github.com/kubernetes/kubernetes/issues/107172
		// 修复 PR： https://github.com/kubernetes/kubernetes/pull/110897  但是社区怕影响使用，暂未合并
		// Fill available CPU and memory stats for full set of required pod stats
		cs := p.makeContainerCPUAndMemoryStats(stats, container)
		p.addPodCPUMemoryStats(ps, types.UID(podSandbox.Metadata.Uid), allInfos, cs)

		// If cadvisor stats is available for the container, use it to populate
		// container stats
		caStats, caFound := caInfos[containerID]
		if !caFound {
			klog.V(4).InfoS("Unable to find cadvisor stats for container", "containerID", containerID)
		} else {
			p.addCadvisorContainerCPUAndMemoryStats(cs, &caStats)
		}
		ps.Containers = append(ps.Containers, *cs)
	}
	// cleanup outdated caches.
	p.cleanupOutdatedCaches()

	for _, s := range sandboxIDToPodStats {
		result = append(result, *s)
	}
	return result, nil
}