kube-apiserver资源注册

kube-apiserver进程会启动三种apiserver: aggregatorserver，kubeapiserver和apiExtensionsServer，其中kubeapiserver启动时会通过go的import和init机制将k8s支持的资源注册到资源注册表中，这样后面的逻辑才能从scheme资源注册表拿到资源信息并启动apiserver。import和init机制如下：

SchemeBuilder

在资源注册过程中用到了一个辅助结构SchemeBuilder，先介绍一下它，其定义如下

#k8s.io/apimachinery/pkg/runtime/scheme_builer.go
// SchemeBuilder collects functions that add things to a scheme. It's to allow
// code to compile without explicitly referencing generated types. You should
// declare one in each package that will have generated deep copy or conversion
// functions.
//SchemeBuilder定义，是个函数数组
type SchemeBuilder []func(*Scheme) error

可看到它就是一个数组，每个数组元素都是个函数指针。

1. 创建SchemeBuilder，同时可以传递几个函数，进行注册

// NewSchemeBuilder calls Register for you.
func NewSchemeBuilder(funcs ...func(*Scheme) error) SchemeBuilder {
    var sb SchemeBuilder
    sb.Register(funcs...)
    return sb
}

2. 创建完SchemeBuilder后，可以调用Register注册函数 

 // Register adds a scheme setup function to the list.
func (sb *SchemeBuilder) Register(funcs ...func(*Scheme) error) {
    for _, f := range funcs {
        *sb = append(*sb, f)
    }
}

 3. 调用AddToScheme时，执行数组中注册的所有函数

// AddToScheme applies all the stored functions to the scheme. A non-nil error
// indicates that one function failed and the attempt was abandoned.
func (sb *SchemeBuilder) AddToScheme(s *Scheme) error {
    for _, f := range *sb {
        if err := f(s); err != nil {
            return err
        }
    }
    return nil
}

kubeapiserver资源注册过程

kube-apiserver进程来说，其导入了legacyscheme和controlplane包。 

#cmd/kube-apiserver/app/server.go:
    "k8s.io/kubernetes/pkg/api/legacyscheme" //创建资源注册表
    "k8s.io/kubernetes/pkg/controlplane" //注册资源到资源注册表

1. legacyscheme包创建了资源注册表，编解码器等

#pkg/api/legacyscheme:
var (
    // Scheme is the default instance of runtime.Scheme to which types in the Kubernetes API are already registered.
    // NOTE: If you are copying this file to start a new api group, STOP! Copy the
    // extensions group instead. This Scheme is special and should appear ONLY in
    // the api group, unless you really know what you're doing.
    // TODO(lavalamp): make the above error impossible.
    Scheme = runtime.NewScheme()

    // Codecs provides access to encoding and decoding for the scheme
    Codecs = serializer.NewCodecFactory(Scheme)

    // ParameterCodec handles versioning of objects that are converted to query parameters.
    ParameterCodec = runtime.NewParameterCodec(Scheme)
)

2.  注册资源

controlplane包里的import_known_versions.go文件导入了kubeapiserver支持的所有资源

#pkg/controlplane/import_known_versions.go:
import (
    // These imports are the API groups the API server will support.
    _ "k8s.io/kubernetes/pkg/apis/admission/install"
    _ "k8s.io/kubernetes/pkg/apis/admissionregistration/install"
    _ "k8s.io/kubernetes/pkg/apis/apiserverinternal/install"
    _ "k8s.io/kubernetes/pkg/apis/apps/install"
    _ "k8s.io/kubernetes/pkg/apis/authentication/install"
    _ "k8s.io/kubernetes/pkg/apis/authorization/install"
    _ "k8s.io/kubernetes/pkg/apis/autoscaling/install"
    _ "k8s.io/kubernetes/pkg/apis/batch/install"
    _ "k8s.io/kubernetes/pkg/apis/certificates/install"
    _ "k8s.io/kubernetes/pkg/apis/coordination/install"
    _ "k8s.io/kubernetes/pkg/apis/core/install"        //core代表的是无分组资源，比如pod
    _ "k8s.io/kubernetes/pkg/apis/discovery/install"
    _ "k8s.io/kubernetes/pkg/apis/events/install"
    _ "k8s.io/kubernetes/pkg/apis/extensions/install"
    _ "k8s.io/kubernetes/pkg/apis/flowcontrol/install"
    _ "k8s.io/kubernetes/pkg/apis/imagepolicy/install"
    _ "k8s.io/kubernetes/pkg/apis/networking/install"
    _ "k8s.io/kubernetes/pkg/apis/node/install"
    _ "k8s.io/kubernetes/pkg/apis/policy/install"
    _ "k8s.io/kubernetes/pkg/apis/rbac/install"
    _ "k8s.io/kubernetes/pkg/apis/scheduling/install"
    _ "k8s.io/kubernetes/pkg/apis/storage/install"
) 

拿其中一类资源(core核心无分组资源)举例来看更详细的过程。

在install.go里注册了核心无分组资源的内部版本和V1版本到资源注册表

#pkg/apis/core/install/install.go:
import (
    "k8s.io/apimachinery/pkg/runtime"
    utilruntime "k8s.io/apimachinery/pkg/util/runtime"
    "k8s.io/kubernetes/pkg/api/legacyscheme" //导入legacyscheme包，为的是获取全局资源注册表legacyscheme.Scheme
    "k8s.io/kubernetes/pkg/apis/core" //导入core包，里面创建了SchemeBuilder
    "k8s.io/kubernetes/pkg/apis/core/v1" //导入core/v1包，里面创建了SchemeBuilder
)

func init() {
    Install(legacyscheme.Scheme)
}

// Install registers the API group and adds types to a scheme
func Install(scheme *runtime.Scheme) {
    utilruntime.Must(core.AddToScheme(scheme)) //注册核心无分组资源的内部版本到资源注册表
    utilruntime.Must(v1.AddToScheme(scheme))   //注册核心无分组资源的V1版本到资源注册表
    utilruntime.Must(scheme.SetVersionPriority(v1.SchemeGroupVersion)) //设置核心无分组资源版本优先级，V1优先级最高
}

导入core包时，会初始化register.go里的全局变量，创建出SchemeBuilder

#pkg/apis/core/register.go:
// GroupName is the group name use in this package
const GroupName = ""

//APIVersionInternal = "__internal"
// SchemeGroupVersion is group version used to register these objects
var SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: runtime.APIVersionInternal}

var (
    // SchemeBuilder object to register various known types
    //创建SchemeBuilder，同时注册addKnownTypes到SchemeBuilder中
    SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes)

    // AddToScheme represents a func that can be used to apply all the registered
    // funcs in a scheme
    AddToScheme = SchemeBuilder.AddToScheme
)

当执行AddToScheme时，会调用注册的addKnownTypes，将所有核心无分组资源注册到资源注册表
func addKnownTypes(scheme *runtime.Scheme) error {
    if err := scheme.AddIgnoredConversionType(&metav1.TypeMeta{}, &metav1.TypeMeta{}); err != nil {
        return err
    }
    scheme.AddKnownTypes(SchemeGroupVersion,
        &Pod{},
        &PodList{},
        &PodStatusResult{},
        &PodTemplate{},
        &PodTemplateList{},
        &ReplicationControllerList{},
        &ReplicationController{},
        &ServiceList{},
        &Service{},
        ...)
}

导入v1/core包时，会初始化register.go里的全局变量

#pkg/apis/core/v1/register.go:
import (
    "k8s.io/api/core/v1" //又导入了包k8s.io/api/core/v1，创建出SchemeBuilder
    "k8s.io/apimachinery/pkg/runtime/schema"
)

var (
    localSchemeBuilder = &v1.SchemeBuilder
    AddToScheme        = localSchemeBuilder.AddToScheme
)

func init() {
    // We only register manually written functions here. The registration of the
    // generated functions takes place in the generated files. The separation
    // makes the code compile even when the generated files are missing.
    //将addDefaultingFuncs和addConversionFuncs注册到SchemeBuilder
    localSchemeBuilder.Register(addDefaultingFuncs, addConversionFuncs)
}

//注册默认函数，用于给资源字段设置默认值
#pkg/apis/core/v1/defaults.go:
func addDefaultingFuncs(scheme *runtime.Scheme) error {
    return RegisterDefaults(scheme)
}

#pkg/apis/core/v1/zz_generated.defaults.go:
func RegisterDefaults(scheme *runtime.Scheme) error {
    scheme.AddTypeDefaultingFunc(&v1.ConfigMap{}, func(obj interface{}) { SetObjectDefaults_ConfigMap(obj.(*v1.ConfigMap)) })
    scheme.AddTypeDefaultingFunc(&v1.ConfigMapList{}, func(obj interface{}) { SetObjectDefaults_ConfigMapList(obj.(*v1.ConfigMapList)) })
    ...
}

//注册版本转换函数
#pkg/apis/core/v1/conversion.go:
func addConversionFuncs(scheme *runtime.Scheme) error {
    // Add field conversion funcs.
    err := scheme.AddFieldLabelConversionFunc(SchemeGroupVersion.WithKind("Pod"),
        func(label, value string) (string, string, error) {
            switch label {
            case "metadata.name",
                "metadata.namespace",
                "spec.nodeName",
                "spec.restartPolicy",
                "spec.schedulerName",
                "spec.serviceAccountName",
                "status.phase",
                "status.podIP",
                "status.podIPs",
                "status.nominatedNodeName":
                return label, value, nil
            // This is for backwards compatibility with old v1 clients which send spec.host
            case "spec.host":
                return "spec.nodeName", value, nil
            default:
                return "", "", fmt.Errorf("field label not supported: %s", label)
            }
        },
    )
    ...
}

//注册自动生成的版本转换函数
#pkg/apis/core/v1/zz_generated.conversion.go:
func init() {
    localSchemeBuilder.Register(RegisterConversions)
}

// RegisterConversions adds conversion functions to the given scheme.
// Public to allow building arbitrary schemes.
func RegisterConversions(s *runtime.Scheme) error {
    if err := s.AddGeneratedConversionFunc((*v1.AWSElasticBlockStoreVolumeSource)(nil), (*core.AWSElasticBlockStoreVolumeSource)(nil), func(a, b interface{}, scope conversion.Scope) error {
        return Convert_v1_AWSElasticBlockStoreVolumeSource_To_core_AWSElasticBlockStoreVolumeSource(a.(*v1.AWSElasticBlockStoreVolumeSource), b.(*core.AWSElasticBlockStoreVolumeSource), scope)
    }); err != nil {
        return err
    }
    if err := s.AddGeneratedConversionFunc((*core.AWSElasticBlockStoreVolumeSource)(nil), (*v1.AWSElasticBlockStoreVolumeSource)(nil), func(a, b interface{}, scope conversion.Scope) error {
        return Convert_core_AWSElasticBlockStoreVolumeSource_To_v1_AWSElasticBlockStoreVolumeSource(a.(*core.AWSElasticBlockStoreVolumeSource), b.(*v1.AWSElasticBlockStoreVolumeSource), scope)
    }); err != nil {
        return err
    }
    ...
}

核心无组资源V1版本，创建NewSchemeBuilder 

#k8s.io/api/core/v1/register.go
// SchemeGroupVersion is group version used to register these objects
var SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1"}

// Resource takes an unqualified resource and returns a Group qualified GroupResource
func Resource(resource string) schema.GroupResource {
    return SchemeGroupVersion.WithResource(resource).GroupResource()
}

var (
    // We only register manually written functions here. The registration of the
    // generated functions takes place in the generated files. The separation
    // makes the code compile even when the generated files are missing.
    //创建SchemeBuilder，同时注册addKnownTypes到SchemeBuilder中
    SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes)
    AddToScheme   = SchemeBuilder.AddToScheme
)

当执行AddToScheme时，会调用注册的addKnownTypes，将所有核心无分组资源注册到资源注册表
// Adds the list of known types to the given scheme.
func addKnownTypes(scheme *runtime.Scheme) error {
    scheme.AddKnownTypes(SchemeGroupVersion,
        &Pod{},
        &PodList{},
        &PodStatusResult{},
        ...)
}

示意图

上述整个资源注册流程可以使用下面示意图表示

apiextensions-apiserver资源注册

apiextensions-apiserver资源注册流程和上面是类似的，不过是注册到另一个全局资源注册表中。

#k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go

import "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/install"

var (

    //创建资源注册表
    Scheme = runtime.NewScheme()
    Codecs = serializer.NewCodecFactory(Scheme)

    // if you modify this, make sure you update the crEncoder
    unversionedVersion = schema.GroupVersion{Group: "", Version: "v1"}
    unversionedTypes   = []runtime.Object{
        &metav1.Status{},
        &metav1.WatchEvent{},
        &metav1.APIVersions{},
        &metav1.APIGroupList{},
        &metav1.APIGroup{},
        &metav1.APIResourceList{},
    }
)

func init() {
    install.Install(Scheme)

    // we need to add the options to empty v1
    metav1.AddToGroupVersion(Scheme, schema.GroupVersion{Group: "", Version: "v1"})

    Scheme.AddUnversionedTypes(unversionedVersion, unversionedTypes...)
}

#k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/install/install.go
import (
    "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
    v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
    "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
)

// Install registers the API group and adds types to a scheme
func Install(scheme *runtime.Scheme) {
    utilruntime.Must(apiextensions.AddToScheme(scheme))
    utilruntime.Must(v1beta1.AddToScheme(scheme))
    utilruntime.Must(v1.AddToScheme(scheme))
    utilruntime.Must(scheme.SetVersionPriority(v1.SchemeGroupVersion, v1beta1.SchemeGroupVersion))
}

kube-aggregator资源注册

同理，不再做过多介绍

#k8s.io/kube-aggregator/pkg/apiserver/scheme/scheme.go
var (
    // Scheme defines methods for serializing and deserializing API objects.
    Scheme = runtime.NewScheme()
    // Codecs provides methods for retrieving codecs and serializers for specific
    // versions and content types.
    Codecs = serializer.NewCodecFactory(Scheme)
)

func init() {
    AddToScheme(Scheme)
    install.Install(Scheme)
}

// AddToScheme adds the types of this group into the given scheme.
func AddToScheme(scheme *runtime.Scheme) {
    utilruntime.Must(v1beta1.AddToScheme(scheme))
    utilruntime.Must(v1.AddToScheme(scheme))
    utilruntime.Must(apiregistration.AddToScheme(scheme))
}

#k8s.io/kube-aggregator/pkg/apis/apiregistration/install/install.go
// Install registers the API group and adds types to a scheme
func Install(scheme *runtime.Scheme) {
    utilruntime.Must(apiregistration.AddToScheme(scheme))
    utilruntime.Must(v1.AddToScheme(scheme))
    utilruntime.Must(v1beta1.AddToScheme(scheme))
    utilruntime.Must(scheme.SetVersionPriority(v1.SchemeGroupVersion, v1beta1.SchemeGroupVersion))
}

#k8s.io/kube-aggregator/pkg/apiserver/apiserver.go
func init() {
    // we need to add the options (like ListOptions) to empty v1
    metav1.AddToGroupVersion(aggregatorscheme.Scheme, schema.GroupVersion{Group: "", Version: "v1"})

    unversioned := schema.GroupVersion{Group: "", Version: "v1"}
    aggregatorscheme.Scheme.AddUnversionedTypes(unversioned,
        &metav1.Status{},
        &metav1.APIVersions{},
        &metav1.APIGroupList{},
        &metav1.APIGroup{},
        &metav1.APIResourceList{},
    )