feat: Create CRDs as apiextensions.k8s.io/v1

Signed-off-by: m.nabokikh <maksim.nabokikh@flant.com>
2021-02-15 11:53:59 +04:00
parent baec4f79ce
commit 7a2472555a
16 changed files with 387 additions and 171 deletions
--- a/storage/kubernetes/client.go
+++ b/storage/kubernetes/client.go
@@ -22,6 +22,7 @@ import (
 	"strings"
 	"time"

+	"github.com/Masterminds/semver"
 	"github.com/ghodss/yaml"
 	"golang.org/x/net/http2"

@@ -47,6 +48,10 @@ type client struct {
 	// API version of the oidc resources. For example "oidc.coreos.com". This is
 	// currently not configurable, but could be in the future.
 	apiVersion string
+	// API version of the custom resource definitions.
+	// Different Kubernetes version requires to create CRD in certain API. It will be discovered automatically on
+	// storage opening.
+	crdAPIVersion string

 	// This is called once the client's Close method is called to signal goroutines,
 	// such as the one creating third party resources, to stop.
@@ -195,6 +200,37 @@ func (cli *client) postResource(apiVersion, namespace, resource string, v interf
 	return checkHTTPErr(resp, http.StatusCreated)
 }

+func (cli *client) detectKubernetesVersion() error {
+	var version struct{ GitVersion string }
+
+	url := cli.baseURL + "/version"
+	resp, err := cli.client.Get(url)
+	if err != nil {
+		return err
+	}
+
+	defer closeResp(resp)
+	if err := checkHTTPErr(resp, http.StatusOK); err != nil {
+		return err
+	}
+
+	if err := json.NewDecoder(resp.Body).Decode(&version); err != nil {
+		return err
+	}
+
+	clusterVersion, err := semver.NewVersion(version.GitVersion)
+	if err != nil {
+		cli.logger.Warnf("cannot detect Kubernetes version (%s): %v", clusterVersion, err)
+		return nil
+	}
+
+	if clusterVersion.LessThan(semver.MustParse("v1.16.0")) {
+		cli.crdAPIVersion = legacyCRDAPIVersion
+	}
+
+	return nil
+}
+
 func (cli *client) delete(resource, name string) error {
 	url := cli.urlFor(cli.apiVersion, cli.namespace, resource, name)
 	req, err := http.NewRequest("DELETE", url, nil)
@@ -351,11 +387,12 @@ func newClient(cluster k8sapi.Cluster, user k8sapi.AuthInfo, namespace string, l
 			Transport: t,
 			Timeout:   15 * time.Second,
 		},
-		baseURL:    cluster.Server,
-		hash:       func() hash.Hash { return fnv.New64() },
-		namespace:  namespace,
-		apiVersion: apiVersion,
-		logger:     logger,
+		baseURL:       cluster.Server,
+		hash:          func() hash.Hash { return fnv.New64() },
+		namespace:     namespace,
+		apiVersion:    apiVersion,
+		crdAPIVersion: crdAPIVersion,
+		logger:        logger,
 	}, nil
 }

--- a/storage/kubernetes/k8sapi/crd_extensions.go
+++ b/storage/kubernetes/k8sapi/crd_extensions.go
@@ -26,6 +26,15 @@ type CustomResourceDefinitionSpec struct {

 	// Scope indicates whether this resource is cluster or namespace scoped.  Default is namespaced
 	Scope ResourceScope `json:"scope" protobuf:"bytes,4,opt,name=scope,casttype=ResourceScope"`
+	// versions is the list of all API versions of the defined custom resource.
+	// Version names are used to compute the order in which served versions are listed in API discovery.
+	// If the version string is "kube-like", it will sort above non "kube-like" version strings, which are ordered
+	// lexicographically. "Kube-like" versions start with a "v", then are followed by a number (the major version),
+	// then optionally the string "alpha" or "beta" and another number (the minor version). These are sorted first
+	// by GA > beta > alpha (where GA is a version with no suffix such as beta or alpha), and then by comparing
+	// major version, then minor version. An example sorted list of versions:
+	// v10, v2, v1, v11beta2, v10beta3, v3beta1, v12alpha1, v11alpha2, foo1, foo10.
+	Versions []CustomResourceDefinitionVersion `json:"versions" protobuf:"bytes,7,rep,name=versions"`
 }

 // CustomResourceDefinitionNames indicates the names to serve this CustomResourceDefinition
@@ -139,3 +148,29 @@ type CustomResourceDefinitionList struct {
 	// Items individual CustomResourceDefinitions
 	Items []CustomResourceDefinition `json:"items" protobuf:"bytes,2,rep,name=items"`
 }
+
+type CustomResourceDefinitionVersion struct {
+	// name is the version name, e.g. “v1”, “v2beta1”, etc.
+	// The custom resources are served under this version at `/apis/<group>/<version>/...` if `served` is true.
+	Name string `json:"name" protobuf:"bytes,1,opt,name=name"`
+	// served is a flag enabling/disabling this version from being served via REST APIs
+	Served bool `json:"served" protobuf:"varint,2,opt,name=served"`
+	// storage indicates this version should be used when persisting custom resources to storage.
+	// There must be exactly one version with storage=true.
+	Storage bool `json:"storage" protobuf:"varint,3,opt,name=storage"`
+	// schema describes the schema used for validation, pruning, and defaulting of this version of the custom resource.
+	// +optional
+	Schema *CustomResourceValidation `json:"schema,omitempty" protobuf:"bytes,4,opt,name=schema"`
+}
+
+// CustomResourceValidation is a list of validation methods for CustomResources.
+type CustomResourceValidation struct {
+	// OpenAPIV3Schema is the OpenAPI v3 schema to be validated against.
+	OpenAPIV3Schema *JSONSchemaProps `json:"openAPIV3Schema,omitempty" protobuf:"bytes,1,opt,name=openAPIV3Schema"`
+}
+
+// JSONSchemaProps is a JSON-Schema following Specification Draft 4 (http://json-schema.org/).
+type JSONSchemaProps struct {
+	Type                   string `json:"type,omitempty" protobuf:"bytes,5,opt,name=type"`
+	XPreserveUnknownFields *bool  `json:"x-kubernetes-preserve-unknown-fields,omitempty" protobuf:"bytes,38,opt,name=xKubernetesPreserveUnknownFields"`
+}
--- a/storage/kubernetes/storage.go
+++ b/storage/kubernetes/storage.go
@@ -88,6 +88,10 @@ func (c *Config) open(logger log.Logger, waitForResources bool) (*client, error)
 		return nil, fmt.Errorf("create client: %v", err)
 	}

+	if err = cli.detectKubernetesVersion(); err != nil {
+		return nil, fmt.Errorf("cannot get kubernetes version: %v", err)
+	}
+
 	ctx, cancel := context.WithCancel(context.Background())

 	logger.Info("creating custom Kubernetes resources")
@@ -100,7 +104,6 @@ func (c *Config) open(logger log.Logger, waitForResources bool) (*client, error)
 		// Try to synchronously create the custom resources once. This doesn't mean
 		// they'll immediately be available, but ensures that the client will actually try
 		// once.
-		logger.Errorf("failed creating custom resources: %v", err)
 		go func() {
 			for {
 				if cli.registerCustomResources() {
@@ -136,21 +139,25 @@ func (c *Config) open(logger log.Logger, waitForResources bool) (*client, error)
 // Creating a custom resource does not mean that they'll be immediately available.
 func (cli *client) registerCustomResources() (ok bool) {
 	ok = true
-	length := len(customResourceDefinitions)
+
+	definitions := customResourceDefinitions(cli.crdAPIVersion)
+	length := len(definitions)
+
 	for i := 0; i < length; i++ {
 		var err error
 		var resourceName string

-		r := customResourceDefinitions[i]
+		r := definitions[i]
 		var i interface{}
-		cli.logger.Infof("checking if custom resource %s has been created already...", r.ObjectMeta.Name)
+		cli.logger.Infof("checking if custom resource %s has already been created...", r.ObjectMeta.Name)
 		if err := cli.list(r.Spec.Names.Plural, &i); err == nil {
 			cli.logger.Infof("The custom resource %s already available, skipping create", r.ObjectMeta.Name)
 			continue
 		} else {
 			cli.logger.Infof("failed to list custom resource %s, attempting to create: %v", r.ObjectMeta.Name, err)
 		}
-		err = cli.postResource("apiextensions.k8s.io/v1beta1", "", "customresourcedefinitions", r)
+
+		err = cli.postResource(cli.crdAPIVersion, "", "customresourcedefinitions", r)
 		resourceName = r.ObjectMeta.Name

 		if err != nil {
@@ -177,7 +184,7 @@ func (cli *client) waitForCRDs(ctx context.Context) error {
 	ctx, cancel := context.WithTimeout(ctx, time.Second*30)
 	defer cancel()

-	for _, crd := range customResourceDefinitions {
+	for _, crd := range customResourceDefinitions(cli.crdAPIVersion) {
 		for {
 			err := cli.isCRDReady(crd.Name)
 			if err == nil {
@@ -199,7 +206,7 @@ func (cli *client) waitForCRDs(ctx context.Context) error {
 // isCRDReady determines if a CRD is ready by inspecting its conditions.
 func (cli *client) isCRDReady(name string) error {
 	var r k8sapi.CustomResourceDefinition
-	err := cli.getResource("apiextensions.k8s.io/v1beta1", "", "customresourcedefinitions", name, &r)
+	err := cli.getResource(cli.crdAPIVersion, "", "customresourcedefinitions", name, &r)
 	if err != nil {
 		return fmt.Errorf("get crd %s: %v", name, err)
 	}
--- a/storage/kubernetes/types.go
+++ b/storage/kubernetes/types.go
@@ -10,169 +10,223 @@ import (
 	"github.com/dexidp/dex/storage/kubernetes/k8sapi"
 )

-var crdMeta = k8sapi.TypeMeta{
-	APIVersion: "apiextensions.k8s.io/v1beta1",
-	Kind:       "CustomResourceDefinition",
-}
+const (
+	apiGroup = "dex.coreos.com"

-const apiGroup = "dex.coreos.com"
+	legacyCRDAPIVersion = "apiextensions.k8s.io/v1beta1"
+	crdAPIVersion       = "apiextensions.k8s.io/v1"
+)

 // The set of custom resource definitions required by the storage. These are managed by
 // the storage so it can migrate itself by creating new resources.
-var customResourceDefinitions = []k8sapi.CustomResourceDefinition{
-	{
-		ObjectMeta: k8sapi.ObjectMeta{
-			Name: "authcodes.dex.coreos.com",
-		},
-		TypeMeta: crdMeta,
-		Spec: k8sapi.CustomResourceDefinitionSpec{
-			Group:   apiGroup,
-			Version: "v1",
-			Names: k8sapi.CustomResourceDefinitionNames{
-				Plural:   "authcodes",
-				Singular: "authcode",
-				Kind:     "AuthCode",
+func customResourceDefinitions(apiVersion string) []k8sapi.CustomResourceDefinition {
+	crdMeta := k8sapi.TypeMeta{
+		APIVersion: apiVersion,
+		Kind:       "CustomResourceDefinition",
+	}
+
+	var version string
+	var scope k8sapi.ResourceScope
+	var versions []k8sapi.CustomResourceDefinitionVersion
+
+	switch apiVersion {
+	case crdAPIVersion:
+		preserveUnknownFields := true
+		versions = []k8sapi.CustomResourceDefinitionVersion{
+			{
+				Name:    "v1",
+				Served:  true,
+				Storage: true,
+				Schema: &k8sapi.CustomResourceValidation{
+					OpenAPIV3Schema: &k8sapi.JSONSchemaProps{
+						Type:                   "object",
+						XPreserveUnknownFields: &preserveUnknownFields,
+					},
+				},
+			},
+		}
+		scope = k8sapi.NamespaceScoped
+	case legacyCRDAPIVersion:
+		version = "v1"
+	default:
+		panic("unknown apiVersion " + apiVersion)
+	}
+
+	return []k8sapi.CustomResourceDefinition{
+		{
+			ObjectMeta: k8sapi.ObjectMeta{
+				Name: "authcodes.dex.coreos.com",
+			},
+			TypeMeta: crdMeta,
+			Spec: k8sapi.CustomResourceDefinitionSpec{
+				Group:    apiGroup,
+				Version:  version,
+				Versions: versions,
+				Scope:    scope,
+				Names: k8sapi.CustomResourceDefinitionNames{
+					Plural:   "authcodes",
+					Singular: "authcode",
+					Kind:     "AuthCode",
+				},
 			},
 		},
-	},
-	{
-		ObjectMeta: k8sapi.ObjectMeta{
-			Name: "authrequests.dex.coreos.com",
-		},
-		TypeMeta: crdMeta,
-		Spec: k8sapi.CustomResourceDefinitionSpec{
-			Group:   apiGroup,
-			Version: "v1",
-			Names: k8sapi.CustomResourceDefinitionNames{
-				Plural:   "authrequests",
-				Singular: "authrequest",
-				Kind:     "AuthRequest",
+		{
+			ObjectMeta: k8sapi.ObjectMeta{
+				Name: "authrequests.dex.coreos.com",
+			},
+			TypeMeta: crdMeta,
+			Spec: k8sapi.CustomResourceDefinitionSpec{
+				Group:    apiGroup,
+				Version:  version,
+				Versions: versions,
+				Scope:    scope,
+				Names: k8sapi.CustomResourceDefinitionNames{
+					Plural:   "authrequests",
+					Singular: "authrequest",
+					Kind:     "AuthRequest",
+				},
 			},
 		},
-	},
-	{
-		ObjectMeta: k8sapi.ObjectMeta{
-			Name: "oauth2clients.dex.coreos.com",
-		},
-		TypeMeta: crdMeta,
-		Spec: k8sapi.CustomResourceDefinitionSpec{
-			Group:   apiGroup,
-			Version: "v1",
-			Names: k8sapi.CustomResourceDefinitionNames{
-				Plural:   "oauth2clients",
-				Singular: "oauth2client",
-				Kind:     "OAuth2Client",
+		{
+			ObjectMeta: k8sapi.ObjectMeta{
+				Name: "oauth2clients.dex.coreos.com",
+			},
+			TypeMeta: crdMeta,
+			Spec: k8sapi.CustomResourceDefinitionSpec{
+				Group:    apiGroup,
+				Version:  version,
+				Versions: versions,
+				Scope:    scope,
+				Names: k8sapi.CustomResourceDefinitionNames{
+					Plural:   "oauth2clients",
+					Singular: "oauth2client",
+					Kind:     "OAuth2Client",
+				},
 			},
 		},
-	},
-	{
-		ObjectMeta: k8sapi.ObjectMeta{
-			Name: "signingkeies.dex.coreos.com",
-		},
-		TypeMeta: crdMeta,
-		Spec: k8sapi.CustomResourceDefinitionSpec{
-			Group:   apiGroup,
-			Version: "v1",
-			Names: k8sapi.CustomResourceDefinitionNames{
-				// `signingkeies` is an artifact from the old TPR pluralization.
-				// Users don't directly interact with this value, hence leaving it
-				// as is.
-				Plural:   "signingkeies",
-				Singular: "signingkey",
-				Kind:     "SigningKey",
+		{
+			ObjectMeta: k8sapi.ObjectMeta{
+				Name: "signingkeies.dex.coreos.com",
+			},
+			TypeMeta: crdMeta,
+			Spec: k8sapi.CustomResourceDefinitionSpec{
+				Group:    apiGroup,
+				Version:  version,
+				Versions: versions,
+				Scope:    scope,
+				Names: k8sapi.CustomResourceDefinitionNames{
+					// `signingkeies` is an artifact from the old TPR pluralization.
+					// Users don't directly interact with this value, hence leaving it
+					// as is.
+					Plural:   "signingkeies",
+					Singular: "signingkey",
+					Kind:     "SigningKey",
+				},
 			},
 		},
-	},
-	{
-		ObjectMeta: k8sapi.ObjectMeta{
-			Name: "refreshtokens.dex.coreos.com",
-		},
-		TypeMeta: crdMeta,
-		Spec: k8sapi.CustomResourceDefinitionSpec{
-			Group:   apiGroup,
-			Version: "v1",
-			Names: k8sapi.CustomResourceDefinitionNames{
-				Plural:   "refreshtokens",
-				Singular: "refreshtoken",
-				Kind:     "RefreshToken",
+		{
+			ObjectMeta: k8sapi.ObjectMeta{
+				Name: "refreshtokens.dex.coreos.com",
+			},
+			TypeMeta: crdMeta,
+			Spec: k8sapi.CustomResourceDefinitionSpec{
+				Group:    apiGroup,
+				Version:  version,
+				Versions: versions,
+				Scope:    scope,
+				Names: k8sapi.CustomResourceDefinitionNames{
+					Plural:   "refreshtokens",
+					Singular: "refreshtoken",
+					Kind:     "RefreshToken",
+				},
 			},
 		},
-	},
-	{
-		ObjectMeta: k8sapi.ObjectMeta{
-			Name: "passwords.dex.coreos.com",
-		},
-		TypeMeta: crdMeta,
-		Spec: k8sapi.CustomResourceDefinitionSpec{
-			Group:   apiGroup,
-			Version: "v1",
-			Names: k8sapi.CustomResourceDefinitionNames{
-				Plural:   "passwords",
-				Singular: "password",
-				Kind:     "Password",
+		{
+			ObjectMeta: k8sapi.ObjectMeta{
+				Name: "passwords.dex.coreos.com",
+			},
+			TypeMeta: crdMeta,
+			Spec: k8sapi.CustomResourceDefinitionSpec{
+				Group:    apiGroup,
+				Version:  version,
+				Versions: versions,
+				Scope:    scope,
+				Names: k8sapi.CustomResourceDefinitionNames{
+					Plural:   "passwords",
+					Singular: "password",
+					Kind:     "Password",
+				},
 			},
 		},
-	},
-	{
-		ObjectMeta: k8sapi.ObjectMeta{
-			Name: "offlinesessionses.dex.coreos.com",
-		},
-		TypeMeta: crdMeta,
-		Spec: k8sapi.CustomResourceDefinitionSpec{
-			Group:   apiGroup,
-			Version: "v1",
-			Names: k8sapi.CustomResourceDefinitionNames{
-				Plural:   "offlinesessionses",
-				Singular: "offlinesessions",
-				Kind:     "OfflineSessions",
+		{
+			ObjectMeta: k8sapi.ObjectMeta{
+				Name: "offlinesessionses.dex.coreos.com",
+			},
+			TypeMeta: crdMeta,
+			Spec: k8sapi.CustomResourceDefinitionSpec{
+				Group:    apiGroup,
+				Version:  version,
+				Versions: versions,
+				Scope:    scope,
+				Names: k8sapi.CustomResourceDefinitionNames{
+					Plural:   "offlinesessionses",
+					Singular: "offlinesessions",
+					Kind:     "OfflineSessions",
+				},
 			},
 		},
-	},
-	{
-		ObjectMeta: k8sapi.ObjectMeta{
-			Name: "connectors.dex.coreos.com",
-		},
-		TypeMeta: crdMeta,
-		Spec: k8sapi.CustomResourceDefinitionSpec{
-			Group:   apiGroup,
-			Version: "v1",
-			Names: k8sapi.CustomResourceDefinitionNames{
-				Plural:   "connectors",
-				Singular: "connector",
-				Kind:     "Connector",
+		{
+			ObjectMeta: k8sapi.ObjectMeta{
+				Name: "connectors.dex.coreos.com",
+			},
+			TypeMeta: crdMeta,
+			Spec: k8sapi.CustomResourceDefinitionSpec{
+				Group:    apiGroup,
+				Version:  version,
+				Versions: versions,
+				Scope:    scope,
+				Names: k8sapi.CustomResourceDefinitionNames{
+					Plural:   "connectors",
+					Singular: "connector",
+					Kind:     "Connector",
+				},
 			},
 		},
-	},
-	{
-		ObjectMeta: k8sapi.ObjectMeta{
-			Name: "devicerequests.dex.coreos.com",
-		},
-		TypeMeta: crdMeta,
-		Spec: k8sapi.CustomResourceDefinitionSpec{
-			Group:   apiGroup,
-			Version: "v1",
-			Names: k8sapi.CustomResourceDefinitionNames{
-				Plural:   "devicerequests",
-				Singular: "devicerequest",
-				Kind:     "DeviceRequest",
+		{
+			ObjectMeta: k8sapi.ObjectMeta{
+				Name: "devicerequests.dex.coreos.com",
+			},
+			TypeMeta: crdMeta,
+			Spec: k8sapi.CustomResourceDefinitionSpec{
+				Group:    apiGroup,
+				Version:  version,
+				Versions: versions,
+				Scope:    scope,
+				Names: k8sapi.CustomResourceDefinitionNames{
+					Plural:   "devicerequests",
+					Singular: "devicerequest",
+					Kind:     "DeviceRequest",
+				},
 			},
 		},
-	},
-	{
-		ObjectMeta: k8sapi.ObjectMeta{
-			Name: "devicetokens.dex.coreos.com",
-		},
-		TypeMeta: crdMeta,
-		Spec: k8sapi.CustomResourceDefinitionSpec{
-			Group:   apiGroup,
-			Version: "v1",
-			Names: k8sapi.CustomResourceDefinitionNames{
-				Plural:   "devicetokens",
-				Singular: "devicetoken",
-				Kind:     "DeviceToken",
+		{
+			ObjectMeta: k8sapi.ObjectMeta{
+				Name: "devicetokens.dex.coreos.com",
+			},
+			TypeMeta: crdMeta,
+			Spec: k8sapi.CustomResourceDefinitionSpec{
+				Group:    apiGroup,
+				Version:  version,
+				Versions: versions,
+				Scope:    scope,
+				Names: k8sapi.CustomResourceDefinitionNames{
+					Plural:   "devicetokens",
+					Singular: "devicetoken",
+					Kind:     "DeviceToken",
+				},
 			},
 		},
-	},
+	}
 }

 // There will only ever be a single keys resource. Maintain this by setting a