From 146481375efb90c8ffd4f8d174c225656cc318f9 Mon Sep 17 00:00:00 2001 From: rithu john Date: Wed, 13 Sep 2017 10:57:54 -0700 Subject: [PATCH] [WIP]: add CRD support --- storage/kubernetes/client.go | 10 +- storage/kubernetes/k8sapi/crd_extensions.go | 138 ++++++++++++++++++++ storage/kubernetes/storage.go | 68 +++++++++- storage/kubernetes/types.go | 132 +++++++++++++++++++ 4 files changed, 342 insertions(+), 6 deletions(-) create mode 100644 storage/kubernetes/k8sapi/crd_extensions.go diff --git a/storage/kubernetes/client.go b/storage/kubernetes/client.go index ccdb895c..9a14d34b 100644 --- a/storage/kubernetes/client.go +++ b/storage/kubernetes/client.go @@ -249,7 +249,7 @@ func (c *client) put(resource, name string, v interface{}) error { return checkHTTPErr(resp, http.StatusOK) } -func newClient(cluster k8sapi.Cluster, user k8sapi.AuthInfo, namespace string, logger logrus.FieldLogger) (*client, error) { +func newClient(cluster k8sapi.Cluster, user k8sapi.AuthInfo, namespace string, logger logrus.FieldLogger, apiVersion string) (*client, error) { tlsConfig := cryptopasta.DefaultTLSConfig() data := func(b string, file string) ([]byte, error) { if b != "" { @@ -325,13 +325,19 @@ func newClient(cluster k8sapi.Cluster, user k8sapi.AuthInfo, namespace string, l } } + // if the apiVersion is not configured default to `oidc.coreos.com/v1` + if apiVersion == "" { + apiVersion = "oidc.coreos.com/v1" + } + + logger.Infof("kubernetes client apiVersion = %s", apiVersion) // TODO(ericchiang): make API Group and version configurable. return &client{ client: &http.Client{Transport: t}, baseURL: cluster.Server, hash: func() hash.Hash { return fnv.New64() }, namespace: namespace, - apiVersion: "oidc.coreos.com/v1", + apiVersion: apiVersion, logger: logger, }, nil } diff --git a/storage/kubernetes/k8sapi/crd_extensions.go b/storage/kubernetes/k8sapi/crd_extensions.go new file mode 100644 index 00000000..0c36f9d0 --- /dev/null +++ b/storage/kubernetes/k8sapi/crd_extensions.go @@ -0,0 +1,138 @@ +/* +Copyright 2017 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package k8sapi + +// CustomResourceDefinitionSpec describes how a user wants their resource to appear +type CustomResourceDefinitionSpec struct { + // Group is the group this resource belongs in + Group string `json:"group" protobuf:"bytes,1,opt,name=group"` + // Version is the version this resource belongs in + Version string `json:"version" protobuf:"bytes,2,opt,name=version"` + // Names are the names used to describe this custom resource + Names CustomResourceDefinitionNames `json:"names" protobuf:"bytes,3,opt,name=names"` + + // Scope indicates whether this resource is cluster or namespace scoped. Default is namespaced + Scope ResourceScope `json:"scope" protobuf:"bytes,4,opt,name=scope,casttype=ResourceScope"` +} + +// CustomResourceDefinitionNames indicates the names to serve this CustomResourceDefinition +type CustomResourceDefinitionNames struct { + // Plural is the plural name of the resource to serve. It must match the name of the CustomResourceDefinition-registration + // too: plural.group and it must be all lowercase. + Plural string `json:"plural" protobuf:"bytes,1,opt,name=plural"` + // Singular is the singular name of the resource. It must be all lowercase Defaults to lowercased + Singular string `json:"singular,omitempty" protobuf:"bytes,2,opt,name=singular"` + // ShortNames are short names for the resource. It must be all lowercase. + ShortNames []string `json:"shortNames,omitempty" protobuf:"bytes,3,opt,name=shortNames"` + // Kind is the serialized kind of the resource. It is normally CamelCase and singular. + Kind string `json:"kind" protobuf:"bytes,4,opt,name=kind"` + // ListKind is the serialized kind of the list for this resource. Defaults to List. + ListKind string `json:"listKind,omitempty" protobuf:"bytes,5,opt,name=listKind"` +} + +// ResourceScope is an enum defining the different scopes availabe to a custom resource +type ResourceScope string + +const ( + ClusterScoped ResourceScope = "Cluster" + NamespaceScoped ResourceScope = "Namespaced" +) + +type ConditionStatus string + +// These are valid condition statuses. "ConditionTrue" means a resource is in the condition. +// "ConditionFalse" means a resource is not in the condition. "ConditionUnknown" means kubernetes +// can't decide if a resource is in the condition or not. In the future, we could add other +// intermediate conditions, e.g. ConditionDegraded. +const ( + ConditionTrue ConditionStatus = "True" + ConditionFalse ConditionStatus = "False" + ConditionUnknown ConditionStatus = "Unknown" +) + +// CustomResourceDefinitionConditionType is a valid value for CustomResourceDefinitionCondition.Type +type CustomResourceDefinitionConditionType string + +const ( + // Established means that the resource has become active. A resource is established when all names are + // accepted without a conflict for the first time. A resource stays established until deleted, even during + // a later NamesAccepted due to changed names. Note that not all names can be changed. + Established CustomResourceDefinitionConditionType = "Established" + // NamesAccepted means the names chosen for this CustomResourceDefinition do not conflict with others in + // the group and are therefore accepted. + NamesAccepted CustomResourceDefinitionConditionType = "NamesAccepted" + // Terminating means that the CustomResourceDefinition has been deleted and is cleaning up. + Terminating CustomResourceDefinitionConditionType = "Terminating" +) + +// CustomResourceDefinitionCondition contains details for the current condition of this pod. +type CustomResourceDefinitionCondition struct { + // Type is the type of the condition. + Type CustomResourceDefinitionConditionType `json:"type" protobuf:"bytes,1,opt,name=type,casttype=CustomResourceDefinitionConditionType"` + // Status is the status of the condition. + // Can be True, False, Unknown. + Status ConditionStatus `json:"status" protobuf:"bytes,2,opt,name=status,casttype=ConditionStatus"` + // Last time the condition transitioned from one status to another. + // +optional + LastTransitionTime Time `json:"lastTransitionTime,omitempty" protobuf:"bytes,3,opt,name=lastTransitionTime"` + // Unique, one-word, CamelCase reason for the condition's last transition. + // +optional + Reason string `json:"reason,omitempty" protobuf:"bytes,4,opt,name=reason"` + // Human-readable message indicating details about last transition. + // +optional + Message string `json:"message,omitempty" protobuf:"bytes,5,opt,name=message"` +} + +// CustomResourceDefinitionStatus indicates the state of the CustomResourceDefinition +type CustomResourceDefinitionStatus struct { + // Conditions indicate state for particular aspects of a CustomResourceDefinition + Conditions []CustomResourceDefinitionCondition `json:"conditions" protobuf:"bytes,1,opt,name=conditions"` + + // AcceptedNames are the names that are actually being used to serve discovery + // They may be different than the names in spec. + AcceptedNames CustomResourceDefinitionNames `json:"acceptedNames" protobuf:"bytes,2,opt,name=acceptedNames"` +} + +// CustomResourceCleanupFinalizer is the name of the finalizer which will delete instances of +// a CustomResourceDefinition +const CustomResourceCleanupFinalizer = "customresourcecleanup.apiextensions.k8s.io" + +// +genclient +// +genclient:nonNamespaced +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// CustomResourceDefinition represents a resource that should be exposed on the API server. Its name MUST be in the format +// <.spec.name>.<.spec.group>. +type CustomResourceDefinition struct { + TypeMeta `json:",inline"` + ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` + + // Spec describes how the user wants the resources to appear + Spec CustomResourceDefinitionSpec `json:"spec,omitempty" protobuf:"bytes,2,opt,name=spec"` + // Status indicates the actual state of the CustomResourceDefinition + Status CustomResourceDefinitionStatus `json:"status,omitempty" protobuf:"bytes,3,opt,name=status"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// CustomResourceDefinitionList is a list of CustomResourceDefinition objects. +type CustomResourceDefinitionList struct { + TypeMeta `json:",inline"` + ListMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` + + // Items individual CustomResourceDefinitions + Items []CustomResourceDefinition `json:"items" protobuf:"bytes,2,rep,name=items"` +} diff --git a/storage/kubernetes/storage.go b/storage/kubernetes/storage.go index 15f7e3a9..92c5e3fc 100644 --- a/storage/kubernetes/storage.go +++ b/storage/kubernetes/storage.go @@ -38,6 +38,8 @@ const ( type Config struct { InCluster bool `json:"inCluster"` KubeConfigFile string `json:"kubeConfigFile"` + APIVersion string `json:"apiVersion"` // API Group and version + UseCRD bool `json:"useCRD"` // Flag option to use CRDs instead of TPRs } // Open returns a storage using Kubernetes third party resource. @@ -52,9 +54,9 @@ func (c *Config) Open(logger logrus.FieldLogger) (storage.Storage, error) { // open returns a kubernetes client, initializing the third party resources used // by dex. // -// errOnTPRs controls if errors creating the resources cause this method to return +// errOnResources controls if errors creating the resources cause this method to return // immediately (used during testing), or if the client will asynchronously retry. -func (c *Config) open(logger logrus.FieldLogger, errOnTPRs bool) (*client, error) { +func (c *Config) open(logger logrus.FieldLogger, errOnResources bool) (*client, error) { if c.InCluster && (c.KubeConfigFile != "") { return nil, errors.New("cannot specify both 'inCluster' and 'kubeConfigFile'") } @@ -77,15 +79,46 @@ func (c *Config) open(logger logrus.FieldLogger, errOnTPRs bool) (*client, error return nil, err } - cli, err := newClient(cluster, user, namespace, logger) + cli, err := newClient(cluster, user, namespace, logger, c.APIVersion) if err != nil { return nil, fmt.Errorf("create client: %v", err) } ctx, cancel := context.WithCancel(context.Background()) + if c.UseCRD { + if !cli.createCustomResourceDefinitions() { + if errOnResources { + cancel() + return nil, fmt.Errorf("failed creating custom resource definitions") + } + } + + // Try to synchronously create the custom resource definitions once. This doesn't mean + // they'll immediately be available, but ensures that the client will actually try + // once. + logger.Errorf("failed creating custom resource definitions: %v", err) + go func() { + for { + if cli.createCustomResourceDefinitions() { + return + } + + select { + case <-ctx.Done(): + return + case <-time.After(30 * time.Second): + } + } + }() + // If the client is closed, stop trying to create third party resources. + cli.cancel = cancel + return cli, nil + + } + if !cli.createThirdPartyResources() { - if errOnTPRs { + if errOnResources { cancel() return nil, fmt.Errorf("failed creating third party resources") } @@ -144,6 +177,33 @@ func (cli *client) createThirdPartyResources() (ok bool) { return ok } +// createCustomResourceDefinitions attempts to create the custom resource definitions(CRDs) +// required by dex. If the CRDs exist, this information is logged. It logs all errors, +// returning true if the CRDs were created successfully. +// +// TODO: Provide an option to wait for the CRDs to actually be available. +func (cli *client) createCustomResourceDefinitions() (ok bool) { + ok = true + for _, r := range customResourceDefinitions { + err := cli.postResource("apiextensions.k8s.io/v1beta1", "", "customresourcedefinition", r) + if err != nil { + switch err { + case storage.ErrAlreadyExists: + cli.logger.Infof("custom resource definition already created %s", r.ObjectMeta.Name) + case storage.ErrNotFound: + cli.logger.Errorf("custom resource definition not found, please enable API group apiextensions.k8s.io/v1beta1") + ok = false + default: + cli.logger.Errorf("creating custom resource definition %s: %v", r.ObjectMeta.Name, err) + ok = false + } + continue + } + cli.logger.Errorf("create custom resource definition %s", r.ObjectMeta.Name) + } + return ok +} + func (cli *client) Close() error { if cli.cancel != nil { cli.cancel() diff --git a/storage/kubernetes/types.go b/storage/kubernetes/types.go index 8dd16eb2..8ffee1d6 100644 --- a/storage/kubernetes/types.go +++ b/storage/kubernetes/types.go @@ -84,6 +84,138 @@ var thirdPartyResources = []k8sapi.ThirdPartyResource{ }, } +var crdMeta = k8sapi.TypeMeta{ + APIVersion: "apiextensions.k8s.io/v1beta1", + Kind: "CustomResourceDefinition", +} + +const apiGroup = "dex.coreos.com" + +// The set of custom resource definitions required by the storage. These are managed by +// the storage so it can migrate itself by creating new resources. +var customResourceDefinitions = []k8sapi.CustomResourceDefinition{ + { + ObjectMeta: k8sapi.ObjectMeta{ + Name: "authcodes.dex.coreos.com", + }, + TypeMeta: crdMeta, + Spec: k8sapi.CustomResourceDefinitionSpec{ + Group: apiGroup, + Version: "v1", + Names: k8sapi.CustomResourceDefinitionNames{ + Plural: "authcodes", + Singular: "authcode", + Kind: "AuthCode", + }, + }, + }, + { + ObjectMeta: k8sapi.ObjectMeta{ + Name: "authrequests.dex.coreos.com", + }, + TypeMeta: crdMeta, + Spec: k8sapi.CustomResourceDefinitionSpec{ + Group: apiGroup, + Version: "v1", + Names: k8sapi.CustomResourceDefinitionNames{ + Plural: "authrequests", + Singular: "authcodrequest", + Kind: "AuthRequests", + }, + }, + }, + { + ObjectMeta: k8sapi.ObjectMeta{ + Name: "oauth2clients.dex.coreos.com", + }, + TypeMeta: crdMeta, + Spec: k8sapi.CustomResourceDefinitionSpec{ + Group: apiGroup, + Version: "v1", + Names: k8sapi.CustomResourceDefinitionNames{ + Plural: "oauth2clients", + Singular: "oauth2client", + Kind: "Oauth2Client", + }, + }, + }, + { + ObjectMeta: k8sapi.ObjectMeta{ + Name: "signingkeies.dex.coreos.com", + }, + TypeMeta: crdMeta, + Spec: k8sapi.CustomResourceDefinitionSpec{ + Group: apiGroup, + Version: "v1", + Names: k8sapi.CustomResourceDefinitionNames{ + Plural: "signingkeies", + Singular: "signingkey", + Kind: "SigningKey", + }, + }, + }, + { + ObjectMeta: k8sapi.ObjectMeta{ + Name: "refreshtokens.dex.coreos.com", + }, + TypeMeta: crdMeta, + Spec: k8sapi.CustomResourceDefinitionSpec{ + Group: apiGroup, + Version: "v1", + Names: k8sapi.CustomResourceDefinitionNames{ + Plural: "refreshtokens", + Singular: "refreshtoken", + Kind: "RefreshToken", + }, + }, + }, + { + ObjectMeta: k8sapi.ObjectMeta{ + Name: "passwords.dex.coreos.com", + }, + TypeMeta: crdMeta, + Spec: k8sapi.CustomResourceDefinitionSpec{ + Group: apiGroup, + Version: "v1", + Names: k8sapi.CustomResourceDefinitionNames{ + Plural: "passwords", + Singular: "password", + Kind: "Password", + }, + }, + }, + { + ObjectMeta: k8sapi.ObjectMeta{ + Name: "offlinesessionses.dex.coreos.com", + }, + TypeMeta: crdMeta, + Spec: k8sapi.CustomResourceDefinitionSpec{ + Group: apiGroup, + Version: "v1", + Names: k8sapi.CustomResourceDefinitionNames{ + Plural: "offlinesessionses", + Singular: "offlinesessions", + Kind: "OfflineSessions", + }, + }, + }, + { + ObjectMeta: k8sapi.ObjectMeta{ + Name: "connectors.dex.coreos.com", + }, + TypeMeta: crdMeta, + Spec: k8sapi.CustomResourceDefinitionSpec{ + Group: apiGroup, + Version: "v1", + Names: k8sapi.CustomResourceDefinitionNames{ + Plural: "connectors", + Singular: "connector", + Kind: "Connector", + }, + }, + }, +} + // There will only ever be a single keys resource. Maintain this by setting a // common name. const keysName = "openid-connect-keys"