Merge pull request #1072 from ericchiang/k8s-test

*: run kubernetes tests in travis

.travis.yml

@@ -8,6 +8,7 @@ go:
 
 services:
   - postgresql
+  - docker
 
 env:
   - DEX_POSTGRES_DATABASE=postgres DEX_POSTGRES_USER=postgres DEX_POSTGRES_HOST="localhost" DEX_LDAP_TESTS=1 DEBIAN_FRONTEND=noninteractive
@@ -21,6 +22,7 @@ install:
 
 script:
   - make testall
+  - ./scripts/test-k8s.sh
 
 notifications:
   email: false

Documentation/dev-integration-tests.md

@@ -2,7 +2,7 @@
 
 ## Kubernetes
 
-Kubernetes tests will only run if the `DEX_KUBECONFIG` environment variable is set.
+Kubernetes tests run against a Kubernetes API server, and are enabled by the `DEX_KUBECONFIG` environment variable:
 
 ```
 $ export DEX_KUBECONFIG=~/.kube/config
@@ -10,7 +10,11 @@ $ go test -v -i ./storage/kubernetes
 $ go test -v ./storage/kubernetes
 ```
 
-Because third party resources creation isn't synchronized it's expected that the tests fail the first time. Fear not, and just run them again.
+These tests can be executed locally using docker by running the following script:
+
+```
+$ ./scripts/test-k8s.sh
+```
 
 ## Postgres
 

scripts/test-k8s.sh

@@ -0,0 +1,56 @@
+#!/bin/bash -e
+
+TEMPDIR=$( mktemp -d )
+
+cat << EOF > $TEMPDIR/kubeconfig
+apiVersion: v1
+kind: Config
+clusters:
+- name: local
+  cluster:
+    server: http://localhost:8080
+users:
+- name: local
+  user:
+contexts:
+- context:
+    cluster: local
+    user: local
+EOF
+
+cleanup () {
+    docker rm -f $( cat $TEMPDIR/etcd )
+    docker rm -f $( cat $TEMPDIR/kube-apiserver )
+    rm -rf $TEMPDIR
+}
+
+trap "{ CODE=$?; cleanup ; exit $CODE; }" EXIT
+
+docker run \
+    --cidfile=$TEMPDIR/etcd \
+    -d \
+    --net=host \
+    gcr.io/google_containers/etcd:3.1.10 \
+    etcd
+
+docker run \
+    --cidfile=$TEMPDIR/kube-apiserver \
+    -d \
+    -v $TEMPDIR:/var/run/kube-test:ro \
+    --net=host \
+    gcr.io/google_containers/kube-apiserver-amd64:v1.7.4 \
+    kube-apiserver \
+    --etcd-servers=http://localhost:2379 \
+    --service-cluster-ip-range=10.0.0.1/16 \
+    --insecure-bind-address=0.0.0.0 \
+    --insecure-port=8080
+
+until $(curl --output /dev/null --silent --head --fail http://localhost:8080/healthz); do
+    printf '.'
+    sleep 1
+done
+echo "API server ready"
+
+export DEX_KUBECONFIG=$TEMPDIR/kubeconfig
+go test -v -i ./storage/kubernetes
+go test -v ./storage/kubernetes

storage/kubernetes/client.go

@@ -157,7 +157,11 @@ func closeResp(r *http.Response) {
 }
 
 func (c *client) get(resource, name string, v interface{}) error {
-	url := c.urlFor(c.apiVersion, c.namespace, resource, name)
+	return c.getResource(c.apiVersion, c.namespace, resource, name, v)
+}
+
+func (c *client) getResource(apiVersion, namespace, resource, name string, v interface{}) error {
+	url := c.urlFor(apiVersion, namespace, resource, name)
 	resp, err := c.client.Get(url)
 	if err != nil {
 		return err

storage/kubernetes/storage.go

@@ -53,9 +53,9 @@ func (c *Config) Open(logger logrus.FieldLogger) (storage.Storage, error) {
 // open returns a kubernetes client, initializing the third party resources used
 // by dex.
 //
-// errOnResources controls if errors creating the resources cause this method to return
+// waitForResources controls if errors creating the resources cause this method to return
 // immediately (used during testing), or if the client will asynchronously retry.
-func (c *Config) open(logger logrus.FieldLogger, errOnResources bool) (*client, error) {
+func (c *Config) open(logger logrus.FieldLogger, waitForResources bool) (*client, error) {
 	if c.InCluster && (c.KubeConfigFile != "") {
 		return nil, errors.New("cannot specify both 'inCluster' and 'kubeConfigFile'")
 	}
@@ -87,7 +87,7 @@ func (c *Config) open(logger logrus.FieldLogger, errOnResources bool) (*client,
 
 	logger.Info("creating custom Kubernetes resources")
 	if !cli.registerCustomResources(c.UseTPR) {
-		if errOnResources {
+		if waitForResources {
 			cancel()
 			return nil, fmt.Errorf("failed creating custom resources")
 		}
@@ -111,6 +111,13 @@ func (c *Config) open(logger logrus.FieldLogger, errOnResources bool) (*client,
 		}()
 	}
 
+	if waitForResources {
+		if err := cli.waitForCRDs(ctx); err != nil {
+			cancel()
+			return nil, err
+		}
+	}
+
 	// If the client is closed, stop trying to create resources.
 	cli.cancel = cancel
 	return cli, nil
@@ -123,9 +130,6 @@ func (c *Config) open(logger logrus.FieldLogger, errOnResources bool) (*client,
 // It logs all errors, returning true if the resources were created successfully.
 //
 // Creating a custom resource does not mean that they'll be immediately available.
-//
-// TODO(ericchiang): Provide an option to wait for the resources to actually
-// be available.
 func (cli *client) registerCustomResources(useTPR bool) (ok bool) {
 	ok = true
 	length := len(customResourceDefinitions)
@@ -165,6 +169,49 @@ func (cli *client) registerCustomResources(useTPR bool) (ok bool) {
 	return ok
 }
 
+// waitForCRDs waits for all CRDs to be in a ready state, and is used
+// by the tests to synchronize before running conformance.
+func (cli *client) waitForCRDs(ctx context.Context) error {
+	ctx, cancel := context.WithTimeout(ctx, time.Second*30)
+	defer cancel()
+
+	for _, crd := range customResourceDefinitions {
+		for {
+			err := cli.isCRDReady(crd.Name)
+			if err == nil {
+				break
+			}
+
+			cli.logger.Errorf("checking CRD: %v", err)
+
+			select {
+			case <-ctx.Done():
+				return errors.New("timed out waiting for CRDs to be available")
+			case <-time.After(time.Millisecond * 100):
+			}
+		}
+	}
+	return nil
+}
+
+// isCRDReady determines if a CRD is ready by inspecting its conditions.
+func (cli *client) isCRDReady(name string) error {
+	var r k8sapi.CustomResourceDefinition
+	err := cli.getResource("apiextensions.k8s.io/v1beta1", "", "customresourcedefinitions", name, &r)
+	if err != nil {
+		return fmt.Errorf("get crd %s: %v", name, err)
+	}
+
+	conds := make(map[string]string) // For debugging, keep the conditions around.
+	for _, c := range r.Status.Conditions {
+		if c.Type == k8sapi.Established && c.Status == k8sapi.ConditionTrue {
+			return nil
+		}
+		conds[string(c.Type)] = string(c.Status)
+	}
+	return fmt.Errorf("crd %s not ready %#v", name, conds)
+}
+
 func (cli *client) Close() error {
 	if cli.cancel != nil {
 		cli.cancel()