Merge pull request #1605 from dexidp/kubernetes-tests

Rewrite kubernetes tests

.github/workflows/ci.yml

@@ -68,9 +68,6 @@ jobs:
           DEX_KEYSTONE_ADMIN_USER: demo
           DEX_KEYSTONE_ADMIN_PASS: DEMO_PASS
 
-      - name: Run Kubernetes tests
-        run: ./scripts/test-k8s.sh
-
       - name: Run linter
         run: make lint
 

.travis.yml

@@ -48,7 +48,6 @@ install:
 
 script:
   - make testall
-  - ./scripts/test-k8s.sh
   - make verify-proto # Ensure proto generation doesn't depend on external packages.
 
 notifications:

Documentation/dev-integration-tests.md

@@ -1,21 +1,5 @@
 # Running integration tests
 
-## Kubernetes
-
-Kubernetes tests run against a Kubernetes API server, and are enabled by the `DEX_KUBECONFIG` environment variable:
-
-```
-$ export DEX_KUBECONFIG=~/.kube/config
-$ go test -v -i ./storage/kubernetes
-$ go test -v ./storage/kubernetes
-```
-
-These tests can be executed locally using docker by running the following script:
-
-```
-$ ./scripts/test-k8s.sh
-```
-
 ## Postgres
 
 Running database tests locally requires:

Makefile

@@ -41,12 +41,25 @@ revendor:
 	@go mod vendor -v
 	@go mod verify
 
-test:
+test: bin/test/kube-apiserver bin/test/etcd
 	@go test -v ./...
 
-testrace:
+testrace: bin/test/kube-apiserver bin/test/etcd
 	@go test -v --race ./...
 
+export TEST_ASSET_KUBE_APISERVER=$(abspath bin/test/kube-apiserver)
+export TEST_ASSET_ETCD=$(abspath bin/test/etcd)
+
+bin/test/kube-apiserver:
+	@mkdir -p bin/test
+	curl -L https://storage.googleapis.com/k8s-c10s-test-binaries/kube-apiserver-$(shell uname)-x86_64 > bin/test/kube-apiserver
+	chmod +x bin/test/kube-apiserver
+
+bin/test/etcd:
+	@mkdir -p bin/test
+	curl -L https://storage.googleapis.com/k8s-c10s-test-binaries/etcd-$(shell uname)-x86_64 > bin/test/etcd
+	chmod +x bin/test/etcd
+
 bin/golangci-lint: bin/golangci-lint-${GOLANGCI_VERSION}
 	@ln -sf golangci-lint-${GOLANGCI_VERSION} bin/golangci-lint
 bin/golangci-lint-${GOLANGCI_VERSION}:

go.mod

@@ -57,6 +57,7 @@ require (
 	gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect
 	gopkg.in/ldap.v2 v2.5.1
 	gopkg.in/square/go-jose.v2 v2.3.1
+	sigs.k8s.io/testing_frameworks v0.1.2
 )
 
 go 1.13

go.sum

@@ -92,6 +92,7 @@ github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.0.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
@@ -172,9 +173,11 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb
 github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c h1:nXxl5PrvVm2L/wCy8dQu6DMTwH4oIuGN8GJDAlqDdVE=
 github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/onsi/ginkgo v1.4.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w=
 github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/gomega v1.3.0/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo=
 github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
@@ -273,6 +276,7 @@ golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTk
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20190409202823-959b441ac422 h1:QzoH/1pFpZguR8NrRHLcO6jKqfv2zpuSqZLgdm7ZmjI=
 golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/net v0.0.0-20180112015858-5ccada7d0a7b/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -300,6 +304,9 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180117170059-2c42eef0765b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -317,6 +324,7 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3 h1:7TYNF4UdlohbFwpNH04CoPMp1cHUZgO1Ebq5r2hIjfo=
 golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20171227012246-e19ae1496984/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
@@ -363,6 +371,8 @@ gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUy
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/ldap.v2 v2.5.1 h1:wiu0okdNfjlBzg6UWvd1Hn8Y+Ux17/u/4nlk4CQr6tU=
@@ -373,6 +383,7 @@ gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
+gopkg.in/yaml.v2 v2.0.0/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
@@ -385,3 +396,5 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk=
+sigs.k8s.io/testing_frameworks v0.1.2 h1:vK0+tvjF0BZ/RYFeZ1E6BYBwHJJXhjuZ3TdsEKH+UQM=
+sigs.k8s.io/testing_frameworks v0.1.2/go.mod h1:ToQrwSC3s8Xf/lADdZp3Mktcql9CG0UAmdJG9th5i0w=

scripts/test-k8s.sh

@@ -1,56 +0,0 @@
-#!/bin/bash -e
-
-TEMPDIR=$( mktemp -d )
-
-cat << EOF > $TEMPDIR/kubeconfig
-apiVersion: v1
-kind: Config
-clusters:
-- name: local
-  cluster:
-    server: http://localhost:8080
-users:
-- name: local
-  user:
-contexts:
-- context:
-    cluster: local
-    user: local
-EOF
-
-cleanup () {
-    docker rm -f $( cat $TEMPDIR/etcd )
-    docker rm -f $( cat $TEMPDIR/kube-apiserver )
-    rm -rf $TEMPDIR
-}
-
-trap "{ CODE=$?; cleanup ; exit $CODE; }" EXIT
-
-docker run \
-    --cidfile=$TEMPDIR/etcd \
-    -d \
-    --net=host \
-    gcr.io/google_containers/etcd:3.1.10 \
-    etcd
-
-docker run \
-    --cidfile=$TEMPDIR/kube-apiserver \
-    -d \
-    -v $TEMPDIR:/var/run/kube-test:ro \
-    --net=host \
-    gcr.io/google_containers/kube-apiserver-amd64:v1.7.4 \
-    kube-apiserver \
-    --etcd-servers=http://localhost:2379 \
-    --service-cluster-ip-range=10.0.0.1/16 \
-    --insecure-bind-address=0.0.0.0 \
-    --insecure-port=8080
-
-until $(curl --output /dev/null --silent --head --fail http://localhost:8080/healthz); do
-    printf '.'
-    sleep 1
-done
-echo "API server ready"
-
-export DEX_KUBECONFIG=$TEMPDIR/kubeconfig
-go test -v -i ./storage/kubernetes
-go test -v ./storage/kubernetes

storage/kubernetes/storage_test.go

@@ -1,39 +1,104 @@
 package kubernetes
 
 import (
-	"fmt"
+	"io/ioutil"
 	"os"
+	"strings"
 	"testing"
 
 	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/suite"
+	"sigs.k8s.io/testing_frameworks/integration"
 
 	"github.com/dexidp/dex/storage"
 	"github.com/dexidp/dex/storage/conformance"
 )
 
-const testKubeConfigEnv = "DEX_KUBECONFIG"
+const kubeconfigTemplate = `apiVersion: v1
+kind: Config
+clusters:
+- name: local
+  cluster:
+    server: SERVERURL
+users:
+- name: local
+  user:
+contexts:
+- context:
+    cluster: local
+    user: local
+`
 
-func TestLoadClient(t *testing.T) {
+func TestStorage(t *testing.T) {
-	loadClient(t)
+	if os.Getenv("TEST_ASSET_KUBE_APISERVER") == "" || os.Getenv("TEST_ASSET_ETCD") == "" {
+		t.Skip("control plane binaries are missing")
 	}
 
-func loadClient(t *testing.T) *client {
+	suite.Run(t, new(StorageTestSuite))
+}
+
+type StorageTestSuite struct {
+	suite.Suite
+
+	controlPlane *integration.ControlPlane
+
+	client *client
+}
+
+func (s *StorageTestSuite) SetupSuite() {
+	s.controlPlane = &integration.ControlPlane{}
+
+	err := s.controlPlane.Start()
+	s.Require().NoError(err)
+}
+
+func (s *StorageTestSuite) TearDownSuite() {
+	s.controlPlane.Stop()
+}
+
+func (s *StorageTestSuite) SetupTest() {
+	f, err := ioutil.TempFile("", "dex-kubeconfig-*")
+	s.Require().NoError(err)
+	defer f.Close()
+
+	_, err = f.WriteString(strings.ReplaceAll(kubeconfigTemplate, "SERVERURL", s.controlPlane.APIURL().String()))
+	s.Require().NoError(err)
+
 	config := Config{
-		KubeConfigFile: os.Getenv(testKubeConfigEnv),
+		KubeConfigFile: f.Name(),
-	}
-	if config.KubeConfigFile == "" {
-		t.Skipf("test environment variable %q not set, skipping", testKubeConfigEnv)
 	}
+
 	logger := &logrus.Logger{
 		Out:       os.Stderr,
 		Formatter: &logrus.TextFormatter{DisableColors: true},
 		Level:     logrus.DebugLevel,
 	}
-	s, err := config.open(logger, true)
+
-	if err != nil {
+	client, err := config.open(logger, true)
-		t.Fatal(err)
+	s.Require().NoError(err)
+
+	s.client = client
 }
-	return s
+
+func (s *StorageTestSuite) TestStorage() {
+	newStorage := func() storage.Storage {
+		for _, resource := range []string{
+			resourceAuthCode,
+			resourceAuthRequest,
+			resourceClient,
+			resourceRefreshToken,
+			resourceKeys,
+			resourcePassword,
+		} {
+			if err := s.client.deleteAll(resource); err != nil {
+				s.T().Fatalf("delete all %q failed: %v", resource, err)
+			}
+		}
+		return s.client
+	}
+
+	conformance.RunTests(s.T(), newStorage)
+	conformance.RunTransactionTests(s.T(), newStorage)
 }
 
 func TestURLFor(t *testing.T) {
@@ -83,27 +148,3 @@ func TestURLFor(t *testing.T) {
 		}
 	}
 }
-
-func TestStorage(t *testing.T) {
-	client := loadClient(t)
-	newStorage := func() storage.Storage {
-		for _, resource := range []string{
-			resourceAuthCode,
-			resourceAuthRequest,
-			resourceClient,
-			resourceRefreshToken,
-			resourceKeys,
-			resourcePassword,
-		} {
-			if err := client.deleteAll(resource); err != nil {
-				// Fatalf sometimes doesn't print the error message.
-				fmt.Fprintf(os.Stderr, "delete all %q failed: %v\n", resource, err)
-				t.Fatalf("delete all %q failed: %v", resource, err)
-			}
-		}
-		return client
-	}
-
-	conformance.RunTests(t, newStorage)
-	conformance.RunTransactionTests(t, newStorage)
-}