Vendor dependencies

This commit is contained in:
Mark Sagi-Kazar
2020-01-31 10:32:00 +01:00
parent ca2d718fe4
commit 0f1927a1ba
1245 changed files with 316980 additions and 43333 deletions

View File

@@ -0,0 +1 @@
assets/bin

View File

@@ -0,0 +1,8 @@
# Integration Testing Framework
A framework for integration testing components of kubernetes. This framework is
intended to work properly both in CI, and on a local dev machine. It therefore
explicitly supports both Linux and Darwin.
For detailed documentation see the
[![GoDoc](https://godoc.org/github.com/kubernetes-sigs/testing_frameworks/integration?status.svg)](https://godoc.org/github.com/kubernetes-sigs/testing_frameworks/integration).

View File

@@ -0,0 +1,24 @@
package addr
import (
"net"
)
// Suggest suggests a address a process can listen on. It returns
// a tuple consisting of a free port and the hostname resolved to its IP.
func Suggest() (port int, resolvedHost string, err error) {
addr, err := net.ResolveTCPAddr("tcp", "localhost:0")
if err != nil {
return
}
l, err := net.ListenTCP("tcp", addr)
if err != nil {
return
}
port = l.Addr().(*net.TCPAddr).Port
defer func() {
err = l.Close()
}()
resolvedHost = addr.IP.String()
return
}

View File

@@ -0,0 +1,136 @@
package integration
import (
"fmt"
"io"
"net/url"
"time"
"sigs.k8s.io/testing_frameworks/integration/addr"
"sigs.k8s.io/testing_frameworks/integration/internal"
)
// APIServer knows how to run a kubernetes apiserver.
type APIServer struct {
// URL is the address the ApiServer should listen on for client connections.
//
// If this is not specified, we default to a random free port on localhost.
URL *url.URL
// SecurePort is the additional secure port that the APIServer should listen on.
SecurePort int
// Path is the path to the apiserver binary.
//
// If this is left as the empty string, we will attempt to locate a binary,
// by checking for the TEST_ASSET_KUBE_APISERVER environment variable, and
// the default test assets directory. See the "Binaries" section above (in
// doc.go) for details.
Path string
// Args is a list of arguments which will passed to the APIServer binary.
// Before they are passed on, they will be evaluated as go-template strings.
// This means you can use fields which are defined and exported on this
// APIServer struct (e.g. "--cert-dir={{ .Dir }}").
// Those templates will be evaluated after the defaulting of the APIServer's
// fields has already happened and just before the binary actually gets
// started. Thus you have access to caluclated fields like `URL` and others.
//
// If not specified, the minimal set of arguments to run the APIServer will
// be used.
Args []string
// CertDir is a path to a directory containing whatever certificates the
// APIServer will need.
//
// If left unspecified, then the Start() method will create a fresh temporary
// directory, and the Stop() method will clean it up.
CertDir string
// EtcdURL is the URL of the Etcd the APIServer should use.
//
// If this is not specified, the Start() method will return an error.
EtcdURL *url.URL
// StartTimeout, StopTimeout specify the time the APIServer is allowed to
// take when starting and stoppping before an error is emitted.
//
// If not specified, these default to 20 seconds.
StartTimeout time.Duration
StopTimeout time.Duration
// Out, Err specify where APIServer should write its StdOut, StdErr to.
//
// If not specified, the output will be discarded.
Out io.Writer
Err io.Writer
processState *internal.ProcessState
}
// Start starts the apiserver, waits for it to come up, and returns an error,
// if occurred.
func (s *APIServer) Start() error {
if s.processState == nil {
if err := s.setProcessState(); err != nil {
return err
}
}
return s.processState.Start(s.Out, s.Err)
}
func (s *APIServer) setProcessState() error {
if s.EtcdURL == nil {
return fmt.Errorf("expected EtcdURL to be configured")
}
var err error
s.processState = &internal.ProcessState{}
s.processState.DefaultedProcessInput, err = internal.DoDefaulting(
"kube-apiserver",
s.URL,
s.CertDir,
s.Path,
s.StartTimeout,
s.StopTimeout,
)
if err != nil {
return err
}
// Defaulting the secure port
if s.SecurePort == 0 {
s.SecurePort, _, err = addr.Suggest()
if err != nil {
return err
}
}
s.processState.HealthCheckEndpoint = "/healthz"
s.URL = &s.processState.URL
s.CertDir = s.processState.Dir
s.Path = s.processState.Path
s.StartTimeout = s.processState.StartTimeout
s.StopTimeout = s.processState.StopTimeout
s.processState.Args, err = internal.RenderTemplates(
internal.DoAPIServerArgDefaulting(s.Args), s,
)
return err
}
// Stop stops this process gracefully, waits for its termination, and cleans up
// the CertDir if necessary.
func (s *APIServer) Stop() error {
return s.processState.Stop()
}
// APIServerDefaultArgs exposes the default args for the APIServer so that you
// can use those to append your own additional arguments.
//
// The internal default arguments are explicitely copied here, we don't want to
// allow users to change the internal ones.
var APIServerDefaultArgs = append([]string{}, internal.APIServerDefaultArgs...)

View File

@@ -0,0 +1,59 @@
package integration
import (
"fmt"
"net/url"
)
// ControlPlane is a struct that knows how to start your test control plane.
//
// Right now, that means Etcd and your APIServer. This is likely to increase in
// future.
type ControlPlane struct {
APIServer *APIServer
Etcd *Etcd
}
// Start will start your control plane processes. To stop them, call Stop().
func (f *ControlPlane) Start() error {
if f.Etcd == nil {
f.Etcd = &Etcd{}
}
if err := f.Etcd.Start(); err != nil {
return err
}
if f.APIServer == nil {
f.APIServer = &APIServer{}
}
f.APIServer.EtcdURL = f.Etcd.URL
return f.APIServer.Start()
}
// Stop will stop your control plane processes, and clean up their data.
func (f *ControlPlane) Stop() error {
if f.APIServer != nil {
if err := f.APIServer.Stop(); err != nil {
return err
}
}
if f.Etcd != nil {
if err := f.Etcd.Stop(); err != nil {
return err
}
}
return nil
}
// APIURL returns the URL you should connect to to talk to your API.
func (f *ControlPlane) APIURL() *url.URL {
return f.APIServer.URL
}
// KubeCtl returns a pre-configured KubeCtl, ready to connect to this
// ControlPlane.
func (f *ControlPlane) KubeCtl() *KubeCtl {
k := &KubeCtl{}
k.Opts = append(k.Opts, fmt.Sprintf("--server=%s", f.APIURL()))
return k
}

View File

@@ -0,0 +1,119 @@
/*
Package integration implements an integration testing framework for kubernetes.
It provides components for standing up a kubernetes API, against which you can test a
kubernetes client, or other kubernetes components. The lifecycle of the components
needed to provide this API is managed by this framework.
Quickstart
If you want to test a kubernetes client against the latest kubernetes APIServer
and Etcd, you can use `./scripts/download-binaries.sh` to download APIServer
and Etcd binaries for your platform. Then add something like the following to
your tests:
cp := &integration.ControlPlane{}
cp.Start()
kubeCtl := cp.KubeCtl()
stdout, stderr, err := kubeCtl.Run("get", "pods")
// You can check on err, stdout & stderr and build up
// your tests
cp.Stop()
Components
Currently the framework provides the following components:
ControlPlane: The ControlPlane wraps Etcd & APIServer (see below) and wires
them together correctly. A ControlPlane can be stopped & started and can
provide the URL to connect to the API. The ControlPlane can also be asked for a
KubeCtl which is already correctly configured for this ControlPlane. The
ControlPlane is a good entry point for default setups.
Etcd: Manages an Etcd binary, which can be started, stopped and connected to.
By default Etcd will listen on a random port for http connections and will
create a temporary directory for its data. To configure it differently, see the
Etcd type documentation below.
APIServer: Manages an Kube-APIServer binary, which can be started, stopped and
connected to. By default APIServer will listen on a random port for http
connections and will create a temporary directory to store the (auto-generated)
certificates. To configure it differently, see the APIServer type
documentation below.
KubeCtl: Wraps around a `kubectl` binary and can `Run(...)` arbitrary commands
against a kubernetes control plane.
Binaries
Etcd, APIServer & KubeCtl use the same mechanism to determine which binaries to
use when they get started.
1. If the component is configured with a `Path` the framework tries to run that
binary.
For example:
myEtcd := &Etcd{
Path: "/some/other/etcd",
}
cp := &integration.ControlPlane{
Etcd: myEtcd,
}
cp.Start()
2. If the Path field on APIServer, Etcd or KubeCtl is left unset and an
environment variable named `TEST_ASSET_KUBE_APISERVER`, `TEST_ASSET_ETCD` or
`TEST_ASSET_KUBECTL` is set, its value is used as a path to the binary for the
APIServer, Etcd or KubeCtl.
3. If neither the `Path` field, nor the environment variable is set, the
framework tries to use the binaries `kube-apiserver`, `etcd` or `kubectl` in
the directory `${FRAMEWORK_DIR}/assets/bin/`.
For convenience this framework ships with
`${FRAMEWORK_DIR}/scripts/download-binaries.sh` which can be used to download
pre-compiled versions of the needed binaries and place them in the default
location (`${FRAMEWORK_DIR}/assets/bin/`).
Arguments for Etcd and APIServer
Those components will start without any configuration. However, if you want or
need to, you can override certain configuration -- one of which are the
arguments used when calling the binary.
When you choose to specify your own set of arguments, those won't be appended
to the default set of arguments, it is your responsibility to provide all the
arguments needed for the binary to start successfully.
However, the default arguments for APIServer and Etcd are exported as
`APIServerDefaultArgs` and `EtcdDefaultArgs` from this package. Treat those
variables as read-only constants. Internally we have a set of default
arguments for defaulting, the `APIServerDefaultArgs` and `EtcdDefaultArgs` are
just copies of those. So when you override them you loose access to the actual
internal default arguments, but your override won't affect the defaulting.
All arguments are interpreted as go templates. Those templates have access to
all exported fields of the `APIServer`/`Etcd` struct. It does not matter if
those fields where explicitly set up or if they were defaulted by calling the
`Start()` method, the template evaluation runs just before the binary is
executed and right after the defaulting of all the struct's fields has
happened.
// When you want to append additional arguments ...
etcd := &Etcd{
// Additional custom arguments will appended to the set of default
// arguments
Args: append(EtcdDefaultArgs, "--additional=arg"),
DataDir: "/my/special/data/dir",
}
// When you want to use a custom set of arguments ...
etcd := &Etcd{
// Only custom arguments will be passed to the binary
Args: []string{"--one=1", "--two=2", "--three=3"},
DataDir: "/my/special/data/dir",
}
*/
package integration

View File

@@ -0,0 +1,114 @@
package integration
import (
"io"
"time"
"net/url"
"sigs.k8s.io/testing_frameworks/integration/internal"
)
// Etcd knows how to run an etcd server.
type Etcd struct {
// URL is the address the Etcd should listen on for client connections.
//
// If this is not specified, we default to a random free port on localhost.
URL *url.URL
// Path is the path to the etcd binary.
//
// If this is left as the empty string, we will attempt to locate a binary,
// by checking for the TEST_ASSET_ETCD environment variable, and the default
// test assets directory. See the "Binaries" section above (in doc.go) for
// details.
Path string
// Args is a list of arguments which will passed to the Etcd binary. Before
// they are passed on, the`y will be evaluated as go-template strings. This
// means you can use fields which are defined and exported on this Etcd
// struct (e.g. "--data-dir={{ .Dir }}").
// Those templates will be evaluated after the defaulting of the Etcd's
// fields has already happened and just before the binary actually gets
// started. Thus you have access to caluclated fields like `URL` and others.
//
// If not specified, the minimal set of arguments to run the Etcd will be
// used.
Args []string
// DataDir is a path to a directory in which etcd can store its state.
//
// If left unspecified, then the Start() method will create a fresh temporary
// directory, and the Stop() method will clean it up.
DataDir string
// StartTimeout, StopTimeout specify the time the Etcd is allowed to
// take when starting and stopping before an error is emitted.
//
// If not specified, these default to 20 seconds.
StartTimeout time.Duration
StopTimeout time.Duration
// Out, Err specify where Etcd should write its StdOut, StdErr to.
//
// If not specified, the output will be discarded.
Out io.Writer
Err io.Writer
processState *internal.ProcessState
}
// Start starts the etcd, waits for it to come up, and returns an error, if one
// occoured.
func (e *Etcd) Start() error {
if e.processState == nil {
if err := e.setProcessState(); err != nil {
return err
}
}
return e.processState.Start(e.Out, e.Err)
}
func (e *Etcd) setProcessState() error {
var err error
e.processState = &internal.ProcessState{}
e.processState.DefaultedProcessInput, err = internal.DoDefaulting(
"etcd",
e.URL,
e.DataDir,
e.Path,
e.StartTimeout,
e.StopTimeout,
)
if err != nil {
return err
}
e.processState.StartMessage = internal.GetEtcdStartMessage(e.processState.URL)
e.URL = &e.processState.URL
e.DataDir = e.processState.Dir
e.Path = e.processState.Path
e.StartTimeout = e.processState.StartTimeout
e.StopTimeout = e.processState.StopTimeout
e.processState.Args, err = internal.RenderTemplates(
internal.DoEtcdArgDefaulting(e.Args), e,
)
return err
}
// Stop stops this process gracefully, waits for its termination, and cleans up
// the DataDir if necessary.
func (e *Etcd) Stop() error {
return e.processState.Stop()
}
// EtcdDefaultArgs exposes the default args for Etcd so that you
// can use those to append your own additional arguments.
//
// The internal default arguments are explicitely copied here, we don't want to
// allow users to change the internal ones.
var EtcdDefaultArgs = append([]string{}, internal.EtcdDefaultArgs...)

View File

@@ -0,0 +1,17 @@
package internal
var APIServerDefaultArgs = []string{
"--etcd-servers={{ if .EtcdURL }}{{ .EtcdURL.String }}{{ end }}",
"--cert-dir={{ .CertDir }}",
"--insecure-port={{ if .URL }}{{ .URL.Port }}{{ end }}",
"--insecure-bind-address={{ if .URL }}{{ .URL.Hostname }}{{ end }}",
"--secure-port={{ if .SecurePort }}{{ .SecurePort }}{{ end }}",
}
func DoAPIServerArgDefaulting(args []string) []string {
if len(args) != 0 {
return args
}
return APIServerDefaultArgs
}

View File

@@ -0,0 +1,28 @@
package internal
import (
"bytes"
"html/template"
)
func RenderTemplates(argTemplates []string, data interface{}) (args []string, err error) {
var t *template.Template
for _, arg := range argTemplates {
t, err = template.New(arg).Parse(arg)
if err != nil {
args = nil
return
}
buf := &bytes.Buffer{}
err = t.Execute(buf, data)
if err != nil {
args = nil
return
}
args = append(args, buf.String())
}
return
}

View File

@@ -0,0 +1,35 @@
package internal
import (
"os"
"path/filepath"
"regexp"
"runtime"
"strings"
)
var assetsPath string
func init() {
_, thisFile, _, ok := runtime.Caller(0)
if !ok {
panic("Could not determine the path of the BinPathFinder")
}
assetsPath = filepath.Join(filepath.Dir(thisFile), "..", "assets", "bin")
}
// BinPathFinder checks the an environment variable, derived from the symbolic name,
// and falls back to a default assets location when this variable is not set
func BinPathFinder(symbolicName string) (binPath string) {
punctuationPattern := regexp.MustCompile("[^A-Z0-9]+")
sanitizedName := punctuationPattern.ReplaceAllString(strings.ToUpper(symbolicName), "_")
leadingNumberPattern := regexp.MustCompile("^[0-9]+")
sanitizedName = leadingNumberPattern.ReplaceAllString(sanitizedName, "")
envVar := "TEST_ASSET_" + sanitizedName
if val, ok := os.LookupEnv(envVar); ok {
return val
}
return filepath.Join(assetsPath, symbolicName)
}

View File

@@ -0,0 +1,38 @@
package internal
import (
"net/url"
)
var EtcdDefaultArgs = []string{
"--listen-peer-urls=http://localhost:0",
"--advertise-client-urls={{ if .URL }}{{ .URL.String }}{{ end }}",
"--listen-client-urls={{ if .URL }}{{ .URL.String }}{{ end }}",
"--data-dir={{ .DataDir }}",
}
func DoEtcdArgDefaulting(args []string) []string {
if len(args) != 0 {
return args
}
return EtcdDefaultArgs
}
func isSecureScheme(scheme string) bool {
// https://github.com/coreos/etcd/blob/d9deeff49a080a88c982d328ad9d33f26d1ad7b6/pkg/transport/listener.go#L53
if scheme == "https" || scheme == "unixs" {
return true
}
return false
}
func GetEtcdStartMessage(listenUrl url.URL) string {
if isSecureScheme(listenUrl.Scheme) {
// https://github.com/coreos/etcd/blob/a7f1fbe00ec216fcb3a1919397a103b41dca8413/embed/serve.go#L167
return "serving client requests on "
}
// https://github.com/coreos/etcd/blob/a7f1fbe00ec216fcb3a1919397a103b41dca8413/embed/serve.go#L124
return "serving insecure client requests on "
}

View File

@@ -0,0 +1,214 @@
package internal
import (
"fmt"
"io"
"io/ioutil"
"net"
"net/http"
"net/url"
"os"
"os/exec"
"path"
"strconv"
"time"
"github.com/onsi/gomega/gbytes"
"github.com/onsi/gomega/gexec"
"sigs.k8s.io/testing_frameworks/integration/addr"
)
type ProcessState struct {
DefaultedProcessInput
Session *gexec.Session
// Healthcheck Endpoint. If we get http.StatusOK from this endpoint, we
// assume the process is ready to operate. E.g. "/healthz". If this is set,
// we ignore StartMessage.
HealthCheckEndpoint string
// HealthCheckPollInterval is the interval which will be used for polling the
// HealthCheckEndpoint.
// If left empty it will default to 100 Milliseconds.
HealthCheckPollInterval time.Duration
// StartMessage is the message to wait for on stderr. If we recieve this
// message, we assume the process is ready to operate. Ignored if
// HealthCheckEndpoint is specified.
//
// The usage of StartMessage is discouraged, favour HealthCheckEndpoint
// instead!
//
// Deprecated: Use HealthCheckEndpoint in favour of StartMessage
StartMessage string
Args []string
// ready holds wether the process is currently in ready state (hit the ready condition) or not.
// It will be set to true on a successful `Start()` and set to false on a successful `Stop()`
ready bool
}
type DefaultedProcessInput struct {
URL url.URL
Dir string
DirNeedsCleaning bool
Path string
StopTimeout time.Duration
StartTimeout time.Duration
}
func DoDefaulting(
name string,
listenUrl *url.URL,
dir string,
path string,
startTimeout time.Duration,
stopTimeout time.Duration,
) (DefaultedProcessInput, error) {
defaults := DefaultedProcessInput{
Dir: dir,
Path: path,
StartTimeout: startTimeout,
StopTimeout: stopTimeout,
}
if listenUrl == nil {
port, host, err := addr.Suggest()
if err != nil {
return DefaultedProcessInput{}, err
}
defaults.URL = url.URL{
Scheme: "http",
Host: net.JoinHostPort(host, strconv.Itoa(port)),
}
} else {
defaults.URL = *listenUrl
}
if dir == "" {
newDir, err := ioutil.TempDir("", "k8s_test_framework_")
if err != nil {
return DefaultedProcessInput{}, err
}
defaults.Dir = newDir
defaults.DirNeedsCleaning = true
}
if path == "" {
if name == "" {
return DefaultedProcessInput{}, fmt.Errorf("must have at least one of name or path")
}
defaults.Path = BinPathFinder(name)
}
if startTimeout == 0 {
defaults.StartTimeout = 20 * time.Second
}
if stopTimeout == 0 {
defaults.StopTimeout = 20 * time.Second
}
return defaults, nil
}
type stopChannel chan struct{}
func (ps *ProcessState) Start(stdout, stderr io.Writer) (err error) {
if ps.ready {
return nil
}
command := exec.Command(ps.Path, ps.Args...)
ready := make(chan bool)
timedOut := time.After(ps.StartTimeout)
var pollerStopCh stopChannel
if ps.HealthCheckEndpoint != "" {
healthCheckURL := ps.URL
healthCheckURL.Path = ps.HealthCheckEndpoint
pollerStopCh = make(stopChannel)
go pollURLUntilOK(healthCheckURL, ps.HealthCheckPollInterval, ready, pollerStopCh)
} else {
startDetectStream := gbytes.NewBuffer()
ready = startDetectStream.Detect(ps.StartMessage)
stderr = safeMultiWriter(stderr, startDetectStream)
}
ps.Session, err = gexec.Start(command, stdout, stderr)
if err != nil {
return err
}
select {
case <-ready:
ps.ready = true
return nil
case <-timedOut:
if pollerStopCh != nil {
close(pollerStopCh)
}
if ps.Session != nil {
ps.Session.Terminate()
}
return fmt.Errorf("timeout waiting for process %s to start", path.Base(ps.Path))
}
}
func safeMultiWriter(writers ...io.Writer) io.Writer {
safeWriters := []io.Writer{}
for _, w := range writers {
if w != nil {
safeWriters = append(safeWriters, w)
}
}
return io.MultiWriter(safeWriters...)
}
func pollURLUntilOK(url url.URL, interval time.Duration, ready chan bool, stopCh stopChannel) {
if interval <= 0 {
interval = 100 * time.Millisecond
}
for {
res, err := http.Get(url.String())
if err == nil && res.StatusCode == http.StatusOK {
ready <- true
return
}
select {
case <-stopCh:
return
default:
time.Sleep(interval)
}
}
}
func (ps *ProcessState) Stop() error {
if ps.Session == nil {
return nil
}
// gexec's Session methods (Signal, Kill, ...) do not check if the Process is
// nil, so we are doing this here for now.
// This should probably be fixed in gexec.
if ps.Session.Command.Process == nil {
return nil
}
detectedStop := ps.Session.Terminate().Exited
timedOut := time.After(ps.StopTimeout)
select {
case <-detectedStop:
break
case <-timedOut:
return fmt.Errorf("timeout waiting for process %s to stop", path.Base(ps.Path))
}
ps.ready = false
if ps.DirNeedsCleaning {
return os.RemoveAll(ps.Dir)
}
return nil
}

View File

@@ -0,0 +1,47 @@
package integration
import (
"bytes"
"io"
"os/exec"
"sigs.k8s.io/testing_frameworks/integration/internal"
)
// KubeCtl is a wrapper around the kubectl binary.
type KubeCtl struct {
// Path where the kubectl binary can be found.
//
// If this is left empty, we will attempt to locate a binary, by checking for
// the TEST_ASSET_KUBECTL environment variable, and the default test assets
// directory. See the "Binaries" section above (in doc.go) for details.
Path string
// Opts can be used to configure additional flags which will be used each
// time the wrapped binary is called.
//
// For example, you might want to use this to set the URL of the APIServer to
// connect to.
Opts []string
}
// Run executes the wrapped binary with some preconfigured options and the
// arguments given to this method. It returns Readers for the stdout and
// stderr.
func (k *KubeCtl) Run(args ...string) (stdout, stderr io.Reader, err error) {
if k.Path == "" {
k.Path = internal.BinPathFinder("kubectl")
}
stdoutBuffer := &bytes.Buffer{}
stderrBuffer := &bytes.Buffer{}
allArgs := append(k.Opts, args...)
cmd := exec.Command(k.Path, allArgs...)
cmd.Stdout = stdoutBuffer
cmd.Stderr = stderrBuffer
err = cmd.Run()
return stdoutBuffer, stderrBuffer, err
}