2026-02-10 10:22:51 -08:00

453 lines
15 KiB
Go

package kubectl
import (
"bytes"
"context"
"fmt"
"io"
"log"
"os"
"slices"
"strings"
"time"
"github.com/avast/retry-go/v4"
"github.com/agentgateway/agentgateway/controller/pkg/utils/cmdutils"
"github.com/agentgateway/agentgateway/controller/pkg/utils/kubeutils/portforward"
"github.com/agentgateway/agentgateway/controller/pkg/utils/requestutils/curl"
"github.com/agentgateway/agentgateway/controller/pkg/utils/threadsafe"
)
// Cli is a utility for executing `kubectl` commands
type Cli struct {
// receiver is the default destination for the kubectl stdout and stderr
receiver io.Writer
// kubeContext is the optional value of the context for a given kubernetes cluster
// If it is not supplied, no context will be included in the command
kubeContext string
}
// PodExecOptions describes the options used to execute a command in a pod
type PodExecOptions struct {
Name string
Namespace string
Container string
}
// NewCli returns an implementation of the kubectl.Cli
func NewCli() *Cli {
return &Cli{
receiver: io.Discard,
kubeContext: "",
}
}
// CurlResponse stores the output from a curl, with separate fields for StdErr, which typically contains headers and
// connection information, and SteOut, which typically contains the response body
type CurlResponse struct {
StdErr string
StdOut string
}
// WithReceiver sets the io.Writer that will be used by default for the stdout and stderr
// of cmdutils.Cmd created by the Cli
// This modifies the value in place, so affects shared references to the Cli and future commands run by the Cli.
// Wrap this in a threadsafe struct to avoid data races when wrapped in io.MultiWriter in cmdutils.
func (c *Cli) WithReceiver(receiver io.Writer) *Cli {
c.receiver = &threadsafe.WriterWrapper{W: receiver}
return c
}
// WithKubeContext sets the --context for the kubectl command invoked by the Cli
func (c *Cli) WithKubeContext(kubeContext string) *Cli {
c.kubeContext = kubeContext
return c
}
// Command returns a Cmd that executes kubectl command, including the --context if it is defined
// The Cmd sets the Stdout and Stderr to the receiver of the Cli
func (c *Cli) Command(ctx context.Context, args ...string) cmdutils.Cmd {
if c.kubeContext != "" {
args = append([]string{"--context", c.kubeContext}, args...)
}
return cmdutils.Command(ctx, "kubectl", args...).
// For convenience, we set the stdout and stderr to the receiver
// This can still be overwritten by consumers who use the commands
WithStdout(c.receiver).
WithStderr(c.receiver)
}
// RunCommand creates a Cmd and then runs it
func (c *Cli) RunCommand(ctx context.Context, args ...string) error {
return c.Command(ctx, args...).Run().Cause()
}
// RunCommandToWriters runs a kubectl command directing stdout/stderr to the provided writers.
// It does not mutate the Cli's default receiver.
func (c *Cli) RunCommandToWriters(ctx context.Context, stdout io.Writer, stderr io.Writer, args ...string) error {
cmd := c.Command(ctx, args...)
if stdout != nil {
cmd = cmd.WithStdout(stdout)
}
if stderr != nil {
cmd = cmd.WithStderr(stderr)
}
return cmd.Run().Cause()
}
// Namespaces returns a sorted list of namespaces or an error if one occurred
func (c *Cli) Namespaces(ctx context.Context) ([]string, error) {
stdout, _, err := c.Execute(ctx, "get", "namespaces", "--sort-by", "metadata.name", "-o", "jsonpath={.items[*].metadata.name}")
if err != nil {
return nil, err
}
return strings.Fields(stdout), nil
}
// Apply applies the resources defined in the bytes, and returns an error if one occurred
func (c *Cli) Apply(ctx context.Context, content []byte, extraArgs ...string) error {
args := append([]string{"apply", "-f", "-", "--server-side", "--field-manager=istio-ci", "--force-conflicts"}, extraArgs...)
return c.Command(ctx, args...).
WithStdin(bytes.NewBuffer(content)).
Run().
Cause()
}
// Create creates the resources defined in the bytes, and returns an error if one occurred
func (c *Cli) Create(ctx context.Context, content []byte, extraArgs ...string) error {
args := append([]string{"create", "-f", "-"}, extraArgs...)
return c.Command(ctx, args...).
WithStdin(bytes.NewBuffer(content)).
Run().
Cause()
}
// ApplyFile applies the resources defined in a file, and returns an error if one occurred
func (c *Cli) ApplyFile(ctx context.Context, fileName string, extraArgs ...string) error {
_, err := c.ApplyFileWithOutput(ctx, fileName, extraArgs...)
return err
}
// ApplyFilePath applies the resources defined at the file path, and returns an error if one occurred.
// If filePath is a directory, this will apply all of the files in the directory.
func (c *Cli) ApplyFilePath(ctx context.Context, filePath string, extraArgs ...string) error {
stat, err := os.Stat(filePath)
if err != nil {
return err
}
if !stat.IsDir() {
_, err := c.ApplyFileWithOutput(ctx, filePath, extraArgs...)
return err
}
args := append([]string{"apply", "-f", filePath, "--server-side", "--field-manager=istio-ci", "--force-conflicts"}, extraArgs...)
return c.Command(ctx, args...).
Run().
Cause()
}
// ApplyFileWithOutput applies the resources defined in a file,
// if an error occurred, it will be returned along with the output of the command
func (c *Cli) ApplyFileWithOutput(ctx context.Context, fileName string, extraArgs ...string) (string, error) {
applyArgs := append([]string{"apply", "-f", fileName, "--server-side", "--field-manager=istio-ci", "--force-conflicts"}, extraArgs...)
fileInput, err := os.Open(fileName)
if err != nil {
return "", err
}
defer fileInput.Close()
runErr := c.Command(ctx, applyArgs...).
WithStdin(fileInput).
Run()
return runErr.OutputString(), runErr.Cause()
}
// Delete deletes the resources defined in the bytes, and returns an error if one occurred
func (c *Cli) Delete(ctx context.Context, content []byte, extraArgs ...string) error {
args := append([]string{"delete", "-f", "-"}, extraArgs...)
return c.Command(ctx, args...).
WithStdin(bytes.NewBuffer(content)).
Run().
Cause()
}
// DeleteFile deletes the resources defined in a file, and returns an error if one occurred
func (c *Cli) DeleteFile(ctx context.Context, fileName string, extraArgs ...string) error {
_, err := c.DeleteFileWithOutput(ctx, fileName, extraArgs...)
return err
}
// DeleteFilePath deletes the resources defined at the file path, and returns an error if one occurred.
// If filePath is a directory, this will delete all of the files in the directory.
func (c *Cli) DeleteFilePath(ctx context.Context, filePath string, extraArgs ...string) error {
stat, err := os.Stat(filePath)
if err != nil {
return err
}
if !stat.IsDir() {
_, err := c.DeleteFileWithOutput(ctx, filePath, extraArgs...)
return err
}
args := append([]string{"delete", "-f", filePath}, extraArgs...)
return c.Command(ctx, args...).
Run().
Cause()
}
// DeleteFileWithOutput deletes the resources defined in a file,
// if an error occurred, it will be returned along with the output of the command
func (c *Cli) DeleteFileWithOutput(ctx context.Context, fileName string, extraArgs ...string) (string, error) {
applyArgs := append([]string{"delete", "-f", fileName}, extraArgs...)
fileInput, err := os.Open(fileName)
if err != nil {
return "", err
}
defer func() {
_ = fileInput.Close()
}()
runErr := c.Command(ctx, applyArgs...).
WithStdin(fileInput).
Run()
return runErr.OutputString(), runErr.Cause()
}
// DeleteFileSafe deletes the resources defined in a file, and returns an error if one occurred
// This differs from DeleteFile in that we always append --ignore-not-found
func (c *Cli) DeleteFileSafe(ctx context.Context, fileName string, extraArgs ...string) error {
safeArgs := append(extraArgs, "--ignore-not-found")
return c.DeleteFile(ctx, fileName, safeArgs...)
}
// Copy copies a file from one location to another
func (c *Cli) Copy(ctx context.Context, from, to string) error {
return c.RunCommand(ctx, "cp", from, to)
}
// SetLabel sets a label on a given resource.
func (c *Cli) SetLabel(
ctx context.Context,
kind, name, namespace string,
label, value string,
) error {
// ex: k -n ns label svc svc-a foo=val --overwrite
return c.labelInner(ctx, kind, name, namespace, label, "=", value)
}
// UnsetLabel unsets a label on a given resource.
func (c *Cli) UnsetLabel(
ctx context.Context,
kind, name, namespace string,
label string,
) error {
// ex: k -n ns label svc svc-a foo- --overwrite
return c.labelInner(ctx, kind, name, namespace, label, "-", "")
}
func (c *Cli) labelInner(
ctx context.Context,
kind, name, namespace string,
label, op, value string,
) error {
// ex: k -n ns label svc svc-a foo=val --overwrite
args := []string{"label", kind, name, label + op + value, "--overwrite"}
if namespace != "" {
args = append(
[]string{"-n", namespace},
args...,
)
}
return c.RunCommand(ctx, args...)
}
// DeploymentRolloutStatus waits for the deployment to complete rolling out
func (c *Cli) DeploymentRolloutStatus(ctx context.Context, deployment string, extraArgs ...string) error {
rolloutArgs := append([]string{
"rollout",
"status",
fmt.Sprintf("deployment/%s", deployment),
}, extraArgs...)
return c.RunCommand(ctx, rolloutArgs...)
}
// StartPortForward creates a PortForwarder based on the provides options, starts it, and returns the PortForwarder
// If an error was encountered while starting the PortForwarder, it is returned as well
// NOTE: It is the callers responsibility to close this port-forward
func (c *Cli) StartPortForward(ctx context.Context, options ...portforward.Option) (portforward.PortForwarder, error) {
options = append([]portforward.Option{
// We define some default values, which users can then override
portforward.WithWriters(c.receiver, c.receiver),
portforward.WithKubeContext(c.kubeContext),
}, options...)
portForwarder := portforward.NewCliPortForwarder(options...)
err := portForwarder.Start(
ctx,
retry.LastErrorOnly(true),
retry.Delay(250*time.Millisecond),
retry.DelayType(retry.FixedDelay),
retry.Attempts(60),
)
return portForwarder, err
}
// CurlFromPod executes a Curl request from the given pod for the given options.
// It differs from CurlFromEphemeralPod in that it does not use an ephemeral container to execute the Curl command
func (c *Cli) CurlFromPod(ctx context.Context, podOpts PodExecOptions, options ...curl.Option) (*CurlResponse, error) {
appendOption := func(option curl.Option) {
options = append(options, option)
}
// The e2e test assertions rely on the transforms.WithCurlHttpResponse to validate the response is what
// we would expect
// For this transform to behave appropriately, we must execute the request with verbose=true
appendOption(curl.VerboseOutput())
curlArgs := curl.BuildArgs(options...)
container := podOpts.Container
if container == "" {
container = "curl"
}
args := append([]string{
"exec",
"--container=" + container,
podOpts.Name,
"-n",
podOpts.Namespace,
"--",
"curl",
"--no-progress-meter",
"--connect-timeout",
"1",
"--max-time",
"5",
}, curlArgs...)
stdout, stderr, err := c.ExecuteOn(ctx, c.kubeContext, args...)
log.Printf("executing curl with args: %s", argsQuotedForShell(args))
return &CurlResponse{StdOut: stdout, StdErr: stderr}, err
}
func argsQuotedForShell(args []string) string {
var builder strings.Builder
for i, arg := range args {
if i > 0 {
builder.WriteString(" ")
}
if strings.Contains(arg, "'") {
builder.WriteString(fmt.Sprintf("%q", arg))
} else if strings.Contains(arg, " ") || strings.Contains(arg, "\"") || arg == "" {
builder.WriteString("'")
builder.WriteString(arg)
builder.WriteString("'")
} else {
builder.WriteString(arg)
}
}
return builder.String()
}
func (c *Cli) ExecuteOn(ctx context.Context, kubeContext string, args ...string) (string, string, error) {
args = append([]string{"--context", kubeContext}, args...)
return c.Execute(ctx, args...)
}
func (c *Cli) Execute(ctx context.Context, args ...string) (string, string, error) {
if c.kubeContext != "" {
if !slices.Contains(args, "--context") {
args = append([]string{"--context", c.kubeContext}, args...)
}
}
stdout := threadsafe.Buffer{}
stderr := threadsafe.Buffer{}
err := cmdutils.Command(ctx, "kubectl", args...).
WithStdout(&stdout).
WithStderr(&stderr).Run().Cause()
return stdout.String(), stderr.String(), err
}
func (c *Cli) Scale(ctx context.Context, namespace string, resource string, replicas uint) error {
err := c.RunCommand(ctx, "scale", "-n", namespace, fmt.Sprintf("--replicas=%d", replicas), resource, "--timeout=300s")
if err != nil {
return err
}
time.Sleep(2 * time.Second) // Sleep a bit so the container starts
return c.RunCommand(ctx, "wait", "-n", namespace, "--for=condition=available", resource, "--timeout=300s")
}
// RestartDeployment restarts a deployment. It does not wait for the deployment to be ready.
func (c *Cli) RestartDeployment(ctx context.Context, name string, extraArgs ...string) error {
args := append([]string{
"rollout",
"restart",
fmt.Sprintf("deployment/%s", name),
}, extraArgs...)
return c.RunCommand(ctx, args...)
}
// RestartDeploymentAndWait restarts a deployment and waits for it to become healthy.
func (c *Cli) RestartDeploymentAndWait(ctx context.Context, name string, extraArgs ...string) error {
if err := c.RestartDeployment(ctx, name, extraArgs...); err != nil {
return err
}
return c.DeploymentRolloutStatus(ctx, name, extraArgs...)
}
// Describe the container status (equivalent of running `kubectl describe`)
func (c *Cli) Describe(ctx context.Context, namespace string, name string) (string, error) {
stdout, stderr, err := c.Execute(ctx, "-n", namespace, "describe", name)
return stdout + stderr, err
}
// GetContainerLogs retrieves the logs for the specified container
func (c *Cli) GetContainerLogs(ctx context.Context, namespace string, name string, options ...LogOption) (string, error) {
args := []string{"-n", namespace, "logs", name}
args = append(args, BuildLogArgs(options...)...)
stdout, stderr, err := c.Execute(ctx, args...)
return stdout + stderr, err
}
// GetPodsInNsWithLabel returns the pods in the specified namespace with the specified label
func (c *Cli) GetPodsInNsWithLabel(ctx context.Context, namespace string, label string) ([]string, error) {
podStdOut := bytes.NewBuffer(nil)
podStdErr := bytes.NewBuffer(nil)
// Fetch the names of the pods with the given label
getPodNamesCmd := c.Command(ctx, "get", "pod", "-n", namespace,
"--selector", label, "--output", "jsonpath='{.items[*].metadata.name}'")
err := getPodNamesCmd.WithStdout(podStdOut).WithStderr(podStdErr).Run().Cause()
if err != nil {
fmt.Printf("error running get pod names command: %v\n", err)
}
// Clean up and check the output
podNamesString := strings.Trim(podStdOut.String(), "'")
if podNamesString == "" {
fmt.Printf("no %s pods found in namespace %s\n", label, namespace)
return []string{}, nil
}
// Split the string on whitespace to get the pod names
podNames := strings.Fields(podNamesString)
return podNames, nil
}
func (c *Cli) GetLeaseHolder(ctx context.Context, namespace string, leaderElectionID string) (string, error) {
stdout, stderr, err := c.Execute(ctx, "get", "leases", "-n", namespace,
leaderElectionID, "--output", "jsonpath='{.spec.holderIdentity}'")
stdout = strings.Trim(stdout, "'")
return stdout + stderr, err
}