Nina Polshakova 0f71c997a4
Fix e2e test setup on mac (#1117)
To test e2e tests locally on a mac you can run the setup with:
```
TEST_MODE=e2e ./controller/test/setup/setup-kind-ci.sh
```

Then run the tests with:
```
 USE_PORTFORWARD=true PERSIST_INSTALL=true CGO_ENABLED=0 go test -vet=off -ldflags='-s -w' --failfast -tags=e2e -v ./controller/test/e2e/tests -run 'TestAgentgatewayIntegration'
```
(Where the regex can select a specific test or run the full suite)

```
--- PASS: TestAgentgatewayIntegration (49.60s)
    --- PASS: TestAgentgatewayIntegration/LocalRateLimit (0.24s)
        --- SKIP: TestAgentgatewayIntegration/LocalRateLimit/TestLocalRateLimitDisabledForRoute (0.00s)
        --- PASS: TestAgentgatewayIntegration/LocalRateLimit/TestLocalRateLimitForGateway (0.05s)
        --- PASS: TestAgentgatewayIntegration/LocalRateLimit/TestLocalRateLimitForRoute (0.07s)
        --- SKIP: TestAgentgatewayIntegration/LocalRateLimit/TestLocalRateLimitForRouteUsingExtensionRef (0.00s)
    --- PASS: TestAgentgatewayIntegration/RBAC (0.13s)
        --- PASS: TestAgentgatewayIntegration/RBAC/TestRBACHeaderAuthorization (0.03s)
    --- PASS: TestAgentgatewayIntegration/MCP (12.19s)
        --- PASS: TestAgentgatewayIntegration/MCP/TestDynamicMCPAdminRouting (0.43s)
        --- PASS: TestAgentgatewayIntegration/MCP/TestDynamicMCPAdminVsUserTools (0.37s)
        --- PASS: TestAgentgatewayIntegration/MCP/TestDynamicMCPDefaultRouting (0.18s)
        --- PASS: TestAgentgatewayIntegration/MCP/TestDynamicMCPUserRouting (0.20s)
        --- PASS: TestAgentgatewayIntegration/MCP/TestMCPAuthn (3.01s)
        --- PASS: TestAgentgatewayIntegration/MCP/TestMCPWorkflow (0.21s)
        --- PASS: TestAgentgatewayIntegration/MCP/TestSSEEndpoint (0.09s)
    --- PASS: TestAgentgatewayIntegration/AIBackend (1.20s)
        --- PASS: TestAgentgatewayIntegration/AIBackend/TestPromptGuard (0.02s)
        --- PASS: TestAgentgatewayIntegration/AIBackend/TestRouting (0.01s)
        --- SKIP: TestAgentgatewayIntegration/AIBackend/TestWebhook (0.00s)
    --- PASS: TestAgentgatewayIntegration/PolicyStatus (0.17s)
        --- PASS: TestAgentgatewayIntegration/PolicyStatus/TestAgwPolicyClearStaleStatus (0.09s)
    --- PASS: TestAgentgatewayIntegration/A2A (1.18s)
        --- PASS: TestAgentgatewayIntegration/A2A/TestA2AAgentCard (0.03s)
        --- PASS: TestAgentgatewayIntegration/A2A/TestA2AHelloWorld (0.03s)
        --- PASS: TestAgentgatewayIntegration/A2A/TestA2AMessageSend (0.01s)
    --- PASS: TestAgentgatewayIntegration/BasicRouting (0.13s)
        --- PASS: TestAgentgatewayIntegration/BasicRouting/TestAgentgatewayHTTPRoute (0.04s)
        --- PASS: TestAgentgatewayIntegration/BasicRouting/TestAgentgatewayTCPRoute (0.02s)
    --- PASS: TestAgentgatewayIntegration/BackendTLSPolicy (0.34s)
        --- PASS: TestAgentgatewayIntegration/BackendTLSPolicy/TestBackendTLSPolicyAndStatus (0.20s)
    --- PASS: TestAgentgatewayIntegration/BasicAuth (0.22s)
        --- PASS: TestAgentgatewayIntegration/BasicAuth/TestGatewayPolicy (0.05s)
        --- PASS: TestAgentgatewayIntegration/BasicAuth/TestRoutePolicy (0.08s)
    --- PASS: TestAgentgatewayIntegration/JwtAuth (0.27s)
        --- PASS: TestAgentgatewayIntegration/JwtAuth/TestGatewayPolicy (0.05s)
        --- PASS: TestAgentgatewayIntegration/JwtAuth/TestGatewayPolicyWithRbac (0.04s)
        --- PASS: TestAgentgatewayIntegration/JwtAuth/TestRoutePolicy (0.06s)
        --- PASS: TestAgentgatewayIntegration/JwtAuth/TestRoutePolicyWithRbac (0.04s)
    --- PASS: TestAgentgatewayIntegration/RemoteJwtAuth (8.89s)
        --- PASS: TestAgentgatewayIntegration/RemoteJwtAuth/TestGatewayPolicyBackend (0.40s)
        --- PASS: TestAgentgatewayIntegration/RemoteJwtAuth/TestGatewayPolicyBackendWithTlsPolicy (0.92s)
        --- PASS: TestAgentgatewayIntegration/RemoteJwtAuth/TestGatewayPolicySvc (0.04s)
        --- PASS: TestAgentgatewayIntegration/RemoteJwtAuth/TestGatewayPolicySvcCaCert (1.06s)
        --- PASS: TestAgentgatewayIntegration/RemoteJwtAuth/TestGatewayPolicyWithRbac (0.88s)
        --- PASS: TestAgentgatewayIntegration/RemoteJwtAuth/TestRoutePolicyBackend (1.08s)
        --- PASS: TestAgentgatewayIntegration/RemoteJwtAuth/TestRoutePolicyBackendAndTlsPolicy (0.04s)
        --- PASS: TestAgentgatewayIntegration/RemoteJwtAuth/TestRoutePolicySvc (0.90s)
        --- PASS: TestAgentgatewayIntegration/RemoteJwtAuth/TestRoutePolicySvcCaCert (1.06s)
        --- PASS: TestAgentgatewayIntegration/RemoteJwtAuth/TestRoutePolicyWithRbac (0.89s)
    --- PASS: TestAgentgatewayIntegration/Tracing (16.41s)
        --- PASS: TestAgentgatewayIntegration/Tracing/TestOTelTracing (15.29s)
    --- PASS: TestAgentgatewayIntegration/ApiKeyAuth (0.21s)
        --- PASS: TestAgentgatewayIntegration/ApiKeyAuth/TestGatewayPolicy (0.06s)
        --- PASS: TestAgentgatewayIntegration/ApiKeyAuth/TestRoutePolicy (0.08s)
    --- PASS: TestAgentgatewayIntegration/Extauth (1.26s)
        --- PASS: TestAgentgatewayIntegration/Extauth/TestExtAuthPolicy (0.05s)
            --- PASS: TestAgentgatewayIntegration/Extauth/TestExtAuthPolicy/request_allowed_with_allow_header (0.02s)
            --- PASS: TestAgentgatewayIntegration/Extauth/TestExtAuthPolicy/request_denied_without_allow_header (0.00s)
            --- PASS: TestAgentgatewayIntegration/Extauth/TestExtAuthPolicy/request_denied_with_deny_header (0.00s)
        --- PASS: TestAgentgatewayIntegration/Extauth/TestExtAuthPolicyMissingBackendRef (0.04s)
            --- PASS: TestAgentgatewayIntegration/Extauth/TestExtAuthPolicyMissingBackendRef/request_denied_for_invalid_extauth_policy_due_to_missing_backendRef (0.02s)
        --- PASS: TestAgentgatewayIntegration/Extauth/TestRouteTargetedExtAuthPolicy (0.05s)
            --- PASS: TestAgentgatewayIntegration/Extauth/TestRouteTargetedExtAuthPolicy/request_allowed_by_default (0.00s)
            --- PASS: TestAgentgatewayIntegration/Extauth/TestRouteTargetedExtAuthPolicy/request_allowed_with_allow_header_on_secured_route (0.02s)
            --- PASS: TestAgentgatewayIntegration/Extauth/TestRouteTargetedExtAuthPolicy/request_denied_without_header_on_secured_route (0.01s)
    --- PASS: TestAgentgatewayIntegration/GlobalRateLimit (4.42s)
        --- PASS: TestAgentgatewayIntegration/GlobalRateLimit/TestCombinedLocalAndGlobalRateLimit (0.07s)
        --- PASS: TestAgentgatewayIntegration/GlobalRateLimit/TestGlobalRateLimitByPath (0.07s)
        --- PASS: TestAgentgatewayIntegration/GlobalRateLimit/TestGlobalRateLimitByRemoteAddress (0.06s)
        --- PASS: TestAgentgatewayIntegration/GlobalRateLimit/TestGlobalRateLimitByUserID (0.04s)
    --- PASS: TestAgentgatewayIntegration/Transformation (0.19s)
        --- PASS: TestAgentgatewayIntegration/Transformation/TestGatewayWithTransformedGRPCRoute (0.06s)
        --- PASS: TestAgentgatewayIntegration/Transformation/TestGatewayWithTransformedHTTPRoute (0.05s)
    --- PASS: TestAgentgatewayIntegration/CSRF (0.13s)
        --- PASS: TestAgentgatewayIntegration/CSRF/TestGatewayLevelCSRF (0.06s)
    --- PASS: TestAgentgatewayIntegration/Extproc (1.71s)
        --- PASS: TestAgentgatewayIntegration/Extproc/TestExtProcWithGatewayTargetRef (0.06s)
            --- PASS: TestAgentgatewayIntegration/Extproc/TestExtProcWithGatewayTargetRef/first_route_should_have_ExtProc_applied_via_Gateway_policy (0.01s)
            --- PASS: TestAgentgatewayIntegration/Extproc/TestExtProcWithGatewayTargetRef/second_route_also_has_ExtProc_applied_via_Gateway_policy (0.00s)
        --- PASS: TestAgentgatewayIntegration/Extproc/TestExtProcWithHTTPRouteTargetRef (0.03s)
            --- PASS: TestAgentgatewayIntegration/Extproc/TestExtProcWithHTTPRouteTargetRef/route_with_ExtProc_applied_should_have_header_modified (0.00s)
            --- PASS: TestAgentgatewayIntegration/Extproc/TestExtProcWithHTTPRouteTargetRef/route_without_ExtProc_should_not_have_header_modified (0.00s)
PASS
ok      github.com/agentgateway/agentgateway/controller/test/e2e/tests  50.401s

```

Changes made:
- `USE_PORTFORWARD` will run the tests with the gateway port-forwarded,
removing the dependency of having to set up the cloud provider kind
workaround https://github.com/kubernetes-sigs/cloud-provider-kind
- Small fixes to makefile to reflect new agentgateway repo paths 
- Fixes setup script to work with mac commands (`wait` fix for sibling
pids, fix proxy build)
- Introduce `docker-ci` to have a fast version of building locally

---------

Signed-off-by: npolshakova <nina.polshakova@solo.io>
2026-03-04 12:47:34 -05:00

348 lines
13 KiB
Go

//go:build e2e
package e2e
import (
"context"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"testing"
"istio.io/istio/pkg/slices"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/labels"
"sigs.k8s.io/controller-runtime/pkg/client"
"github.com/agentgateway/agentgateway/controller/pkg/utils/fsutils"
"github.com/agentgateway/agentgateway/controller/pkg/utils/helmutils"
"github.com/agentgateway/agentgateway/controller/test/e2e/testutils/actions"
"github.com/agentgateway/agentgateway/controller/test/e2e/testutils/assertions"
"github.com/agentgateway/agentgateway/controller/test/e2e/testutils/cluster"
"github.com/agentgateway/agentgateway/controller/test/e2e/testutils/install"
testruntime "github.com/agentgateway/agentgateway/controller/test/e2e/testutils/runtime"
"github.com/agentgateway/agentgateway/controller/test/helpers"
"github.com/agentgateway/agentgateway/controller/test/testutils"
)
// CreateTestInstallation is the simplest way to construct a TestInstallation in kgateway.
// It is syntactic sugar on top of CreateTestInstallationForCluster
func CreateTestInstallation(
t *testing.T,
installContext *install.Context,
) *TestInstallation {
runtimeContext := testruntime.NewContext()
clusterContext := cluster.MustKindContext(runtimeContext.ClusterName)
if err := install.ValidateInstallContext(installContext); err != nil {
// We error loudly if the context is misconfigured
panic(err)
}
return CreateTestInstallationForCluster(t, runtimeContext, clusterContext, installContext)
}
// CreateTestInstallationForCluster is the standard way to construct a TestInstallation
// It accepts context objects from 3 relevant sources:
//
// runtime - These are properties that are supplied at runtime and will impact how tests are executed
// cluster - These are properties that are used to connect to the Kubernetes cluster
// install - These are properties that are relevant to how the kgateway installation will be configured
func CreateTestInstallationForCluster(
t *testing.T,
runtimeContext testruntime.Context,
clusterContext *cluster.Context,
installContext *install.Context,
) *TestInstallation {
installation := &TestInstallation{
// RuntimeContext contains the set of properties that are defined at runtime by whoever is invoking tests
RuntimeContext: runtimeContext,
// ClusterContext contains the metadata about the Kubernetes Cluster that is used for this TestCluster
ClusterContext: clusterContext,
// Maintain a reference to the Metadata used for this installation
Metadata: installContext,
// Create an actions provider, and point it to the running installation
Actions: actions.NewActionsProvider().
WithClusterContext(clusterContext).
WithInstallContext(installContext),
// Create an assertions provider, and point it to the running installation
Assertions: assertions.NewProvider(t).
WithClusterContext(clusterContext).
WithInstallContext(installContext),
// Create an assertions provider function that returns a new provider for each test
// This ensures each test gets its own properly scoped testing.T
AssertionsT: func(t *testing.T) *assertions.Provider {
return assertions.NewProvider(t).
WithClusterContext(clusterContext).
WithInstallContext(installContext)
},
// GeneratedFiles contains the unique location where files generated during the execution
// of tests against this installation will be stored
// By creating a unique location, per TestInstallation and per Cluster.Name we guarantee isolation
// between TestInstallation outputs per CI run
GeneratedFiles: MustGeneratedFiles(installContext.InstallNamespace, clusterContext.Name),
}
testutils.Cleanup(t, func() {
installation.finalize()
})
return installation
}
// TestInstallation is the structure around a set of tests that validate behavior for an installation
// of kgateway.
type TestInstallation struct {
fmt.Stringer
// RuntimeContext contains the set of properties that are defined at runtime by whoever is invoking tests
RuntimeContext testruntime.Context
// ClusterContext contains the metadata about the Kubernetes Cluster that is used for this TestCluster
ClusterContext *cluster.Context
// Metadata contains the properties used to install kgateway
Metadata *install.Context
// Actions is the entity that creates actions that can be executed by the Operator
Actions *actions.Provider
// Assertions is the entity that creates assertions that can be executed by the Operator
// DEPRECATED: Use AssertionsT instead (which is scoped to a specific test and not the root suite)
Assertions *assertions.Provider
// AssertionsT is a function that creates assertions for a specific test using the test-scoped testing.T
// This ensures that assertion failures are properly attributed to the correct test
AssertionsT func(*testing.T) *assertions.Provider
// GeneratedFiles is the collection of directories and files that this test installation _may_ create
GeneratedFiles GeneratedFiles
// IstioctlBinary is the path to the istioctl binary that can be used to interact with Istio
IstioctlBinary string
}
func (i *TestInstallation) String() string {
return i.Metadata.InstallNamespace
}
func (i *TestInstallation) finalize() {
if err := os.RemoveAll(i.GeneratedFiles.TempDir); err != nil {
panic(fmt.Sprintf("Failed to remove temporary directory: %s", i.GeneratedFiles.TempDir))
}
}
// InstallFromLocalChart installs the controller and CRD chart based on the `ChartType` of the underlying
// TestInstallation. By default `kgateway` will be installed but can be set to `agentgateway`
func (i *TestInstallation) InstallFromLocalChart(ctx context.Context, t *testing.T) {
i.InstallAgentgatewayCRDsFromLocalChart(ctx, t)
i.InstallAgentgatewayCoreFromLocalChart(ctx, t)
}
// InstallAgentgatewayCRDsFromLocalChart installs the agentgateway CRD chart from the local filesystem
func (i *TestInstallation) InstallAgentgatewayCRDsFromLocalChart(ctx context.Context, t *testing.T) {
if testutils.ShouldSkipInstallAndTeardown() {
return
}
// Check if we should skip installation if the release already exists (PERSIST_INSTALL or FAIL_FAST_AND_PERSIST mode)
if testutils.ShouldPersistInstall() || testutils.ShouldFailFastAndPersist() {
if i.releaseExists(ctx, helmutils.AgentgatewayCRDChartName, i.Metadata.InstallNamespace) {
return
}
}
// Use absolute chart paths so tests work regardless of current working directory.
crdChartPath := filepath.Join(fsutils.GetModuleRoot(), "controller", "install", "helm", "agentgateway-crds")
// install the CRD chart first
err := i.Actions.Helm().WithReceiver(os.Stdout).Upgrade(
ctx,
helmutils.InstallOpts{
CreateNamespace: true,
ReleaseName: helmutils.AgentgatewayCRDChartName,
Namespace: i.Metadata.InstallNamespace,
Chart: crdChartPath,
})
i.AssertionsT(t).Require.NoError(err)
}
// InstallAgentgatewayCoreFromLocalChart installs the agentgateway main chart from the local filesystem
func (i *TestInstallation) InstallAgentgatewayCoreFromLocalChart(ctx context.Context, t *testing.T) {
if testutils.ShouldSkipInstallAndTeardown() {
return
}
// Check if we should skip installation if the release already exists (PERSIST_INSTALL or FAIL_FAST_AND_PERSIST mode)
if testutils.ShouldPersistInstall() || testutils.ShouldFailFastAndPersist() {
if i.releaseExists(ctx, helmutils.AgentgatewayChartName, i.Metadata.InstallNamespace) {
return
}
}
// Use absolute chart paths so tests work regardless of current working directory.
coreChartPath := filepath.Join(fsutils.GetModuleRoot(), "controller", "install", "helm", "agentgateway")
// and then install the main chart
err := i.Actions.Helm().WithReceiver(os.Stdout).Upgrade(
ctx,
helmutils.InstallOpts{
Namespace: i.Metadata.InstallNamespace,
CreateNamespace: true,
ValuesFiles: []string{
i.Metadata.ProfileValuesManifestFile,
i.Metadata.ValuesManifestFile,
ManifestPath("agent-gateway-integration.yaml"),
},
ReleaseName: helmutils.AgentgatewayChartName,
Chart: coreChartPath,
ExtraArgs: i.Metadata.ExtraHelmArgs,
})
i.AssertionsT(t).Require.NoError(err)
i.AssertionsT(t).EventuallyGatewayInstallSucceeded(ctx)
}
func (i *TestInstallation) Uninstall(ctx context.Context, t *testing.T) {
i.UninstallAgentgatewayCore(ctx, t)
i.UninstallAgentgatewayCRDs(ctx, t)
}
// UninstallAgentgatewayCore uninstalls the agentgateway main chart
func (i *TestInstallation) UninstallAgentgatewayCore(ctx context.Context, t *testing.T) {
if testutils.ShouldSkipInstallAndTeardown() || testutils.ShouldPersistInstall() {
return
}
// Check if the release exists before attempting to uninstall
if !i.releaseExists(ctx, helmutils.AgentgatewayChartName, i.Metadata.InstallNamespace) {
// Release doesn't exist, nothing to uninstall
return
}
// uninstall the main chart first
err := i.Actions.Helm().Uninstall(
ctx,
helmutils.UninstallOpts{
Namespace: i.Metadata.InstallNamespace,
ReleaseName: helmutils.AgentgatewayChartName,
ExtraArgs: []string{"--wait"}, // Default timeout is 5m
},
)
i.AssertionsT(t).Require.NoError(err, "failed to uninstall main chart")
i.AssertionsT(t).EventuallyGatewayUninstallSucceeded(ctx)
}
// UninstallAgentgatewayCRDs uninstalls the agentgateway CRD chart
func (i *TestInstallation) UninstallAgentgatewayCRDs(ctx context.Context, t *testing.T) {
if testutils.ShouldSkipInstallAndTeardown() || testutils.ShouldPersistInstall() {
return
}
// Check if the release exists before attempting to uninstall
if !i.releaseExists(ctx, helmutils.AgentgatewayCRDChartName, i.Metadata.InstallNamespace) {
// Release doesn't exist, nothing to uninstall
return
}
// uninstall the CRD chart
err := i.Actions.Helm().Uninstall(
ctx,
helmutils.UninstallOpts{
Namespace: i.Metadata.InstallNamespace,
ReleaseName: helmutils.AgentgatewayCRDChartName,
ExtraArgs: []string{"--wait"}, // Default timeout is 5m
},
)
i.AssertionsT(t).Require.NoError(err, "failed to uninstall CRD chart")
}
// PreFailHandler is the function that is invoked if a test in the given TestInstallation fails
func (i *TestInstallation) PreFailHandler(ctx context.Context, t *testing.T) {
i.preFailHandler(ctx, t, i.GeneratedFiles.FailureDir)
}
// PerTestPreFailHandler is the function that is invoked if a test in the given TestInstallation fails
func (i *TestInstallation) PerTestPreFailHandler(ctx context.Context, t *testing.T, testName string) {
i.preFailHandler(ctx, t, filepath.Join(i.GeneratedFiles.FailureDir, testName))
}
// preFailHandler is the function that is invoked if a test in the given TestInstallation fails
func (i *TestInstallation) preFailHandler(ctx context.Context, t *testing.T, dir string) {
// The idea here is we want to accumulate ALL information about this TestInstallation into a single directory
// That way we can upload it in CI, or inspect it locally
err := os.Mkdir(dir, os.ModePerm)
// We don't want to fail on the output directory already existing. This could occur
// if multiple tests running in the same cluster from the same installation namespace
// fail.
if err != nil && !errors.Is(err, fs.ErrExist) {
i.AssertionsT(t).Require.NoError(err, "failed to create failure directory")
}
// The kubernetes/e2e tests may use multiple namespaces, so we need to dump all of them
namespaces, err := i.Actions.Kubectl().Namespaces(ctx)
i.AssertionsT(t).Require.NoError(err, "failed to get namespaces for failure dump")
namespaces = slices.Filter(namespaces, func(s string) bool {
return s != "kube-node-lease" &&
s != "kube-public" &&
s != "kube-system" &&
s != "local-path-storage" &&
s != "metallb-system"
})
// Dump the logs and state of the cluster
helpers.StandardKgatewayDumpOnFail(os.Stdout, i.Actions.Kubectl(), dir, namespaces)
}
func (i *TestInstallation) releaseExists(ctx context.Context, releaseName, namespace string) bool {
l := &corev1.SecretList{}
if err := i.ClusterContext.Client.List(ctx, l, &client.ListOptions{
Namespace: namespace,
LabelSelector: labels.SelectorFromSet(map[string]string{
"owner": "helm",
"name": releaseName,
}),
}); err != nil {
return false
}
return len(l.Items) > 0
}
// GeneratedFiles is a collection of files that are generated during the execution of a set of tests
type GeneratedFiles struct {
// TempDir is the directory where any temporary files should be created
// Tests may create files for any number of reasons:
// - A: When a test renders objects in a file, and then uses this file to create and delete values
// - B: When a test invokes a command that produces a file as a side effect
// Files in this directory are an implementation detail of the test itself.
// As a result, it is the callers responsibility to clean up the TempDir when the tests complete
TempDir string
// FailureDir is the directory where any assets that are produced on failure will be created
FailureDir string
}
// MustGeneratedFiles returns GeneratedFiles, or panics if there was an error generating the directories
func MustGeneratedFiles(tmpDirId, clusterId string) GeneratedFiles {
tmpDir, err := os.MkdirTemp("", tmpDirId)
if err != nil {
panic(err)
}
// output path is in the format of bug_report/cluster_name
failureDir := filepath.Join(testruntime.PathToBugReport(), clusterId)
err = os.MkdirAll(failureDir, os.ModePerm)
if err != nil {
panic(err)
}
return GeneratedFiles{
TempDir: tmpDir,
FailureDir: failureDir,
}
}