Skip to content

Commit f058043

Browse files
bjee19bjee19
authored
Add connection to NGINX One Console (#3676)
Add connection to NGINX One Console by configuring Agent to sent telemetry data to NGINX One Console endpoint. Problem: Users of NGF and NGINX One Console would like to see fleet management telemetry in their console. Solution: Update NGINX Agent configuration to send telemetry data to NGINX One Console when a user specifies their data plane key secret. Testing: Added unit tests and manually verified NGF metrics are sent to the NGINX One Console when using a staging endpoint.
1 parent 316db99 commit f058043

23 files changed

+1075
-120
lines changed

‎charts/nginx-gateway-fabric/README.md

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -264,7 +264,7 @@ The following table lists the configurable parameters of the NGINX Gateway Fabri
264264
| `certGenerator.ttlSecondsAfterFinished` | How long to wait after the cert generator job has finished before it is removed by the job controller. | int | `30` |
265265
| `clusterDomain` | The DNS cluster domain of your Kubernetes cluster. | string | `"cluster.local"` |
266266
| `gateways` | A list of Gateway objects. View https://gateway-api.sigs.k8s.io/reference/spec/#gateway for full Gateway reference. | list | `[]` |
267-
| `nginx` | The nginx section contains the configuration for all NGINX data plane deployments installed by the NGINX Gateway Fabric control plane. | object | `{"config":{},"container":{"hostPorts":[],"lifecycle":{},"readinessProbe":{},"resources":{},"volumeMounts":[]},"debug":false,"image":{"pullPolicy":"Always","repository":"ghcr.io/nginx/nginx-gateway-fabric/nginx","tag":"edge"},"imagePullSecret":"","imagePullSecrets":[],"kind":"deployment","plus":false,"pod":{},"replicas":1,"service":{"externalTrafficPolicy":"Local","loadBalancerClass":"","loadBalancerIP":"","loadBalancerSourceRanges":[],"nodePorts":[],"type":"LoadBalancer"},"usage":{"caSecretName":"","clientSSLSecretName":"","endpoint":"","resolver":"","secretName":"nplus-license","skipVerify":false}}` |
267+
| `nginx` | The nginx section contains the configuration for all NGINX data plane deployments installed by the NGINX Gateway Fabric control plane. | object | `{"config":{},"container":{"hostPorts":[],"lifecycle":{},"readinessProbe":{},"resources":{},"volumeMounts":[]},"debug":false,"image":{"pullPolicy":"Always","repository":"ghcr.io/nginx/nginx-gateway-fabric/nginx","tag":"edge"},"imagePullSecret":"","imagePullSecrets":[],"kind":"deployment","nginxOneConsole":{"dataplaneKeySecretName":"","endpointHost":"agent.connect.nginx.com","endpointPort":443,"skipVerify":false},"plus":false,"pod":{},"replicas":1,"service":{"externalTrafficPolicy":"Local","loadBalancerClass":"","loadBalancerIP":"","loadBalancerSourceRanges":[],"nodePorts":[],"type":"LoadBalancer"},"usage":{"caSecretName":"","clientSSLSecretName":"","endpoint":"","resolver":"","secretName":"nplus-license","skipVerify":false}}` |
268268
| `nginx.config` | The configuration for the data plane that is contained in the NginxProxy resource. This is applied globally to all Gateways managed by this instance of NGINX Gateway Fabric. | object | `{}` |
269269
| `nginx.container` | The container configuration for the NGINX container. This is applied globally to all Gateways managed by this instance of NGINX Gateway Fabric. | object | `{"hostPorts":[],"lifecycle":{},"readinessProbe":{},"resources":{},"volumeMounts":[]}` |
270270
| `nginx.container.hostPorts` | A list of HostPorts to expose on the host. This configuration allows containers to bind to a specific port on the host node, enabling external network traffic to reach the container directly through the host's IP address and port. Use this option when you need to expose container ports on the host for direct access, such as for debugging, legacy integrations, or when NodePort/LoadBalancer services are not suitable. Note: Using hostPort may have security and scheduling implications, as it ties pods to specific nodes and ports. | list | `[]` |
@@ -276,6 +276,11 @@ The following table lists the configurable parameters of the NGINX Gateway Fabri
276276
| `nginx.imagePullSecret` | The name of the secret containing docker registry credentials. Secret must exist in the same namespace as the helm release. The control plane will copy this secret into any namespace where NGINX is deployed. | string | `""` |
277277
| `nginx.imagePullSecrets` | A list of secret names containing docker registry credentials. Secrets must exist in the same namespace as the helm release. The control plane will copy these secrets into any namespace where NGINX is deployed. | list | `[]` |
278278
| `nginx.kind` | The kind of NGINX deployment. | string | `"deployment"` |
279+
| `nginx.nginxOneConsole` | Configuration for NGINX One Console. | object | `{"dataplaneKeySecretName":"","endpointHost":"agent.connect.nginx.com","endpointPort":443,"skipVerify":false}` |
280+
| `nginx.nginxOneConsole.dataplaneKeySecretName` | Name of the secret which holds the dataplane key that is required to authenticate with the NGINX One Console. Secret must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). | string | `""` |
281+
| `nginx.nginxOneConsole.endpointHost` | The Endpoint host that the NGINX One Console telemetry metrics will be sent to. | string | `"agent.connect.nginx.com"` |
282+
| `nginx.nginxOneConsole.endpointPort` | The endpoint port that the NGINX One Console telemetry metrics will be sent to. | int | `443` |
283+
| `nginx.nginxOneConsole.skipVerify` | Skip TLS verification for NGINX One Console connections. | bool | `false` |
279284
| `nginx.plus` | Is NGINX Plus image being used. | bool | `false` |
280285
| `nginx.pod` | The pod configuration for the NGINX data plane pod. This is applied globally to all Gateways managed by this instance of NGINX Gateway Fabric. | object | `{}` |
281286
| `nginx.replicas` | The number of replicas of the NGINX Deployment. | int | `1` |

‎charts/nginx-gateway-fabric/templates/deployment.yaml

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,18 @@ spec:
103103
{{- if .Capabilities.APIVersions.Has "security.openshift.io/v1/SecurityContextConstraints" }}
104104
- --nginx-scc={{ include "nginx-gateway.scc-name" . }}-nginx
105105
{{- end}}
106+
{{- if .Values.nginx.nginxOneConsole.dataplaneKeySecretName }}
107+
- --nginx-one-dataplane-key-secret={{ .Values.nginx.nginxOneConsole.dataplaneKeySecretName }}
108+
{{- if .Values.nginx.nginxOneConsole.endpointHost }}
109+
- --nginx-one-telemetry-endpoint-host={{ .Values.nginx.nginxOneConsole.endpointHost }}
110+
{{- end }}
111+
{{- if .Values.nginx.nginxOneConsole.endpointPort }}
112+
- --nginx-one-telemetry-endpoint-port={{ .Values.nginx.nginxOneConsole.endpointPort }}
113+
{{- end }}
114+
{{- if .Values.nginx.nginxOneConsole.skipVerify }}
115+
- --nginx-one-tls-skip-verify
116+
{{- end }}
117+
{{- end }}
106118
env:
107119
- name: POD_NAMESPACE
108120
valueFrom:

‎charts/nginx-gateway-fabric/values.schema.json

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -445,6 +445,44 @@
445445
"required": [],
446446
"title": "kind"
447447
},
448+
"nginxOneConsole": {
449+
"description": "Configuration for NGINX One Console.",
450+
"properties": {
451+
"dataplaneKeySecretName": {
452+
"default": "",
453+
"description": "Name of the secret which holds the dataplane key that is required to authenticate with the NGINX One Console.\nSecret must exist in the same namespace that the NGINX Gateway Fabric control plane is running in\n(default namespace: nginx-gateway).",
454+
"required": [],
455+
"title": "dataplaneKeySecretName",
456+
"type": "string"
457+
},
458+
"endpointHost": {
459+
"default": "agent.connect.nginx.com",
460+
"description": "The Endpoint host that the NGINX One Console telemetry metrics will be sent to.",
461+
"required": [],
462+
"title": "endpointHost",
463+
"type": "string"
464+
},
465+
"endpointPort": {
466+
"default": 443,
467+
"description": "The endpoint port that the NGINX One Console telemetry metrics will be sent to.",
468+
"maximum": 65535,
469+
"minimum": 1,
470+
"required": [],
471+
"title": "endpointPort",
472+
"type": "integer"
473+
},
474+
"skipVerify": {
475+
"default": false,
476+
"description": "Skip TLS verification for NGINX One Console connections.",
477+
"required": [],
478+
"title": "skipVerify",
479+
"type": "boolean"
480+
}
481+
},
482+
"required": [],
483+
"title": "nginxOneConsole",
484+
"type": "object"
485+
},
448486
"plus": {
449487
"default": false,
450488
"description": "Is NGINX Plus image being used.",

‎charts/nginx-gateway-fabric/values.yaml

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -212,6 +212,27 @@ nginx:
212212
# -- Is NGINX Plus image being used.
213213
plus: false
214214

215+
# -- Configuration for NGINX One Console.
216+
nginxOneConsole:
217+
# -- Name of the secret which holds the dataplane key that is required to authenticate with the NGINX One Console.
218+
# Secret must exist in the same namespace that the NGINX Gateway Fabric control plane is running in
219+
# (default namespace: nginx-gateway).
220+
dataplaneKeySecretName: ""
221+
222+
# -- The Endpoint host that the NGINX One Console telemetry metrics will be sent to.
223+
endpointHost: "agent.connect.nginx.com"
224+
225+
# @schema
226+
# type: integer
227+
# minimum: 1
228+
# maximum: 65535
229+
# @schema
230+
# -- The endpoint port that the NGINX One Console telemetry metrics will be sent to.
231+
endpointPort: 443
232+
233+
# -- Skip TLS verification for NGINX One Console connections.
234+
skipVerify: false
235+
215236
# -- The name of the secret containing docker registry credentials.
216237
# Secret must exist in the same namespace as the helm release. The control
217238
# plane will copy this secret into any namespace where NGINX is deployed.

‎cmd/gateway/commands.go

Lines changed: 74 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ import (
66
"os"
77
"runtime/debug"
88
"strconv"
9-
"strings"
109
"time"
1110

1211
"github.com/spf13/cobra"
@@ -38,8 +37,9 @@ const (
3837
`The controller name must be of the form: DOMAIN/PATH. The controller's domain is '%s'`
3938
plusFlag = "nginx-plus"
4039

41-
serverTLSSecret = "server-tls"
42-
agentTLSSecret = "agent-tls"
40+
serverTLSSecret = "server-tls"
41+
agentTLSSecret = "agent-tls"
42+
nginxOneTelemetryEndpointHost = "agent.connect.nginx.com"
4343
)
4444

4545
func createRootCommand() *cobra.Command {
@@ -58,27 +58,31 @@ func createRootCommand() *cobra.Command {
5858
func createControllerCommand() *cobra.Command {
5959
// flag names
6060
const (
61-
configFlag = "config"
62-
serviceFlag = "service"
63-
agentTLSSecretFlag = "agent-tls-secret"
64-
metricsDisableFlag = "metrics-disable"
65-
metricsSecureFlag = "metrics-secure-serving"
66-
metricsPortFlag = "metrics-port"
67-
healthDisableFlag = "health-disable"
68-
healthPortFlag = "health-port"
69-
leaderElectionDisableFlag = "leader-election-disable"
70-
leaderElectionLockNameFlag = "leader-election-lock-name"
71-
productTelemetryDisableFlag = "product-telemetry-disable"
72-
gwAPIExperimentalFlag = "gateway-api-experimental-features"
73-
nginxDockerSecretFlag = "nginx-docker-secret" //nolint:gosec // not credentials
74-
usageReportSecretFlag = "usage-report-secret"
75-
usageReportEndpointFlag = "usage-report-endpoint"
76-
usageReportResolverFlag = "usage-report-resolver"
77-
usageReportSkipVerifyFlag = "usage-report-skip-verify"
78-
usageReportClientSSLSecretFlag = "usage-report-client-ssl-secret" //nolint:gosec // not credentials
79-
usageReportCASecretFlag = "usage-report-ca-secret" //nolint:gosec // not credentials
80-
snippetsFiltersFlag = "snippets-filters"
81-
nginxSCCFlag = "nginx-scc"
61+
configFlag = "config"
62+
serviceFlag = "service"
63+
agentTLSSecretFlag = "agent-tls-secret"
64+
nginxOneDataplaneKeySecretFlag = "nginx-one-dataplane-key-secret" //nolint:gosec // not credentials
65+
nginxOneTelemetryEndpointHostFlag = "nginx-one-telemetry-endpoint-host"
66+
nginxOneTelemetryEndpointPortFlag = "nginx-one-telemetry-endpoint-port"
67+
nginxOneTLSSkipVerifyFlag = "nginx-one-tls-skip-verify"
68+
metricsDisableFlag = "metrics-disable"
69+
metricsSecureFlag = "metrics-secure-serving"
70+
metricsPortFlag = "metrics-port"
71+
healthDisableFlag = "health-disable"
72+
healthPortFlag = "health-port"
73+
leaderElectionDisableFlag = "leader-election-disable"
74+
leaderElectionLockNameFlag = "leader-election-lock-name"
75+
productTelemetryDisableFlag = "product-telemetry-disable"
76+
gwAPIExperimentalFlag = "gateway-api-experimental-features"
77+
nginxDockerSecretFlag = "nginx-docker-secret" //nolint:gosec // not credentials
78+
usageReportSecretFlag = "usage-report-secret"
79+
usageReportEndpointFlag = "usage-report-endpoint"
80+
usageReportResolverFlag = "usage-report-resolver"
81+
usageReportSkipVerifyFlag = "usage-report-skip-verify"
82+
usageReportClientSSLSecretFlag = "usage-report-client-ssl-secret" //nolint:gosec // not credentials
83+
usageReportCASecretFlag = "usage-report-ca-secret" //nolint:gosec // not credentials
84+
snippetsFiltersFlag = "snippets-filters"
85+
nginxSCCFlag = "nginx-scc"
8286
)
8387

8488
// flag values
@@ -101,7 +105,19 @@ func createControllerCommand() *cobra.Command {
101105
validator: validateResourceName,
102106
value: agentTLSSecret,
103107
}
104-
nginxSCCName = stringValidatingValue{
108+
nginxOneConsoleDataplaneKeySecretName = stringValidatingValue{
109+
validator: validateResourceName,
110+
}
111+
nginxOneConsoleTelemetryEndpointHost = stringValidatingValue{
112+
validator: validateResourceName,
113+
value: nginxOneTelemetryEndpointHost,
114+
}
115+
nginxOneConsoleTelemetryEndpointPort = intValidatingValue{
116+
validator: validateAnyPort,
117+
value: 443,
118+
}
119+
nginxOneConsoleTLSSkipVerify bool
120+
nginxSCCName = stringValidatingValue{
105121
validator: validateResourceName,
106122
}
107123
disableMetrics bool
@@ -257,6 +273,12 @@ func createControllerCommand() *cobra.Command {
257273
NginxDockerSecretNames: nginxDockerSecrets.values,
258274
AgentTLSSecretName: agentTLSSecretName.value,
259275
NGINXSCCName: nginxSCCName.value,
276+
NginxOneConsoleTelemetryConfig: config.NginxOneConsoleTelemetryConfig{
277+
DataplaneKeySecretName: nginxOneConsoleDataplaneKeySecretName.value,
278+
EndpointHost: nginxOneConsoleTelemetryEndpointHost.value,
279+
EndpointPort: nginxOneConsoleTelemetryEndpointPort.value,
280+
EndpointTLSSkipVerify: nginxOneConsoleTLSSkipVerify,
281+
},
260282
}
261283

262284
if err := controller.StartManager(conf); err != nil {
@@ -304,6 +326,32 @@ func createControllerCommand() *cobra.Command {
304326
`NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway).`,
305327
)
306328

329+
cmd.Flags().Var(
330+
&nginxOneConsoleDataplaneKeySecretName,
331+
nginxOneDataplaneKeySecretFlag,
332+
`The name of the Secret containing the NGINX One Console's dataplane key. Must exist in the same namespace that `+
333+
`the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway).`,
334+
)
335+
336+
cmd.Flags().Var(
337+
&nginxOneConsoleTelemetryEndpointHost,
338+
nginxOneTelemetryEndpointHostFlag,
339+
`The host of the NGINX One Console's telemetry endpoint.`,
340+
)
341+
342+
cmd.Flags().Var(
343+
&nginxOneConsoleTelemetryEndpointPort,
344+
nginxOneTelemetryEndpointPortFlag,
345+
`The port of the NGINX One Console's telemetry endpoint.`,
346+
)
347+
348+
cmd.Flags().BoolVar(
349+
&nginxOneConsoleTLSSkipVerify,
350+
nginxOneTLSSkipVerifyFlag,
351+
false,
352+
"Disable client verification of the NGINX One Console's telemetry endpoint server certificate.",
353+
)
354+
307355
cmd.Flags().BoolVar(
308356
&disableMetrics,
309357
metricsDisableFlag,
@@ -741,19 +789,13 @@ func createGatewayPodConfig(version, svcName string) (config.GatewayPodConfig, e
741789
return config.GatewayPodConfig{}, err
742790
}
743791

744-
// use image tag version if set, otherwise fall back to binary version
745-
ngfVersion := version
746-
if imageParts := strings.Split(image, ":"); len(imageParts) == 2 {
747-
ngfVersion = imageParts[1]
748-
}
749-
750792
c := config.GatewayPodConfig{
751793
ServiceName: svcName,
752794
Namespace: ns,
753795
Name: name,
754796
UID: podUID,
755797
InstanceName: instance,
756-
Version: ngfVersion,
798+
Version: version,
757799
Image: image,
758800
}
759801

‎cmd/gateway/commands_test.go

Lines changed: 65 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -156,6 +156,10 @@ func TestControllerCmdFlagValidation(t *testing.T) {
156156
"--usage-report-client-ssl-secret=client-secret",
157157
"--snippets-filters",
158158
"--nginx-scc=nginx-sscc-name",
159+
"--nginx-one-dataplane-key-secret=dataplane-key-secret",
160+
"--nginx-one-telemetry-endpoint-host=telemetry-endpoint-host",
161+
"--nginx-one-telemetry-endpoint-port=443",
162+
"--nginx-one-tls-skip-verify",
159163
},
160164
wantErr: false,
161165
},
@@ -426,6 +430,66 @@ func TestControllerCmdFlagValidation(t *testing.T) {
426430
wantErr: true,
427431
expectedErrPrefix: `invalid argument "!@#$" for "--nginx-scc" flag: invalid format: `,
428432
},
433+
{
434+
name: "nginx-one-dataplane-key-secret is set to empty string",
435+
args: []string{
436+
"--nginx-one-dataplane-key-secret=",
437+
},
438+
wantErr: true,
439+
expectedErrPrefix: `invalid argument "" for "--nginx-one-dataplane-key-secret" flag: must be set`,
440+
},
441+
{
442+
name: "nginx-one-dataplane-key-secret is invalid",
443+
args: []string{
444+
"--nginx-one-dataplane-key-secret=!@#$",
445+
},
446+
wantErr: true,
447+
expectedErrPrefix: `invalid argument "!@#$" for "--nginx-one-dataplane-key-secret" flag: invalid format: `,
448+
},
449+
{
450+
name: "nginx-one-telemetry-endpoint-host is set to empty string",
451+
args: []string{
452+
"--nginx-one-telemetry-endpoint-host=",
453+
},
454+
wantErr: true,
455+
expectedErrPrefix: `invalid argument "" for "--nginx-one-telemetry-endpoint-host" flag: must be set`,
456+
},
457+
{
458+
name: "nginx-one-telemetry-endpoint-host is invalid",
459+
args: []string{
460+
"--nginx-one-telemetry-endpoint-host=!@#$",
461+
},
462+
wantErr: true,
463+
expectedErrPrefix: `invalid argument "!@#$" for "--nginx-one-telemetry-endpoint-host" ` +
464+
`flag: invalid format: `,
465+
},
466+
{
467+
name: "nginx-one-telemetry-endpoint-port is invalid type",
468+
args: []string{
469+
"--nginx-one-telemetry-endpoint-port=invalid", // not an int
470+
},
471+
wantErr: true,
472+
expectedErrPrefix: `invalid argument "invalid" for "--nginx-one-telemetry-endpoint-port" ` +
473+
`flag: failed to parse int value: strconv.ParseInt: parsing "invalid": invalid syntax`,
474+
},
475+
{
476+
name: "nginx-one-telemetry-endpoint-port is outside of range",
477+
args: []string{
478+
"--nginx-one-telemetry-endpoint-port=65536", // outside of range
479+
},
480+
wantErr: true,
481+
expectedErrPrefix: `invalid argument "65536" for "--nginx-one-telemetry-endpoint-port" flag:` +
482+
` port outside of valid port range [1 - 65535]: 65536`,
483+
},
484+
{
485+
name: "nginx-one-tls-skip-verify is not a bool",
486+
expectedErrPrefix: `invalid argument "not-a-bool" for "--nginx-one-tls-skip-verify" flag:` +
487+
` strconv.ParseBool: parsing "not-a-bool": invalid syntax`,
488+
args: []string{
489+
"--nginx-one-tls-skip-verify=not-a-bool",
490+
},
491+
wantErr: true,
492+
},
429493
}
430494

431495
// common flags validation is tested separately
@@ -753,21 +817,13 @@ func TestCreateGatewayPodConfig(t *testing.T) {
753817
Name: "my-pod",
754818
UID: "1234",
755819
InstanceName: "my-pod-xyz",
756-
Version: "tag",
820+
Version: "0.0.0",
757821
Image: "my-pod-image:tag",
758822
}
759823
cfg, err := createGatewayPodConfig(version, "svc")
760824
g.Expect(err).To(Not(HaveOccurred()))
761825
g.Expect(cfg).To(Equal(expCfg))
762826

763-
// unset image tag and use provided version
764-
g.Expect(os.Setenv("IMAGE_NAME", "my-pod-image")).To(Succeed())
765-
expCfg.Version = version
766-
expCfg.Image = "my-pod-image"
767-
cfg, err = createGatewayPodConfig(version, "svc")
768-
g.Expect(err).To(Not(HaveOccurred()))
769-
g.Expect(cfg).To(Equal(expCfg))
770-
771827
// unset image name
772828
g.Expect(os.Unsetenv("IMAGE_NAME")).To(Succeed())
773829
cfg, err = createGatewayPodConfig(version, "svc")

‎cmd/gateway/validation.go

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,15 @@ func validatePort(port int) error {
157157
return nil
158158
}
159159

160+
// validateAnyPort makes sure a given port is inside the valid range for all ports.
161+
// This includes protected ports (1-1023) and unprivileged ports (1024-65535).
162+
func validateAnyPort(port int) error {
163+
if port < 1 || port > 65535 {
164+
return fmt.Errorf("port outside of valid port range [1 - 65535]: %v", port)
165+
}
166+
return nil
167+
}
168+
160169
// ensureNoPortCollisions checks if the same port has been defined multiple times.
161170
func ensureNoPortCollisions(ports ...int) error {
162171
seen := make(map[int]struct{})

0 commit comments

Comments
 (0)