Postgresql discovery config - add OTel native receiver

cmd/otelcol/config/collector/config.d.linux/receivers/smartagent-postgresql.discovery.yaml → cmd/otelcol/config/collector/config.d.linux/receivers/postgresql.discovery.yaml

@@ -6,39 +6,22 @@
 # Please note that any changes made to this file will need to be reconciled during  #
 # upgrades of the Collector.                                                        #
 #####################################################################################
-# smartagent/postgresql:
+# postgresql:
 #   enabled: true
 #   rule:
 #     docker_observer: type == "container" and any([name, image, command], {# matches "(?i)postgres"}) and not (command matches "splunk.discovery")
 #     host_observer: type == "hostport" and command matches "(?i)postgres" and not (command matches "splunk.discovery")
 #     k8s_observer: type == "port" and pod.name matches "(?i)postgres"
 #   config:
-#     default:
-#       type: postgresql
-#       connectionString: 'sslmode=disable user={{.username}} password={{.password}}'
-#       params:
-#         username: splunk.discovery.default
-#         password: splunk.discovery.default
-#       masterDBName: splunk.discovery.default
+#     default: {}
 #   status:
 #     metrics:
 #       successful:
-#         - strict: postgres_query_count
+#         - strict: postgresql.commits
 #           first_only: true
 #           log_record:
 #             severity_text: info
 #             body: PostgreSQL receiver is working!
-#       partial:
-#         - strict: postgres_rows_inserted
-#           first_only: true
-#           log_record:
-#             severity_text: info
-#             body: >-
-#               Make sure that
-#               `CREATE EXTENSION IF NOT EXISTS pg_stat_statements;`
-#               has been run for each database you would like to monitor.
-#               For example:
-#               `psql --dbname "<db-name>" -c "CREATE EXTENSION IF NOT EXISTS pg_stat_statements;"`
 #     statements:
 #       failed:
 #         - regexp: 'connect: network is unreachable'
@@ -61,19 +44,28 @@
 #             append_pattern: true
 #             body: >-
 #               Please ensure your user credentials are correctly specified with
-#               `--set splunk.discovery.receivers.smartagent/postgresql.config.params::username="<username>"` and
-#               `--set splunk.discovery.receivers.smartagent/postgresql.config.params::password="<password>"` or
-#               `SPLUNK_DISCOVERY_RECEIVERS_smartagent_x2f_postgresql_CONFIG_params_x3a__x3a_username="<username>"` and
-#               `SPLUNK_DISCOVERY_RECEIVERS_smartagent_x2f_postgresql_CONFIG_params_x3a__x3a_password="<password>"` environment variables.
+#               `--set splunk.discovery.receivers.postgresql.config.username="<username>"` and
+#               `--set splunk.discovery.receivers.postgresql.config.password="<password>"` or
+#               `SPLUNK_DISCOVERY_RECEIVERS_postgresql_CONFIG_username="<username>"` and
+#               `SPLUNK_DISCOVERY_RECEIVERS_postgresql_CONFIG_password="<password>"` environment variables.
 #         - regexp: 'pq: database .* does not exist'
 #           first_only: true
 #           log_record:
 #             severity_text: info
 #             append_pattern: true
 #             body: >-
 #               Make sure the target database is correctly specified using the
-#               `--set splunk.discovery.receivers.smartagent/postgresql.config.masterDBName="<db>"` command or the
-#               `SPLUNK_DISCOVERY_RECEIVERS_smartagent_x2f_postgresql_CONFIG_masterDBName="<db>"` environment variable.
+#               `--set splunk.discovery.receivers.postgresql.config.databases="[<db>]"` command or the
+#               `SPLUNK_DISCOVERY_RECEIVERS_postgresql_CONFIG_databases="[<db>]"` environment variable.
+#         - regexp: 'pq: SSL is not enabled on the server'
+#           first_only: true
+#           log_record:
+#             severity_text: info
+#             append_pattern: true
+#             body: >-
+#               Make sure the target database has SSL enabled or set insecure using the
+#               `--set splunk.discovery.receivers.postgresql.config.tls::insecure="<boolean>"` command or the
+#               `SPLUNK_DISCOVERY_RECEIVERS_postgresql_CONFIG_tls_x3a__x3a_insecure="<boolean>"` environment variable.
 #         - regexp: 'pq: pg_stat_statements must be loaded via shared_preload_libraries'
 #           first_only: true
 #           log_record:

internal/confmapprovider/discovery/bundle/bundle.d/receivers/smartagent-postgresql.discovery.yaml → internal/confmapprovider/discovery/bundle/bundle.d/receivers/postgresql.discovery.yaml

@@ -2,39 +2,22 @@
 #                               Do not edit manually!                               #
 # All changes must be made to associated .tmpl file before running 'make bundle.d'. #
 #####################################################################################
-smartagent/postgresql:
+postgresql:
   enabled: true
   rule:
     docker_observer: type == "container" and any([name, image, command], {# matches "(?i)postgres"}) and not (command matches "splunk.discovery")
     host_observer: type == "hostport" and command matches "(?i)postgres" and not (command matches "splunk.discovery")
     k8s_observer: type == "port" and pod.name matches "(?i)postgres"
   config:
-    default:
-      type: postgresql
-      connectionString: 'sslmode=disable user={{.username}} password={{.password}}'
-      params:
-        username: splunk.discovery.default
-        password: splunk.discovery.default
-      masterDBName: splunk.discovery.default
+    default: {}
   status:
     metrics:
       successful:
-        - strict: postgres_query_count
+        - strict: postgresql.commits
           first_only: true
           log_record:
             severity_text: info
             body: PostgreSQL receiver is working!
-      partial:
-        - strict: postgres_rows_inserted
-          first_only: true
-          log_record:
-            severity_text: info
-            body: >-
-              Make sure that
-              `CREATE EXTENSION IF NOT EXISTS pg_stat_statements;`
-              has been run for each database you would like to monitor.
-              For example:
-              `psql --dbname "<db-name>" -c "CREATE EXTENSION IF NOT EXISTS pg_stat_statements;"`
     statements:
       failed:
         - regexp: 'connect: network is unreachable'
@@ -57,19 +40,28 @@ smartagent/postgresql:
             append_pattern: true
             body: >-
               Please ensure your user credentials are correctly specified with
-              `--set splunk.discovery.receivers.smartagent/postgresql.config.params::username="<username>"` and
-              `--set splunk.discovery.receivers.smartagent/postgresql.config.params::password="<password>"` or
-              `SPLUNK_DISCOVERY_RECEIVERS_smartagent_x2f_postgresql_CONFIG_params_x3a__x3a_username="<username>"` and
-              `SPLUNK_DISCOVERY_RECEIVERS_smartagent_x2f_postgresql_CONFIG_params_x3a__x3a_password="<password>"` environment variables.
+              `--set splunk.discovery.receivers.postgresql.config.username="<username>"` and
+              `--set splunk.discovery.receivers.postgresql.config.password="<password>"` or
+              `SPLUNK_DISCOVERY_RECEIVERS_postgresql_CONFIG_username="<username>"` and
+              `SPLUNK_DISCOVERY_RECEIVERS_postgresql_CONFIG_password="<password>"` environment variables.
         - regexp: 'pq: database .* does not exist'
           first_only: true
           log_record:
             severity_text: info
             append_pattern: true
             body: >-
               Make sure the target database is correctly specified using the
-              `--set splunk.discovery.receivers.smartagent/postgresql.config.masterDBName="<db>"` command or the
-              `SPLUNK_DISCOVERY_RECEIVERS_smartagent_x2f_postgresql_CONFIG_masterDBName="<db>"` environment variable.
+              `--set splunk.discovery.receivers.postgresql.config.databases="[<db>]"` command or the
+              `SPLUNK_DISCOVERY_RECEIVERS_postgresql_CONFIG_databases="[<db>]"` environment variable.
+        - regexp: 'pq: SSL is not enabled on the server'
+          first_only: true
+          log_record:
+            severity_text: info
+            append_pattern: true
+            body: >-
+              Make sure the target database has SSL enabled or set insecure using the
+              `--set splunk.discovery.receivers.postgresql.config.tls::insecure="<boolean>"` command or the
+              `SPLUNK_DISCOVERY_RECEIVERS_postgresql_CONFIG_tls_x3a__x3a_insecure="<boolean>"` environment variable.
         - regexp: 'pq: pg_stat_statements must be loaded via shared_preload_libraries'
           first_only: true
           log_record:

internal/confmapprovider/discovery/bundle/bundle.d/receivers/smartagent-postgresql.discovery.yaml.tmpl → internal/confmapprovider/discovery/bundle/bundle.d/receivers/postgresql.discovery.yaml.tmpl

@@ -1,36 +1,19 @@
-{{ receiver "smartagent/postgresql" }}:
+{{ receiver "postgresql" }}:
   enabled: true
   rule:
     docker_observer: type == "container" and any([name, image, command], {# matches "(?i)postgres"}) and not (command matches "splunk.discovery")
     host_observer: type == "hostport" and command matches "(?i)postgres" and not (command matches "splunk.discovery")
     k8s_observer: type == "port" and pod.name matches "(?i)postgres"
   config:
-    default:
-      type: postgresql
-      connectionString: 'sslmode=disable user={{ "{{.username}}" }} password={{ "{{.password}}" }}'
-      params:
-        username: {{ defaultValue }}
-        password: {{ defaultValue }}
-      masterDBName: {{ defaultValue }}
+    default: {}
   status:
     metrics:
       successful:
-        - strict: postgres_query_count
+        - strict: postgresql.commits
           first_only: true
           log_record:
             severity_text: info
             body: PostgreSQL receiver is working!
-      partial:
-        - strict: postgres_rows_inserted
-          first_only: true
-          log_record:
-            severity_text: info
-            body: >-
-              Make sure that
-              `CREATE EXTENSION IF NOT EXISTS pg_stat_statements;`
-              has been run for each database you would like to monitor.
-              For example:
-              `psql --dbname "<db-name>" -c "CREATE EXTENSION IF NOT EXISTS pg_stat_statements;"`
     statements:
       failed:
         - regexp: 'connect: network is unreachable'
@@ -53,19 +36,28 @@
             append_pattern: true
             body: >-
               Please ensure your user credentials are correctly specified with
-              `--set {{ configProperty "params" "username" "<username>" }}` and
-              `--set {{ configProperty "params" "password" "<password>" }}` or
-              `{{ configPropertyEnvVar "params" "username" "<username>" }}` and
-              `{{ configPropertyEnvVar "params" "password" "<password>" }}` environment variables.
+              `--set {{ configProperty "username" "<username>" }}` and
+              `--set {{ configProperty "password" "<password>" }}` or
+              `{{ configPropertyEnvVar "username" "<username>" }}` and
+              `{{ configPropertyEnvVar "password" "<password>" }}` environment variables.
         - regexp: 'pq: database .* does not exist'
           first_only: true
           log_record:
             severity_text: info
             append_pattern: true
             body: >-
               Make sure the target database is correctly specified using the
-              `--set {{ configProperty "masterDBName" "<db>" }}` command or the
-              `{{ configPropertyEnvVar "masterDBName" "<db>" }}` environment variable.
+              `--set {{ configProperty "databases" "[<db>]" }}` command or the
+              `{{ configPropertyEnvVar "databases" "[<db>]" }}` environment variable.
+        - regexp: 'pq: SSL is not enabled on the server'
+          first_only: true
+          log_record:
+            severity_text: info
+            append_pattern: true
+            body: >-
+              Make sure the target database has SSL enabled or set insecure using the
+              `--set {{ configProperty "tls::insecure" "<boolean>" }}` command or the
+              `{{ configPropertyEnvVar "tls::insecure" "<boolean>" }}` environment variable.
         - regexp: 'pq: pg_stat_statements must be loaded via shared_preload_libraries'
           first_only: true
           log_record:

internal/confmapprovider/discovery/bundle/bundle_gen.go

@@ -19,12 +19,12 @@
 // TODO: add a pre-bundle.d step that generates these generate directives for easier maintenance/management across platforms
 //go:generate go install github.com/signalfx/splunk-otel-collector/internal/confmapprovider/discovery/bundle/cmd/discoverybundler
 
-//go:generate discoverybundler -r -t bundle.d/receivers/smartagent-postgresql.discovery.yaml.tmpl
-//go:generate discoverybundler -r -c -d ../../../../cmd/otelcol/config/collector/config.d.linux/receivers -t bundle.d/receivers/smartagent-postgresql.discovery.yaml.tmpl
 //go:generate discoverybundler -r -t bundle.d/receivers/smartagent-collectd-mysql.discovery.yaml.tmpl
 //go:generate discoverybundler -r -c -d ../../../../cmd/otelcol/config/collector/config.d.linux/receivers -t bundle.d/receivers/smartagent-collectd-mysql.discovery.yaml.tmpl
 //go:generate discoverybundler -r -t bundle.d/receivers/smartagent-collectd-nginx.discovery.yaml.tmpl
 //go:generate discoverybundler -r -c -d ../../../../cmd/otelcol/config/collector/config.d.linux/receivers -t bundle.d/receivers/smartagent-collectd-nginx.discovery.yaml.tmpl
+//go:generate discoverybundler -r -t bundle.d/receivers/postgresql.discovery.yaml.tmpl
+//go:generate discoverybundler -r -c -d ../../../../cmd/otelcol/config/collector/config.d.linux/receivers -t bundle.d/receivers/postgresql.discovery.yaml.tmpl
 //go:generate discoverybundler -r -t bundle.d/receivers/oracledb.discovery.yaml.tmpl
 //go:generate discoverybundler -r -c -d ../../../../cmd/otelcol/config/collector/config.d.linux/receivers -t bundle.d/receivers/oracledb.discovery.yaml.tmpl
 //go:generate discoverybundler -r -t bundle.d/receivers/redis.discovery.yaml.tmpl

internal/confmapprovider/discovery/bundle/bundle_other_test.go

@@ -28,10 +28,10 @@ func TestBundleDir(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, []string{
 		"bundle.d/receivers/oracledb.discovery.yaml",
+		"bundle.d/receivers/postgresql.discovery.yaml",
 		"bundle.d/receivers/redis.discovery.yaml",
 		"bundle.d/receivers/smartagent-collectd-mysql.discovery.yaml",
 		"bundle.d/receivers/smartagent-collectd-nginx.discovery.yaml",
-		"bundle.d/receivers/smartagent-postgresql.discovery.yaml",
 	}, receivers)
 
 	extensions, err := fs.Glob(BundledFS, "bundle.d/extensions/*.discovery.yaml")

internal/confmapprovider/discovery/bundle/bundle_windows.go

@@ -28,6 +28,6 @@ import (
 //
 //go:embed bundle.d/extensions/*.discovery.yaml
 //go:embed bundle.d/receivers/oracledb.discovery.yaml
+//go:embed bundle.d/receivers/postgresql.discovery.yaml
 //go:embed bundle.d/receivers/redis.discovery.yaml
-//go:embed bundle.d/receivers/smartagent-postgresql.discovery.yaml
 var BundledFS embed.FS

internal/confmapprovider/discovery/bundle/bundle_windows_test.go

@@ -28,8 +28,8 @@ func TestBundleDir(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, []string{
 		"bundle.d/receivers/oracledb.discovery.yaml",
+		"bundle.d/receivers/postgresql.discovery.yaml",
 		"bundle.d/receivers/redis.discovery.yaml",
-		"bundle.d/receivers/smartagent-postgresql.discovery.yaml",
 	}, receivers)
 
 	extensions, err := fs.Glob(BundledFS, "bundle.d/extensions/*.discovery.yaml")

tests/receivers/smartagent/postgresql/bundled_test.go → tests/receivers/postgresql/bundled_test.go

@@ -38,6 +38,41 @@ import (
 	"github.com/signalfx/splunk-otel-collector/tests/testutils/kubeutils/manifests"
 )
 
+func TestPostgresqlDockerObserver(t *testing.T) {
+	testutils.SkipIfNotContainerTest(t)
+	if runtime.GOOS == "darwin" {
+		t.Skip("unable to share sockets between mac and d4m vm: https://github.com/docker/for-mac/issues/483#issuecomment-758836836")
+	}
+
+	testutils.AssertAllMetricsReceived(t, "bundled.yaml", "otlp_exporter.yaml",
+		postgresqldb, []testutils.CollectorBuilder{
+			func(c testutils.Collector) testutils.Collector {
+				cc := c.(*testutils.CollectorContainer)
+				cc.Container = cc.Container.WithBinds("/var/run/docker.sock:/var/run/docker.sock:ro")
+				cc.Container = cc.Container.WillWaitForLogs("Discovering for next")
+				cc.Container = cc.Container.WithUser(fmt.Sprintf("999:%d", testutils.GetDockerGID(t)))
+				return cc
+			},
+			func(collector testutils.Collector) testutils.Collector {
+				return collector.WithEnv(map[string]string{
+					"SPLUNK_DISCOVERY_DURATION": "10s",
+					// confirm that debug logging doesn't affect runtime
+					"SPLUNK_DISCOVERY_LOG_LEVEL": "debug",
+				}).WithArgs(
+					"--discovery",
+					"--set", "splunk.discovery.receivers.postgresql.config.username=otelu",
+					"--set", "splunk.discovery.receivers.postgresql.config.password=otelp",
+					"--set", "splunk.discovery.receivers.postgresql.config.tls::insecure=true",
+					//Disabling postgresql.backends metric that doesn't dependably show up in metrics during testing
+					"--set", "splunk.discovery.receivers.postgresql.config.metrics::postgresql.backends::enabled=false",
+					"--set", `splunk.discovery.extensions.k8s_observer.enabled=false`,
+					"--set", `splunk.discovery.extensions.host_observer.enabled=false`,
+				)
+			},
+		},
+	)
+}
+
 func TestK8sObserver(t *testing.T) {
 	testutils.SkipIfNotContainerTest(t)
 	tc := testutils.NewTestcase(t, testutils.OTLPReceiverSinkAllInterfaces)
@@ -76,30 +111,30 @@ func TestK8sObserver(t *testing.T) {
 	sout, serr, err = cluster.Apply(manifests.RenderAll(t, configMap, ds))
 	require.NoError(t, err, "stdout: %s, stderr: %s", sout, serr)
 
-	pods := cluster.WaitForPods(ds.Name, namespace.Name, 5*time.Minute)
+	pods := cluster.WaitForPods(ds.Name, namespace.Name, 10*time.Minute)
 	require.Len(t, pods, 1)
 	collectorName := pods[0].Name
 
-	expectedMetrics := tc.ResourceMetrics("all.yaml")
+	expectedMetrics := tc.ResourceMetrics("all_k8s.yaml")
 	require.NoError(t, tc.OTLPReceiverSink.AssertAllMetricsReceived(t, *expectedMetrics, 30*time.Second))
 
 	stdOut, stdErr, err := cluster.Kubectl("logs", "-n", namespace.Name, collectorName)
 	require.NoError(t, err)
 	require.Contains(
 		t, stdOut.String(),
-		fmt.Sprintf(`Successfully discovered "smartagent/postgresql" using "k8s_observer" endpoint "k8s_observer/%s/(5432)`, postgresUID),
+		fmt.Sprintf(`Successfully discovered "postgresql" using "k8s_observer" endpoint "k8s_observer/%s/(5432)`, postgresUID),
 		stdErr.String(),
 	)
 }
 
 type testCluster struct{ *kubeutils.KindCluster }
 
 func (cluster testCluster) createPostgres(name, namespace, serviceAccount string) string {
-	dbsql, err := os.ReadFile(filepath.Join(".", "testdata", "server", "initdb.d", "db.sql"))
+	dbsql, err := os.ReadFile(filepath.Join(".", "testdata", "server_k8s", "initdb.d", "db.sql"))
 	require.NoError(cluster.Testcase, err)
 	cmContent := map[string]any{"db.sql": string(dbsql)}
 
-	initsh, err := os.ReadFile(filepath.Join(".", "testdata", "server", "initdb.d", "init.sh"))
+	initsh, err := os.ReadFile(filepath.Join(".", "testdata", "server_k8s", "initdb.d", "init.sh"))
 	require.NoError(cluster.Testcase, err)
 	cmContent["init.sh"] = string(initsh)
 
@@ -231,7 +266,7 @@ func (cluster testCluster) createPostgres(name, namespace, serviceAccount string
 	)
 	require.NoError(cluster.Testcase, err)
 
-	cluster.WaitForPods(postgres.Name, namespace, 5*time.Minute)
+	cluster.WaitForPods(postgres.Name, namespace, 10*time.Minute)
 	return string(postgres.UID)
 }
 
@@ -332,10 +367,11 @@ func (cluster testCluster) daemonSet(namespace, serviceAccount, configMap string
 				Image: testutils.GetCollectorImageOrSkipTest(cluster.Testcase),
 				Command: []string{
 					"/otelcol", "--config=/config/config.yaml", "--discovery",
-					"--set", "splunk.discovery.receivers.smartagent/postgresql.config.params::username='${env:PG_USERNAME}'",
-					"--set", "splunk.discovery.receivers.smartagent/postgresql.config.params::password='${env:PG_PASSWORD}'",
-					"--set", "splunk.discovery.receivers.smartagent/postgresql.config.masterDBName=test_db",
-					"--set", `splunk.discovery.receivers.smartagent/postgresql.config.extraMetrics=["*"]`,
+					"--set", "splunk.discovery.receivers.postgresql.config.username='${env:PG_USERNAME}'",
+					"--set", "splunk.discovery.receivers.postgresql.config.password='${env:PG_PASSWORD}'",
+					"--set", "splunk.discovery.receivers.postgresql.config.tls::insecure=true",
+					//Disabling postgresql.backends metric that doesn't dependably show up in metrics during testing
+					"--set", "splunk.discovery.receivers.postgresql.config.metrics::postgresql.backends::enabled=false",
 				},
 				Env: []corev1.EnvVar{
 					{Name: "PG_USERNAME", Value: "test_user"},

tests/receivers/postgresql/testdata/client/Dockerfile

@@ -0,0 +1,6 @@
+ARG POSTGRES_VERSION=13-alpine
+FROM postgres:${POSTGRES_VERSION}
+
+COPY requests.sh /usr/local/bin/requests.sh
+
+CMD ["requests.sh"]