VAULT-38193 Add database observations to Vault (hashicorp#8727) (hashicorp#8802)

* VAULT-38193 database observations (WIP)

* VAULT-38193 database observations

* nil check

* make it consistent

* Clean up

Co-authored-by: Violet Hynes <[email protected]>

Author: hc-github-team-secure-vault-core

builtin/logical/database/backend.go

@@ -496,6 +496,36 @@ func (b *databaseBackend) dbEvent(ctx context.Context,
 	}
 }
 
+type AdditionalDatabaseMetadata struct {
+	key   string
+	value interface{}
+}
+
+func recordDatabaseObservation(ctx context.Context, b *databaseBackend, req *logical.Request, connectionName string, observationType string,
+	additionalMetadata ...AdditionalDatabaseMetadata,
+) {
+	metadata := map[string]interface{}{
+		"path":       req.Path,
+		"client_id":  req.ClientID,
+		"entity_id":  req.EntityID,
+		"request_id": req.ID,
+	}
+
+	if connectionName != "" {
+		metadata["connection_name"] = connectionName
+	}
+
+	for _, meta := range additionalMetadata {
+		metadata[meta.key] = meta.value
+	}
+
+	err := b.RecordObservation(ctx, observationType, metadata)
+
+	if err != nil && !errors.Is(err, framework.ErrNoObservations) {
+		b.Logger().Error("error recording observation", "observationType", observationType, "error", err)
+	}
+}
+
 func (b *databaseBackend) getDatabaseConfigNameFromRotationID(path string) (string, error) {
 	if !databaseConfigNameFromRotationIDRegex.MatchString(path) {
 		return "", fmt.Errorf("no name found from rotation ID")

builtin/logical/database/observation_consts.go

@@ -0,0 +1,60 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package database
+
+const (
+	// Connection related observations:
+
+	ObservationTypeDatabaseConfigWrite     = "database/connection/config/write"
+	ObservationTypeDatabaseConfigDelete    = "database/connection/config/delete"
+	ObservationTypeDatabaseConfigRead      = "database/connection/config/read"
+	ObservationTypeDatabaseConnectionReset = "database/connection/reset"
+
+	// Reload related observations:
+	// Note: the following three observations mean that, for any reload, Vault will emit
+	// n+1 observations, where n is the number of connections that we attempt to reload.
+	// database/plugin/reload will not contain a connection_name, as it will be a summary
+	// database/connection/reload/* will be per-plugin, and will contain a connection_name
+
+	// ObservationTypeDatabaseReloadPlugin is emitted when a plugin reload is issued
+	ObservationTypeDatabaseReloadPlugin = "database/plugin/reload"
+	// ObservationTypeDatabaseReloadSuccess is emitted for each connection successfully reloaded
+	ObservationTypeDatabaseReloadSuccess = "database/connection/reload/success"
+	// ObservationTypeDatabaseReloadFail is emitted for each connection unsuccessfully reloaded
+	ObservationTypeDatabaseReloadFail = "database/connection/reload/fail"
+
+	// Role related observations
+
+	ObservationTypeDatabaseRoleCreate = "database/role/create"
+	ObservationTypeDatabaseRoleUpdate = "database/role/update"
+	ObservationTypeDatabaseRoleRead   = "database/role/read"
+	// ObservationTypeDatabaseRoleDelete is emitted whenever a role is deleted.
+	// Note that this observation does not include a connection_name, to avoid doing an
+	// additional storage read.
+	ObservationTypeDatabaseRoleDelete = "database/role/delete"
+
+	ObservationTypeDatabaseCredentialCreateSuccess = "database/credential/create/success"
+	ObservationTypeDatabaseCredentialCreateFail    = "database/credential/create/fail"
+	ObservationTypeDatabaseCredentialRenew         = "database/credential/renew"
+	ObservationTypeDatabaseCredentialRevoke        = "database/credential/revoke"
+
+	// Rotate-root observations
+
+	ObservationTypeDatabaseRotateRootSuccess = "database/rotate-root/success"
+	ObservationTypeDatabaseRotateRootFailure = "database/rotate-root/fail"
+
+	// Static role observations
+
+	ObservationTypeDatabaseRotateStaticRoleSuccess = "database/static-role/rotate/success"
+	ObservationTypeDatabaseRotateStaticRoleFailure = "database/static-role/rotate/fail"
+	ObservationTypeDatabaseStaticRoleCreate        = "database/static-role/create"
+	ObservationTypeDatabaseStaticRoleUpdate        = "database/static-role/update"
+	ObservationTypeDatabaseStaticRoleRead          = "database/static-role/read"
+	// ObservationTypeDatabaseStaticRoleDelete is emitted whenever a static role is deleted.
+	// Note that this observation does not include a connection_name, to avoid doing an
+	// additional storage read.
+	ObservationTypeDatabaseStaticRoleDelete = "database/static-role/delete"
+
+	ObservationTypeDatabaseStaticCredentialRead = "database/static-credential/read"
+)

builtin/logical/database/path_config_connection.go

@@ -117,6 +117,7 @@ func (b *databaseBackend) pathConnectionReset() framework.OperationFunc {
 		}
 
 		b.dbEvent(ctx, "reset", req.Path, name, false)
+		recordDatabaseObservation(ctx, b, req, name, ObservationTypeDatabaseConnectionReset)
 		return nil, nil
 	}
 }
@@ -194,15 +195,24 @@ func (b *databaseBackend) reloadPlugin() framework.OperationFunc {
 				if err := b.reloadConnection(ctx, req.Storage, connName); err != nil {
 					b.Logger().Error("failed to reload connection", "name", connName, "error", err)
 					b.dbEvent(ctx, "reload-connection-fail", req.Path, "", false, "name", connName)
+					recordDatabaseObservation(ctx, b, req, connName, ObservationTypeDatabaseReloadFail,
+						AdditionalDatabaseMetadata{key: "plugin_name", value: pluginName})
 					reloadFailed = append(reloadFailed, connName)
 				} else {
 					b.Logger().Debug("reloaded connection", "name", connName)
 					b.dbEvent(ctx, "reload-connection", req.Path, "", true, "name", connName)
+					recordDatabaseObservation(ctx, b, req, connName, ObservationTypeDatabaseReloadSuccess,
+						AdditionalDatabaseMetadata{key: "plugin_name", value: pluginName})
 					reloaded = append(reloaded, connName)
 				}
 			}
 		}
 
+		recordDatabaseObservation(ctx, b, req, "", ObservationTypeDatabaseReloadPlugin,
+			AdditionalDatabaseMetadata{key: "plugin_name", value: pluginName},
+			AdditionalDatabaseMetadata{key: "reloaded", value: reloaded},
+			AdditionalDatabaseMetadata{key: "reload_failed", value: reloadFailed})
+
 		resp := &logical.Response{
 			Data: map[string]interface{}{
 				"connections": reloaded,
@@ -423,6 +433,9 @@ func (b *databaseBackend) connectionReadHandler() framework.OperationFunc {
 		// remove extra nested AutomatedRotationParams key
 		// before returning response
 		delete(resp.Data, "AutomatedRotationParams")
+
+		recordDatabaseObservation(ctx, b, req, name, ObservationTypeDatabaseConfigRead)
+
 		return resp, nil
 	}
 }
@@ -445,6 +458,7 @@ func (b *databaseBackend) connectionDeleteHandler() framework.OperationFunc {
 		}
 
 		b.dbEvent(ctx, "config-delete", req.Path, name, true)
+		recordDatabaseObservation(ctx, b, req, name, ObservationTypeDatabaseConfigDelete)
 		return nil, nil
 	}
 }
@@ -658,6 +672,8 @@ available at https://www.snowflake.com/en/blog/blocking-single-factor-password-a
 		}
 
 		b.dbEvent(ctx, "config-write", req.Path, name, true)
+		recordDatabaseObservation(ctx, b, req, name, ObservationTypeDatabaseConfigWrite)
+
 		if len(resp.Warnings) == 0 {
 			return nil, nil
 		}

builtin/logical/database/path_creds_create.go

@@ -87,6 +87,19 @@ func (b *databaseBackend) pathCredsCreateRead() framework.OperationFunc {
 			return logical.ErrorResponse(fmt.Sprintf("unknown role: %s", name)), nil
 		}
 
+		defer func() {
+			if err == nil && (resp == nil || !resp.IsError()) {
+				recordDatabaseObservation(ctx, b, req, role.DBName, ObservationTypeDatabaseCredentialCreateSuccess,
+					AdditionalDatabaseMetadata{key: "role_name", value: name},
+					AdditionalDatabaseMetadata{key: "credential_type", value: role.CredentialType.String()})
+			} else {
+				b.dbEvent(ctx, "creds-create-fail", req.Path, name, modified)
+				recordDatabaseObservation(ctx, b, req, role.DBName, ObservationTypeDatabaseCredentialCreateFail,
+					AdditionalDatabaseMetadata{key: "role_name", value: name},
+					AdditionalDatabaseMetadata{key: "credential_type", value: role.CredentialType.String()})
+			}
+		}()
+
 		dbConfig, err := b.DatabaseConfig(ctx, req.Storage, role.DBName)
 		if err != nil {
 			return nil, err
@@ -281,6 +294,10 @@ func (b *databaseBackend) pathStaticCredsRead() framework.OperationFunc {
 			respData["rsa_private_key"] = string(role.StaticAccount.PrivateKey)
 		}
 
+		recordDatabaseObservation(ctx, b, req, role.DBName, ObservationTypeDatabaseStaticCredentialRead,
+			AdditionalDatabaseMetadata{key: "role_name", value: name},
+			AdditionalDatabaseMetadata{key: "credential_type", value: role.CredentialType.String()})
+
 		return &logical.Response{
 			Data: respData,
 		}, nil

builtin/logical/database/path_roles.go

@@ -257,6 +257,8 @@ func (b *databaseBackend) pathRoleDelete(ctx context.Context, req *logical.Reque
 		return nil, err
 	}
 	b.dbEvent(ctx, "role-delete", req.Path, name, true)
+	recordDatabaseObservation(ctx, b, req, "", ObservationTypeDatabaseRoleDelete,
+		AdditionalDatabaseMetadata{key: "role_name", value: name})
 	return nil, nil
 }
 
@@ -298,11 +300,14 @@ func (b *databaseBackend) pathStaticRoleDelete(ctx context.Context, req *logical
 	}
 
 	b.dbEvent(ctx, "static-role-delete", req.Path, name, true)
+	recordDatabaseObservation(ctx, b, req, "", ObservationTypeDatabaseStaticRoleDelete,
+		AdditionalDatabaseMetadata{key: "role_name", value: name})
 	return nil, merr.ErrorOrNil()
 }
 
 func (b *databaseBackend) pathStaticRoleRead(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) {
-	role, err := b.StaticRole(ctx, req.Storage, d.Get("name").(string))
+	roleName := d.Get("name").(string)
+	role, err := b.StaticRole(ctx, req.Storage, roleName)
 	if err != nil {
 		return nil, err
 	}
@@ -344,13 +349,17 @@ func (b *databaseBackend) pathStaticRoleRead(ctx context.Context, req *logical.R
 		data["rotation_statements"] = []string{}
 	}
 
+	recordDatabaseObservation(ctx, b, req, role.DBName, ObservationTypeDatabaseStaticRoleRead,
+		AdditionalDatabaseMetadata{key: "role_name", value: roleName})
+
 	return &logical.Response{
 		Data: data,
 	}, nil
 }
 
 func (b *databaseBackend) pathRoleRead(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) {
-	role, err := b.Role(ctx, req.Storage, d.Get("name").(string))
+	roleName := d.Get("name").(string)
+	role, err := b.Role(ctx, req.Storage, roleName)
 	if err != nil {
 		return nil, err
 	}
@@ -384,6 +393,9 @@ func (b *databaseBackend) pathRoleRead(ctx context.Context, req *logical.Request
 		data["renew_statements"] = []string{}
 	}
 
+	recordDatabaseObservation(ctx, b, req, role.DBName, ObservationTypeDatabaseRoleRead,
+		AdditionalDatabaseMetadata{key: "role_name", value: roleName})
+
 	return &logical.Response{
 		Data: data,
 	}, nil
@@ -427,6 +439,7 @@ func (b *databaseBackend) pathRoleCreateUpdate(ctx context.Context, req *logical
 	createOperation := (req.Operation == logical.CreateOperation)
 
 	// DB Attributes
+	var credentialType string
 	{
 		if dbNameRaw, ok := data.GetOk("db_name"); ok {
 			role.DBName = dbNameRaw.(string)
@@ -438,7 +451,7 @@ func (b *databaseBackend) pathRoleCreateUpdate(ctx context.Context, req *logical
 		}
 
 		if credentialTypeRaw, ok := data.GetOk("credential_type"); ok {
-			credentialType := credentialTypeRaw.(string)
+			credentialType = credentialTypeRaw.(string)
 			if err := role.setCredentialType(credentialType); err != nil {
 				return logical.ErrorResponse(err.Error()), nil
 			}
@@ -515,6 +528,21 @@ func (b *databaseBackend) pathRoleCreateUpdate(ctx context.Context, req *logical
 	}
 
 	b.dbEvent(ctx, fmt.Sprintf("role-%s", req.Operation), req.Path, name, true)
+
+	if createOperation {
+		recordDatabaseObservation(ctx, b, req, role.DBName, ObservationTypeDatabaseRoleCreate,
+			AdditionalDatabaseMetadata{key: "role_name", value: name},
+			AdditionalDatabaseMetadata{key: "credential_type", value: credentialType},
+			AdditionalDatabaseMetadata{key: "default_ttl", value: role.DefaultTTL},
+			AdditionalDatabaseMetadata{key: "max_ttl", value: role.MaxTTL})
+	} else {
+		recordDatabaseObservation(ctx, b, req, role.DBName, ObservationTypeDatabaseRoleUpdate,
+			AdditionalDatabaseMetadata{key: "role_name", value: name},
+			AdditionalDatabaseMetadata{key: "credential_type", value: credentialType},
+			AdditionalDatabaseMetadata{key: "default_ttl", value: role.DefaultTTL},
+			AdditionalDatabaseMetadata{key: "max_ttl", value: role.MaxTTL})
+	}
+
 	return nil, nil
 }
 
@@ -636,8 +664,9 @@ func (b *databaseBackend) pathStaticRoleCreateUpdate(ctx context.Context, req *l
 		role.Statements.Rotation = data.Get("rotation_statements").([]string)
 	}
 
+	var credentialType string
 	if credentialTypeRaw, ok := data.GetOk("credential_type"); ok {
-		credentialType := credentialTypeRaw.(string)
+		credentialType = credentialTypeRaw.(string)
 		if err := role.setCredentialType(credentialType); err != nil {
 			return logical.ErrorResponse(err.Error()), nil
 		}
@@ -799,6 +828,20 @@ func (b *databaseBackend) pathStaticRoleCreateUpdate(ctx context.Context, req *l
 	}
 	b.dbEvent(ctx, fmt.Sprintf("static-role-%s", req.Operation), req.Path, name, true)
 
+	if req.Operation == logical.CreateOperation {
+		recordDatabaseObservation(ctx, b, req, role.DBName, ObservationTypeDatabaseStaticRoleCreate,
+			AdditionalDatabaseMetadata{key: "role_name", value: name},
+			AdditionalDatabaseMetadata{key: "credential_type", value: credentialType},
+			AdditionalDatabaseMetadata{key: "default_ttl", value: role.DefaultTTL},
+			AdditionalDatabaseMetadata{key: "max_ttl", value: role.MaxTTL})
+	} else {
+		recordDatabaseObservation(ctx, b, req, role.DBName, ObservationTypeDatabaseStaticRoleUpdate,
+			AdditionalDatabaseMetadata{key: "role_name", value: name},
+			AdditionalDatabaseMetadata{key: "credential_type", value: credentialType},
+			AdditionalDatabaseMetadata{key: "default_ttl", value: role.DefaultTTL},
+			AdditionalDatabaseMetadata{key: "max_ttl", value: role.MaxTTL})
+	}
+
 	if len(response.Warnings) == 0 {
 		return nil, nil
 	}

builtin/logical/database/path_rotate_credentials.go

@@ -96,8 +96,10 @@ func (b *databaseBackend) rotateRootCredentials(ctx context.Context, req *logica
 	defer func() {
 		if err == nil {
 			b.dbEvent(ctx, "rotate-root", req.Path, name, modified)
+			recordDatabaseObservation(ctx, b, req, name, ObservationTypeDatabaseRotateRootSuccess)
 		} else {
 			b.dbEvent(ctx, "rotate-root-fail", req.Path, name, modified)
+			recordDatabaseObservation(ctx, b, req, name, ObservationTypeDatabaseRotateRootFailure)
 		}
 	}()
 
@@ -221,6 +223,20 @@ func (b *databaseBackend) pathRotateRoleCredentialsUpdate() framework.OperationF
 			return logical.ErrorResponse("no static role found for role name"), nil
 		}
 
+		// We defer after we've found that the static role exists, otherwise it's not really fair to say
+		// that the rotation failed.
+		defer func() {
+			if err == nil {
+				recordDatabaseObservation(ctx, b, req, role.DBName, ObservationTypeDatabaseRotateStaticRoleSuccess,
+					AdditionalDatabaseMetadata{key: "role_name", value: name},
+					AdditionalDatabaseMetadata{key: "credential_type", value: role.CredentialType.String()})
+			} else {
+				recordDatabaseObservation(ctx, b, req, role.DBName, ObservationTypeDatabaseRotateStaticRoleFailure,
+					AdditionalDatabaseMetadata{key: "role_name", value: name},
+					AdditionalDatabaseMetadata{key: "credential_type", value: role.CredentialType.String()})
+			}
+		}()
+
 		// In create/update of static accounts, we only care if the operation
 		// err'd , and this call does not return credentials
 		item, err := b.popFromRotationQueueByKey(name)

builtin/logical/database/secret_creds.go

@@ -89,6 +89,10 @@ func (b *databaseBackend) secretCredsRenew() framework.OperationFunc {
 		resp := &logical.Response{Secret: req.Secret}
 		resp.Secret.TTL = role.DefaultTTL
 		resp.Secret.MaxTTL = role.MaxTTL
+
+		recordDatabaseObservation(ctx, b, req, role.DBName, ObservationTypeDatabaseCredentialRenew,
+			AdditionalDatabaseMetadata{key: "role_name", value: role.Name},
+			AdditionalDatabaseMetadata{key: "credential_type", value: role.CredentialType.String()})
 		return resp, nil
 	}
 }
@@ -111,11 +115,12 @@ func (b *databaseBackend) secretCredsRevoke() framework.OperationFunc {
 		if !ok {
 			return nil, fmt.Errorf("no role name was provided")
 		}
+		roleNameString := roleNameRaw.(string)
 
 		var dbName string
 		var statements v4.Statements
 
-		role, err := b.Role(ctx, req.Storage, roleNameRaw.(string))
+		role, err := b.Role(ctx, req.Storage, roleNameString)
 		if err != nil {
 			return nil, err
 		}
@@ -168,6 +173,10 @@ func (b *databaseBackend) secretCredsRevoke() framework.OperationFunc {
 			b.CloseIfShutdown(dbi, err)
 			return nil, err
 		}
+
+		recordDatabaseObservation(ctx, b, req, dbName, ObservationTypeDatabaseCredentialRevoke,
+			AdditionalDatabaseMetadata{key: "role_name", value: roleNameString})
+
 		return resp, nil
 	}
 }