fix(robot): refresh stale cache on bare-metal name lookup miss

Author: guettli

hcloud/cloud_test.go

@@ -34,6 +34,7 @@ import (
 	"github.com/syself/hetzner-cloud-controller-manager/internal/annotation"
 	"github.com/syself/hetzner-cloud-controller-manager/internal/credentials"
 	"github.com/syself/hetzner-cloud-controller-manager/internal/hcops"
+	robotclient "github.com/syself/hetzner-cloud-controller-manager/internal/robot/client"
 	hrobot "github.com/syself/hrobot-go"
 	"github.com/syself/hrobot-go/models"
 	corev1 "k8s.io/api/core/v1"
@@ -44,7 +45,15 @@ type testEnv struct {
 	Server      *httptest.Server
 	Mux         *http.ServeMux
 	Client      *hcloud.Client
-	RobotClient hrobot.RobotClient
+	RobotClient robotclient.Client
+}
+
+type testRobotClient struct {
+	hrobot.RobotClient
+}
+
+func (c testRobotClient) ServerGetListForceRefresh() ([]models.Server, error) {
+	return c.ServerGetList()
 }
 
 func (env *testEnv) Teardown() {
@@ -70,7 +79,7 @@ func newTestEnv() testEnv {
 		Server:      server,
 		Mux:         mux,
 		Client:      client,
-		RobotClient: robotClient,
+		RobotClient: testRobotClient{RobotClient: robotClient},
 	}
 }
 

hcloud/instances.go

@@ -49,7 +49,12 @@ type instances struct {
 var errServerNotFound = fmt.Errorf("server not found")
 
 func newInstances(client *hcloud.Client, robotClient robotclient.Client, addressFamily addressFamily, networkID int64) *instances {
-	return &instances{client, robotClient, addressFamily, networkID}
+	return &instances{
+		client:        client,
+		robotClient:   robotClient,
+		addressFamily: addressFamily,
+		networkID:     networkID,
+	}
 }
 
 // lookupServer attempts to locate the corresponding hcloud.Server or models.Server (robot server) for a given v1.Node.

hcloud/instances_test.go

@@ -26,6 +26,7 @@ import (
 
 	"github.com/hetznercloud/hcloud-go/v2/hcloud"
 	"github.com/hetznercloud/hcloud-go/v2/hcloud/schema"
+	"github.com/syself/hetzner-cloud-controller-manager/internal/robot/client/cache"
 	"github.com/syself/hrobot-go/models"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -173,6 +174,69 @@ func TestInstances_InstanceExists(t *testing.T) {
 	}
 }
 
+func TestInstances_InstanceExistsRobotServerCreatedAfterCacheFill(t *testing.T) {
+	env := newTestEnv()
+	defer env.Teardown()
+
+	resetEnv := Setenv(t,
+		"ROBOT_USER_NAME", "user",
+		"ROBOT_PASSWORD", "pass",
+		"CACHE_TIMEOUT", "1h",
+	)
+	defer resetEnv()
+
+	// servers backs the Robot list response and is mutated during the test.
+	servers := make([]models.Server, 0, 2)
+	servers = append(servers, models.Server{
+		ServerIP:      "123.123.123.123",
+		ServerIPv6Net: "2a01:f48:111:4221::",
+		ServerNumber:  321,
+		Name:          "bm-existing",
+	})
+	env.Mux.HandleFunc("/robot/server", func(w http.ResponseWriter, _ *http.Request) {
+		responses := make([]models.ServerResponse, 0, len(servers))
+		for _, server := range servers {
+			responses = append(responses, models.ServerResponse{Server: server})
+		}
+		json.NewEncoder(w).Encode(responses)
+	})
+
+	robotClient, err := cache.NewCachedRobotClient(t.TempDir(), env.Server.Client(), env.Server.URL+"/robot")
+	if err != nil {
+		t.Fatalf("Unexpected error creating cached robot client: %v", err)
+	}
+
+	instances := newInstances(env.Client, robotClient, AddressFamilyIPv4, 0)
+
+	// Warm the cache while bm-new does not exist yet.
+	exists, err := instances.InstanceExists(context.TODO(), &corev1.Node{
+		ObjectMeta: metav1.ObjectMeta{Name: "bm-existing"},
+	})
+	if err != nil {
+		t.Fatalf("Unexpected error warming cache: %v", err)
+	}
+	if !exists {
+		t.Fatal("Expected bm-existing to exist")
+	}
+
+	servers = append(servers, models.Server{
+		ServerIP:      "123.123.123.124",
+		ServerIPv6Net: "2a01:f48:111:4222::",
+		ServerNumber:  322,
+		Name:          "bm-new",
+	})
+
+	exists, err = instances.InstanceExists(context.TODO(), &corev1.Node{
+		ObjectMeta: metav1.ObjectMeta{Name: "bm-new"},
+	})
+	if err != nil {
+		t.Fatalf("Unexpected error for bm-new: %v", err)
+	}
+	if !exists {
+		t.Fatal("Expected bm-new to exist after it was created")
+	}
+}
+
 func TestInstances_InstanceShutdown(t *testing.T) {
 	env := newTestEnv()
 	defer env.Teardown()

hcloud/util.go

@@ -71,13 +71,18 @@ func getRobotServerByName(c robotclient.Client, node *corev1.Node) (server *mode
 		return nil, fmt.Errorf("%s: %w", op, err)
 	}
 
-	for i, s := range serverList {
-		if s.Name == node.Name {
-			server = &serverList[i]
-		}
+	server = findRobotServerByName(serverList, string(node.Name))
+	if server != nil {
+		return server, nil
 	}
 
-	return server, nil
+	serverList, err = c.ServerGetListForceRefresh()
+	if err != nil {
+		hcops.HandleRateLimitExceededError(err, node)
+		return nil, fmt.Errorf("%s: force refresh after cache miss: %w", op, err)
+	}
+
+	return findRobotServerByName(serverList, string(node.Name)), nil
 }
 
 func getRobotServerByID(c robotclient.Client, id int, node *corev1.Node) (s *models.Server, e error) {
@@ -116,6 +121,15 @@ func getRobotServerByID(c robotclient.Client, id int, node *corev1.Node) (s *mod
 	return server, nil
 }
 
+func findRobotServerByName(serverList []models.Server, name string) *models.Server {
+	for i, s := range serverList {
+		if s.Name == name {
+			return &serverList[i]
+		}
+	}
+	return nil
+}
+
 func isHCloudServerByName(name string) bool {
 	return !strings.HasPrefix(name, hostNamePrefixRobot)
 }

internal/mocks/robot.go

@@ -14,6 +14,10 @@ func (m *RobotClient) ServerGetList() ([]models.Server, error) {
 	return getRobotServers(args, 0), args.Error(1)
 }
 
+func (m *RobotClient) ServerGetListForceRefresh() ([]models.Server, error) {
+	return m.ServerGetList()
+}
+
 func (m *RobotClient) SetCredentials(_, _ string) error {
 	args := m.Called()
 	return args.Error(3)

internal/robot/client/cache/client.go

@@ -90,22 +90,9 @@ func NewCachedRobotClient(rootDir string, httpClient *http.Client, baseURL strin
 
 func (c *cacheRobotClient) ServerGet(id int) (*models.Server, error) {
 	if c.shouldSync() {
-		list, err := c.robotClient.ServerGetList()
-		if err != nil {
+		if _, err := c.sync(); err != nil {
 			return nil, err
 		}
-
-		// populate list
-		c.l = list
-
-		// remove all entries from map and populate it freshly
-		c.m = make(map[int]*models.Server)
-		for i, server := range list {
-			c.m[server.ServerNumber] = &list[i]
-		}
-
-		// set time of last update
-		c.lastUpdate = time.Now()
 	}
 
 	server, found := c.m[id]
@@ -119,27 +106,17 @@ func (c *cacheRobotClient) ServerGet(id int) (*models.Server, error) {
 
 func (c *cacheRobotClient) ServerGetList() ([]models.Server, error) {
 	if c.shouldSync() {
-		list, err := c.robotClient.ServerGetList()
-		if err != nil {
-			return list, err
-		}
-
-		// populate list
-		c.l = list
-
-		// remove all entries from map and populate it freshly
-		c.m = make(map[int]*models.Server)
-		for i, server := range list {
-			c.m[server.ServerNumber] = &list[i]
-		}
-
-		// set time of last update
-		c.lastUpdate = time.Now()
+		return c.sync()
 	}
 
 	return c.l, nil
 }
 
+// ServerGetListForceRefresh bypasses the timeout check and reloads the cache from Robot.
+func (c *cacheRobotClient) ServerGetListForceRefresh() ([]models.Server, error) {
+	return c.sync()
+}
+
 func (c *cacheRobotClient) shouldSync() bool {
 	// map is nil means we have no cached value yet
 	if c.m == nil {
@@ -161,3 +138,24 @@ func (c *cacheRobotClient) SetCredentials(username, password string) error {
 	c.m = nil
 	return nil
 }
+
+func (c *cacheRobotClient) sync() ([]models.Server, error) {
+	list, err := c.robotClient.ServerGetList()
+	if err != nil {
+		return list, err
+	}
+
+	// populate list
+	c.l = list
+
+	// remove all entries from map and repopulate it from the current list
+	c.m = make(map[int]*models.Server)
+	for i, server := range list {
+		c.m[server.ServerNumber] = &list[i]
+	}
+
+	// set time of last update
+	c.lastUpdate = time.Now()
+
+	return c.l, nil
+}

internal/robot/client/interface.go

@@ -5,5 +5,6 @@ import "github.com/syself/hrobot-go/models"
 type Client interface {
 	ServerGet(id int) (*models.Server, error)
 	ServerGetList() ([]models.Server, error)
+	ServerGetListForceRefresh() ([]models.Server, error)
 	SetCredentials(username, password string) error
 }