Serve HTTP bootstrap data from datastore before disk

pkg/bootstrap/bootstrap.go

@@ -3,7 +3,6 @@ package bootstrap
 import (
 	"encoding/json"
 	"io"
-	"net/http"
 	"os"
 	"path/filepath"
 	"time"
@@ -13,13 +12,6 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-func Handler(bootstrap *config.ControlRuntimeBootstrap) http.Handler {
-	return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
-		rw.Header().Set("Content-Type", "application/json")
-		ReadFromDisk(rw, bootstrap)
-	})
-}
-
 // ReadFromDisk reads the bootstrap data from the files on disk and
 // writes their content in JSON form to the given io.Writer.
 func ReadFromDisk(w io.Writer, bootstrap *config.ControlRuntimeBootstrap) error {

pkg/cluster/bootstrap.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io"
 	"net"
+	"net/http"
 	"os"
 	"path/filepath"
 	"reflect"
@@ -81,7 +82,7 @@ func (c *Cluster) Bootstrap(ctx context.Context, clusterReset bool) error {
 func (c *Cluster) shouldBootstrapLoad(ctx context.Context) (bool, bool, error) {
 	// Non-nil managedDB indicates that the database is either initialized, initializing, or joining
 	if c.managedDB != nil {
-		c.config.Runtime.HTTPBootstrap = true
+		c.config.Runtime.HTTPBootstrap = c.serveBootstrap()
 
 		isInitialized, err := c.managedDB.IsInitialized()
 		if err != nil {
@@ -387,11 +388,30 @@ func isNewerFile(path string, file bootstrap.File) (updated bool, newerOnDisk bo
 	return true, false, nil
 }
 
+// serveBootstrap sends bootstrap data to the client, a server that is joining the cluster and
+// has only a server token, and cannot use CA certs/keys to access the datastore directly.
+func (c *Cluster) serveBootstrap() http.Handler {
+	return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
+		rw.Header().Set("Content-Type", "application/json")
+		// Try getting data from the datastore first. Token has already been validated by the request handler.
+		_, token, _ := req.BasicAuth()
+		data, err := c.getBootstrapData(req.Context(), token)
+		if err != nil {
+			// If we failed to read data from the datastore, just send data from disk.
+			logrus.Warnf("Failed to retrieve HTTP bootstrap data from datastore; falling back to disk for %s: %v", req.RemoteAddr, err)
+			bootstrap.ReadFromDisk(rw, &c.config.Runtime.ControlRuntimeBootstrap)
+			return
+		}
+		logrus.Infof("Serving HTTP bootstrap from datastore for %s", req.RemoteAddr)
+		rw.Write(data)
+	})
+}
+
 // httpBootstrap retrieves bootstrap data (certs and keys, etc) from the remote server via HTTP
 // and loads it into the ControlRuntimeBootstrap struct. Unlike the storage bootstrap path,
 // this data does not need to be decrypted since it is generated on-demand by an existing server.
 func (c *Cluster) httpBootstrap(ctx context.Context) error {
-	content, err := c.clientAccessInfo.Get("/v1-" + version.Program + "/server-bootstrap")
+	content, err := c.clientAccessInfo.Get("/v1-"+version.Program+"/server-bootstrap", clientaccess.WithTimeout(15*time.Second))
 	if err != nil {
 		return err
 	}
@@ -399,7 +419,8 @@ func (c *Cluster) httpBootstrap(ctx context.Context) error {
 	return c.ReconcileBootstrapData(ctx, bytes.NewReader(content), &c.config.Runtime.ControlRuntimeBootstrap, true)
 }
 
-func (c *Cluster) retrieveInitializedDBdata(ctx context.Context) (*bytes.Buffer, error) {
+// readBootstrapFromDisk returns a buffer holding the JSON-serialized bootstrap data read from disk.
+func (c *Cluster) readBootstrapFromDisk() (*bytes.Buffer, error) {
 	var buf bytes.Buffer
 	if err := bootstrap.ReadFromDisk(&buf, &c.config.Runtime.ControlRuntimeBootstrap); err != nil {
 		return nil, err
@@ -408,13 +429,16 @@ func (c *Cluster) retrieveInitializedDBdata(ctx context.Context) (*bytes.Buffer,
 	return &buf, nil
 }
 
-// bootstrap performs cluster bootstrapping, either via HTTP (for managed databases) or direct load from datastore.
+// bootstrap retrieves cluster bootstrap data: CA certs and other common config. This uses HTTP
+// for etcd (as this node does not yet have CA data available), and direct load from datastore
+// when using kine.
 func (c *Cluster) bootstrap(ctx context.Context) error {
 	c.joining = true
 
-	// bootstrap managed database via HTTPS
-	if c.config.Runtime.HTTPBootstrap {
-		// Assuming we should just compare on managed databases
+	if c.config.Runtime.HTTPBootstrap != nil {
+		// We can only compare config when we have a server URL that we are joining against -
+		// if loading directly from the datastore we do not have any way to get the config
+		// from another server for comparison.
 		if err := c.compareConfig(); err != nil {
 			return errors.Wrap(err, "failed to validate server configuration")
 		}
@@ -517,7 +541,7 @@ func (c *Cluster) reconcileEtcd(ctx context.Context) error {
 		}
 	}
 
-	data, err := c.retrieveInitializedDBdata(reconcileCtx)
+	data, err := c.readBootstrapFromDisk()
 	if err != nil {
 		return err
 	}

pkg/cluster/storage.go

@@ -246,6 +246,25 @@ func (c *Cluster) storageBootstrap(ctx context.Context) error {
 	})
 }
 
+// getBootstrapData makes a single attempt to retrieve and decrypt bootstrap data from the datastore.
+func (c *Cluster) getBootstrapData(ctx context.Context, token string) ([]byte, error) {
+	storageClient, err := client.New(c.config.Runtime.EtcdConfig)
+	if err != nil {
+		return nil, err
+	}
+	defer storageClient.Close()
+
+	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+	defer cancel()
+
+	value, err := storageClient.Get(ctx, storageKey(token))
+	if err != nil {
+		return nil, err
+	}
+
+	return decrypt(token, value.Data)
+}
+
 // getBootstrapKeyFromStorage will list all keys that has prefix /bootstrap and will check for key that is
 // hashed with empty string and will check for any key that is hashed by different token than the one
 // passed to it, it will return error if it finds a key that is hashed with different token and will return

pkg/daemons/config/types.go

@@ -312,7 +312,6 @@ type ControlRuntimeBootstrap struct {
 type ControlRuntime struct {
 	ControlRuntimeBootstrap
 
-	HTTPBootstrap                        bool
 	APIServerReady                       <-chan struct{}
 	ContainerRuntimeReady                <-chan struct{}
 	ETCDReady                            <-chan struct{}
@@ -342,6 +341,7 @@ type ControlRuntime struct {
 	AgentToken         string
 	APIServer          http.Handler
 	Handler            http.Handler
+	HTTPBootstrap      http.Handler
 	Tunnel             http.Handler
 	Authenticator      authenticator.Request
 

pkg/server/handlers/handlers.go

@@ -14,7 +14,6 @@ import (
 	"time"
 
 	"github.com/gorilla/mux"
-	"github.com/k3s-io/k3s/pkg/bootstrap"
 	"github.com/k3s-io/k3s/pkg/cli/cmds"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/etcd"
@@ -199,8 +198,8 @@ func Readyz(control *config.Control) http.Handler {
 }
 
 func Bootstrap(control *config.Control) http.Handler {
-	if control.Runtime.HTTPBootstrap {
-		return bootstrap.Handler(&control.Runtime.ControlRuntimeBootstrap)
+	if control.Runtime.HTTPBootstrap != nil {
+		return control.Runtime.HTTPBootstrap
 	}
 	return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
 		logrus.Warnf("Received HTTP bootstrap request from %s, but embedded etcd is not enabled.", req.RemoteAddr)