Override Cassandra configuration and launch Cassandra directly

* Bypass the entrypoint script of the chosen Docker image.
* Instead, make direct changes to the Cassandra.yaml file.
* And execute ``cassandra`` directly.
* Add a pilot integration test

.gitignore

@@ -6,3 +6,4 @@
 .generate_exes
 .get_deps
 bin/
+**/.test/

Makefile

@@ -114,6 +114,6 @@ helm_verify:
 	DEBUG=1 vendor/sigs.k8s.io/testing_frameworks/integration/scripts/download-binaries.sh
 	touch	.download_integration_test_binaries
 
-test_integration: .download_integration_test_binaries apiserver
+test_integration: .download_integration_test_binaries apiserver pilot-cassandra
 	TEST_ASSET_NAVIGATOR_APISERVER=$(TEST_ASSET_NAVIGATOR_APISERVER) \
-	go test -v ./test/integration/...
+	go test $(GO_TEST_ARGS) -v ./test/integration/...

docs/cassandra.rst

@@ -25,6 +25,8 @@ All the C* nodes (pods) in a ``nodepool`` have the same configuration and the fo
 
 .. include:: configure-scheduler.rst
 
+.. _availability-zones-cassandra:
+
 Cassandra Across Multiple Availability Zones
 --------------------------------------------
 
@@ -240,6 +242,37 @@ Navigator will add C* nodes, one at a time, until the desired number of nodes is
 
 You can look at ``CassandraCluster.Status.NodePools[<nodepoolname>].ReadyReplicas`` to see the current number of healthy C* nodes in each ``nodepool``.
 
+Pilots and Cassandra Docker Images
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+By default, Navigator will use the `Cassandra Docker images from DockerHub <https://hub.docker.com/_/cassandra/>`_.
+It will use an image with a tag matching the supplied ``CassandraCluster.Spec.Version`` field.
+If you prefer to use your own container image you should configure the ``CassandraCluster.Spec.Image`` fields.
+
+Navigator installs a ``navigator-pilot-cassandra`` executable into each Pod at the path ``/pilot``.
+This ``pilot`` process connects to the API server to:
+get extra configuration settings,
+report the status of this C* node, and to
+perform leader election of a single pilot in the cluster.
+
+The ``pilot`` overrides the following keys in the default ``/etc/cassandra/cassandra.yaml`` file:
+
+* ``cluster_name``: This will be set to match the name of the corresponding ``CassandraCluster`` resource in the API server.
+* ``listen_address`` / ``listen_interface`` / ``broadcast_address`` / ``rpc_address`` / ``broadcast_rpc_address``:  These keys and the corresponding values will be removed from the default configuration.
+  This ensures that Cassandra process listens and communicates using the IP address currently associated with the fully qualified domain name of the Pod.
+  This is important if the Pod is moved to another node and is assigned a different IP address.
+  Removing these settings from the Configuration file ensures that Cassandra uses the most recent IP address that Kubernetes has assigned to the Pod and that other C* nodes in the cluster are notified of the change of IP address.
+* ``seed_provider``: This is set to ``io.jetstack.cassandra.KubernetesSeedProvider`` which allows Cassandra to look up the seed IP addresses from a Kubernetes service.
+  The ``seed_provider.*.seeds`` sub key will be removed.
+  This is to avoid the risk of nodes mistakenly joining the cluster as seeds if the seed provider service is temporarily unavailable.
+
+The ``pilot`` also overwrites ``cassandra-rackdc.properties`` with values derived from the ``CassandraCluster.Spec.Nodepools`` (see :ref:`availability-zones-cassandra`).
+
+Finally the ``pilot`` executes ``/usr/bin/cassandra`` directly.
+
+.. note::
+   The default entry point (e.g. `/docker-entrypoint.sh <https://github.com/docker-library/cassandra/blob/master/3.11/docker-entrypoint.sh>`_ is ignored.
+
 Supported Versions
 ------------------
 

hack/testdata/testpilot.yaml

@@ -1,10 +1,11 @@
 apiVersion: navigator.jetstack.io/v1alpha1
 kind: Pilot
 metadata:
-  name: richards-pilot
+  name: pilot1
+  namespace: ns1
   ownerReferences:
   - apiVersion: navigator.jetstack.io/v1alpha1
-    kind: Pilot
-    name: richards-pilot
+    kind: CassandraCluster
+    name: cluster1
     uid: 23c88696-cf7b-11e7-9ec9-0a580a200927
     controller: true

internal/test/integration/framework/framework.go

@@ -9,6 +9,8 @@ import (
 
 	"sigs.k8s.io/testing_frameworks/integration"
 
+	"github.com/pkg/errors"
+
 	"github.com/jetstack/navigator/internal/test/integration/framework/internal"
 )
 
@@ -127,6 +129,54 @@ func (f NavigatorControlPlane) NavigatorCtl() *NavigatorCtl {
 	}
 }
 
+func (f NavigatorControlPlane) NavigatorKubeConfig(path string) error {
+	ctl := f.KubeCtl()
+	ctl.Opts = []string{"--kubeconfig", path}
+	_, _, err := ctl.Run(
+		"config",
+		"set-credentials",
+		"user1",
+		"--client-certificate",
+		filepath.Join(f.NavigatorAPIServer.CertDir, "apiserver.crt"),
+		"--client-key",
+		filepath.Join(f.NavigatorAPIServer.CertDir, "apiserver.key"),
+	)
+	if err != nil {
+		return errors.Wrap(err, "unable to create user")
+	}
+	_, _, err = ctl.Run(
+		"config",
+		"set-cluster",
+		"integration1",
+		"--server",
+		f.NavigatorAPIURL().String(),
+		"--certificate-authority",
+		filepath.Join(f.NavigatorAPIServer.CertDir, "apiserver.crt"),
+	)
+	if err != nil {
+		return errors.Wrap(err, "unable to create cluster")
+	}
+	_, _, err = ctl.Run(
+		"config",
+		"set-context",
+		"default",
+		"--cluster", "integration1",
+		"--user", "user1",
+	)
+	if err != nil {
+		return errors.Wrap(err, "unable to create context")
+	}
+	_, _, err = ctl.Run(
+		"config",
+		"use-context",
+		"default",
+	)
+	if err != nil {
+		return errors.Wrap(err, "unable to use context")
+	}
+	return nil
+}
+
 type NavigatorCtl struct {
 	*integration.KubeCtl
 }

internal/test/util/testfs/testfs.go

@@ -0,0 +1,53 @@
+package testfs
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+type TestFs struct {
+	t *testing.T
+	d string
+}
+
+func New(t *testing.T) *TestFs {
+	d := fmt.Sprintf(".test/%s", t.Name())
+
+	err := os.RemoveAll(d)
+	if err != nil && !os.IsNotExist(err) {
+		t.Fatalf("Error while removing old test directory: %s", err)
+	}
+
+	err = os.MkdirAll(d, os.ModePerm)
+	require.NoError(t, err)
+
+	return &TestFs{
+		t: t,
+		d: d,
+	}
+}
+
+func (tfs *TestFs) TempPath(name string) string {
+	outPath := path.Join(tfs.d, name)
+	tmpFile, err := ioutil.TempFile(tfs.d, name)
+	require.NoError(tfs.t, err)
+	err = tmpFile.Close()
+	require.NoError(tfs.t, err)
+	err = os.Rename(tmpFile.Name(), outPath)
+	require.NoError(tfs.t, err)
+	return outPath
+}
+
+func (tfs *TestFs) TempDir(name string) string {
+	outPath := path.Join(tfs.d, name)
+	tmpDir, err := ioutil.TempDir(tfs.d, name)
+	require.NoError(tfs.t, err)
+	err = os.Rename(tmpDir, outPath)
+	require.NoError(tfs.t, err)
+	return outPath
+}

pkg/config/config.go

@@ -0,0 +1,49 @@
+package config
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/golang/glog"
+	"github.com/pkg/errors"
+	"github.com/spf13/viper"
+)
+
+type config struct {
+	*viper.Viper
+}
+
+type Interface interface {
+	WriteConfigAs(filename string) error
+	AllSettings() map[string]interface{}
+}
+
+var _ Interface = &config{}
+
+func NewFromYaml(path string) (Interface, error) {
+	path, err := filepath.Abs(path)
+	if err != nil {
+		return nil, errors.Wrap(err, "unable to read absolute path")
+	}
+	glog.V(4).Infof("Reading file: %q", path)
+	c := &config{viper.New()}
+	c.SetConfigType("yaml")
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, errors.Wrap(err, "unable to open file")
+	}
+	err = c.ReadConfig(f)
+	if err != nil {
+		return nil, errors.Wrap(err, "unable to read file")
+	}
+	return c, nil
+}
+
+func (c *config) WriteConfigAs(filename string) error {
+	path, err := filepath.Abs(filename)
+	if err != nil {
+		return errors.Wrap(err, "unable to read absolute path")
+	}
+	glog.V(4).Infof("Writing file: %q", path)
+	return c.Viper.WriteConfigAs(path)
+}

pkg/config/config_test.go

@@ -0,0 +1,29 @@
+package config_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/jetstack/navigator/internal/test/util/testfs"
+	"github.com/jetstack/navigator/pkg/config"
+)
+
+func TestRoundTrip(t *testing.T) {
+	tfs := testfs.New(t)
+
+	inPath := "testdata/config1.yaml"
+	outPath := tfs.TempPath("outPath1.yaml")
+
+	c1, err := config.NewFromYaml(inPath)
+	require.NoError(t, err)
+
+	err = c1.WriteConfigAs(outPath)
+	require.NoError(t, err)
+
+	c2, err := config.NewFromYaml(outPath)
+	require.NoError(t, err)
+
+	assert.Equal(t, c1.AllSettings(), c2.AllSettings())
+}

pkg/config/testdata/config1.yaml

@@ -0,0 +1,3 @@
+foo:
+- bar
+- baz

pkg/controllers/cassandra/nodepool/resource.go

@@ -196,6 +196,8 @@ func StatefulSetForCluster(
 									Name:  "HEAP_NEWSIZE",
 									Value: "100M",
 								},
+								// XXX: Remove all these environment variables
+								// They won't be necessary.
 								// Deliberately set to a single space to force Cassandra to do a host name lookup.
 								// See https://github.com/apache/cassandra/blob/cassandra-3.11.2/conf/cassandra.yaml#L592
 								{

pkg/pilot/cassandra/v3/options.go

@@ -18,9 +18,11 @@ import (
 )
 
 const (
-	defaultResyncPeriod = time.Second * 30
-	defaultConfigDir    = "/etc/pilot"
-	defaultJolokiaURL   = "http://127.0.0.1:8778/jolokia/"
+	defaultResyncPeriod        = time.Second * 30
+	defaultConfigDir           = "/etc/pilot"
+	defaultJolokiaURL          = "http://127.0.0.1:8778/jolokia/"
+	defaultCassandraPath       = "/usr/bin/cassandra"
+	defaultCassandraConfigPath = "/etc/cassandra/cassandra.yaml"
 )
 
 // PilotOptions are the options required to run this Pilot. This can be used to
@@ -44,6 +46,12 @@ type PilotOptions struct {
 	// JolokiaURL is the base URL of the Jolokia REST API server.
 	JolokiaURL string
 
+	// CassandraPath is the path to the cassandra executable
+	CassandraPath string
+
+	// CassandraConfigPath is the path to cassandra.yaml
+	CassandraConfigPath string
+
 	// GenericPilotOptions contains options for the genericpilot
 	GenericPilotOptions *genericpilot.Options
 
@@ -74,6 +82,8 @@ func (o *PilotOptions) AddFlags(flags *pflag.FlagSet) {
 	flags.DurationVar(&o.ResyncPeriod, "resync-period", defaultResyncPeriod, "Re-sync period for control loops operated by the pilot")
 	flags.StringVar(&o.ConfigDir, "config-dir", defaultConfigDir, "Base directory for additional Pilot configuration")
 	flags.StringVar(&o.JolokiaURL, "jolokia-url", defaultJolokiaURL, "The base URL of the Jolokia REST API server")
+	flags.StringVar(&o.CassandraPath, "cassandra-path", defaultCassandraPath, "The path to the cassandra executable")
+	flags.StringVar(&o.CassandraConfigPath, "cassandra-config-path", defaultCassandraConfigPath, "The path to cassandra.yaml")
 }
 
 func (o *PilotOptions) Complete() error {

pkg/pilot/cassandra/v3/pilot.go

@@ -42,24 +42,6 @@ func NewPilot(opts *PilotOptions) (*Pilot, error) {
 		pilotInformerSynced: pilotInformer.Informer().HasSynced,
 		nodeTool:            opts.nodeTool,
 	}
-
-	// hack to test the seedprovider, this should use whatever pattern is decided upon here:
-	//   https://github.com/jetstack/navigator/issues/251
-	cfgPath := "/etc/cassandra/cassandra.yaml"
-	read, err := ioutil.ReadFile(cfgPath)
-	if err != nil {
-		return nil, err
-	}
-
-	newContents := strings.Replace(string(read),
-		"org.apache.cassandra.locator.SimpleSeedProvider",
-		"io.jetstack.cassandra.KubernetesSeedProvider", -1)
-
-	err = ioutil.WriteFile(cfgPath, []byte(newContents), 0)
-	if err != nil {
-		return nil, err
-	}
-
 	return p, nil
 }
 
@@ -71,11 +53,15 @@ func (p *Pilot) WaitForCacheSync(stopCh <-chan struct{}) error {
 }
 
 func (p *Pilot) Hooks() *hook.Hooks {
-	return &hook.Hooks{}
+	return &hook.Hooks{
+		PreStart: []hook.Interface{
+			hook.New("WriteConfig", p.WriteConfig),
+		},
+	}
 }
 
 func (p *Pilot) CmdFunc(pilot *v1alpha1.Pilot) (*exec.Cmd, error) {
-	cmd := exec.Command("/docker-entrypoint.sh")
+	cmd := exec.Command(p.Options.CassandraPath, "-f")
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
 	return cmd, nil
@@ -129,3 +115,29 @@ func (p *Pilot) LivenessCheck() error {
 	_, err := p.nodeTool.Status()
 	return err
 }
+
+// XXX: Add a WriteConfig method which takes
+func (p *Pilot) WriteConfig(pilot *v1alpha1.Pilot) error {
+	// Look for cassandra.yaml in directory given by $CASSANDRA_CONFIG environment variable.
+	// Unmarshall it as an unstructured map of strings.
+	// Remove keys
+	// Set SeedProvider key
+	// Write back out (backing up the original)
+	// NB: Comments will be lost.
+	// Write the properties file. (backing up the original (if present))
+
+	// XXX Remove all this
+	// hack to test the seedprovider, this should use whatever pattern is decided upon here:
+	//   https://github.com/jetstack/navigator/issues/251
+	cfgPath := p.Options.CassandraConfigPath
+	read, err := ioutil.ReadFile(cfgPath)
+	if err != nil && !os.IsNotExist(err) {
+		return err
+	}
+
+	newContents := strings.Replace(string(read),
+		"org.apache.cassandra.locator.SimpleSeedProvider",
+		"io.jetstack.cassandra.KubernetesSeedProvider", -1)
+
+	return ioutil.WriteFile(cfgPath, []byte(newContents), 0)
+}