package main import ( "bytes" "errors" "flag" "fmt" "io" "net" "os" "path/filepath" "reflect" "runtime" "strconv" "strings" "time" ) const ( DiscoModeNone = "" DiscoModeConsulKV = "consul-kv" DiscoModeEtcdKV = "etcd-kv" DiscoModeDNS = "dns" DiscoModeDNSSRV = "dns-srv" HTTPAddrFlag = "http-addr" HTTPAdvAddrFlag = "http-adv-addr" RaftAddrFlag = "raft-addr" RaftAdvAddrFlag = "raft-adv-addr" HTTPx509CertFlag = "http-cert" HTTPx509KeyFlag = "http-key" NodeX509CertFlag = "node-cert" NodeX509KeyFlag = "node-key" ) // Config represents the configuration as set by command-line flags. // All variables will be set, unless explicit noted. type Config struct { // DataPath is path to node data. Always set. DataPath string // HTTPAddr is the bind network address for the HTTP Server. // It never includes a trailing HTTP or HTTPS. HTTPAddr string // HTTPAdv is the advertised HTTP server network. HTTPAdv string // HTTPAllowOrigin is the value to set for Access-Control-Allow-Origin HTTP header. HTTPAllowOrigin string // AuthFile is the path to the authentication file. May not be set. AuthFile string `filepath:"true"` // AutoBackupFile is the path to the auto-backup file. May not be set. AutoBackupFile string `filepath:"true"` // AutoRestoreFile is the path to the auto-restore file. May not be set. AutoRestoreFile string `filepath:"true"` // HTTPx509CACert is the path to the CA certficate file for when this node verifies // other certificates for any HTTP communications. May not be set. HTTPx509CACert string `filepath:"true"` // HTTPx509Cert is the path to the X509 cert for the HTTP server. May not be set. HTTPx509Cert string `filepath:"true"` // HTTPx509Key is the path to the private key for the HTTP server. May not be set. HTTPx509Key string `filepath:"true"` // NoHTTPVerify disables checking other nodes' server HTTP X509 certs for validity. NoHTTPVerify bool // HTTPVerifyClient indicates whether the HTTP server should verify client certificates. HTTPVerifyClient bool // NodeX509CACert is the path to the CA certficate file for when this node verifies // other certificates for any inter-node communications. May not be set. NodeX509CACert string `filepath:"true"` // NodeX509Cert is the path to the X509 cert for the Raft server. May not be set. NodeX509Cert string `filepath:"true"` // NodeX509Key is the path to the X509 key for the Raft server. May not be set. NodeX509Key string `filepath:"true"` // NoNodeVerify disables checking other nodes' Node X509 certs for validity. NoNodeVerify bool // NodeVerifyClient indicates whether a node should verify client certificates from // other nodes. NodeVerifyClient bool // NodeID is the Raft ID for the node. NodeID string // RaftAddr is the bind network address for the Raft server. RaftAddr string // RaftAdv is the advertised Raft server address. RaftAdv string // JoinAddrs is the list of Raft addresses to use for a join attempt. JoinAddrs string // JoinAttempts is the number of times a node should attempt to join using a // given address. JoinAttempts int // JoinInterval is the time between retrying failed join operations. JoinInterval time.Duration // JoinAs sets the user join attempts should be performed as. May not be set. JoinAs string // BootstrapExpect is the minimum number of nodes required for a bootstrap. BootstrapExpect int // BootstrapExpectTimeout is the maximum time a bootstrap operation can take. BootstrapExpectTimeout time.Duration // DisoMode sets the discovery mode. May not be set. DiscoMode string // DiscoKey sets the discovery prefix key. DiscoKey string // DiscoConfig sets the path to any discovery configuration file. May not be set. DiscoConfig string // OnDiskPath sets the path to the SQLite file. May not be set. OnDiskPath string // FKConstraints enables SQLite foreign key constraints. FKConstraints bool // RaftLogLevel sets the minimum logging level for the Raft subsystem. RaftLogLevel string // RaftNonVoter controls whether this node is a voting, read-only node. RaftNonVoter bool // RaftSnapThreshold is the number of outstanding log entries that trigger snapshot. RaftSnapThreshold uint64 // RaftSnapInterval sets the threshold check interval. RaftSnapInterval time.Duration // RaftLeaderLeaseTimeout sets the leader lease timeout. RaftLeaderLeaseTimeout time.Duration // RaftHeartbeatTimeout specifies the time in follower state without contact // from a Leader before the node attempts an election. RaftHeartbeatTimeout time.Duration // RaftElectionTimeout specifies the time in candidate state without contact // from a Leader before the node attempts an election. RaftElectionTimeout time.Duration // RaftApplyTimeout sets the Log-apply timeout. RaftApplyTimeout time.Duration // RaftShutdownOnRemove sets whether Raft should be shutdown if the node is removed RaftShutdownOnRemove bool // RaftClusterRemoveOnShutdown sets whether the node should remove itself from the cluster on shutdown RaftClusterRemoveOnShutdown bool // RaftStepdownOnShutdown sets whether Leadership should be relinquished on shutdown RaftStepdownOnShutdown bool // RaftReapNodeTimeout sets the duration after which a non-reachable voting node is // reaped i.e. removed from the cluster. RaftReapNodeTimeout time.Duration // RaftReapReadOnlyNodeTimeout sets the duration after which a non-reachable non-voting node is // reaped i.e. removed from the cluster. RaftReapReadOnlyNodeTimeout time.Duration // ClusterConnectTimeout sets the timeout when initially connecting to another node in // the cluster, for non-Raft communications. ClusterConnectTimeout time.Duration // WriteQueueCap is the default capacity of Execute queues WriteQueueCap int // WriteQueueBatchSz is the default batch size for Execute queues WriteQueueBatchSz int // WriteQueueTimeout is the default time after which any data will be sent on // Execute queues, if a batch size has not been reached. WriteQueueTimeout time.Duration // WriteQueueTx controls whether writes from the queue are done within a transaction. WriteQueueTx bool // CPUProfile enables CPU profiling. CPUProfile string // MemProfile enables memory profiling. MemProfile string } // Validate checks the configuration for internal consistency, and activates // important rqlite policies. It must be called at least once on a Config // object before the Config object is used. It is OK to call more than // once. func (c *Config) Validate() error { dataPath, err := filepath.Abs(c.DataPath) if err != nil { return fmt.Errorf("failed to determine absolute data path: %s", err.Error()) } c.DataPath = dataPath err = c.CheckFilePaths() if err != nil { return err } if !bothUnsetSet(c.HTTPx509Cert, c.HTTPx509Key) { return fmt.Errorf("either both -%s and -%s must be set, or neither", HTTPx509CertFlag, HTTPx509KeyFlag) } if !bothUnsetSet(c.NodeX509Cert, c.NodeX509Key) { return fmt.Errorf("either both -%s and -%s must be set, or neither", NodeX509CertFlag, NodeX509KeyFlag) } if c.RaftAddr == c.HTTPAddr { return errors.New("HTTP and Raft addresses must differ") } // Enforce policies regarding addresses if c.RaftAdv == "" { c.RaftAdv = c.RaftAddr } if c.HTTPAdv == "" { c.HTTPAdv = c.HTTPAddr } // Node ID policy if c.NodeID == "" { c.NodeID = c.RaftAdv } // Perfom some address validity checks. if strings.HasPrefix(strings.ToLower(c.HTTPAddr), "http") || strings.HasPrefix(strings.ToLower(c.HTTPAdv), "http") { return errors.New("HTTP options should not include protocol (http:// or https://)") } if _, _, err := net.SplitHostPort(c.HTTPAddr); err != nil { return errors.New("HTTP bind address not valid") } hadv, _, err := net.SplitHostPort(c.HTTPAdv) if err != nil { return errors.New("HTTP advertised HTTP address not valid") } if addr := net.ParseIP(hadv); addr != nil && addr.IsUnspecified() { return fmt.Errorf("advertised HTTP address is not routable (%s), specify it via -%s or -%s", hadv, HTTPAddrFlag, HTTPAdvAddrFlag) } if _, rp, err := net.SplitHostPort(c.RaftAddr); err != nil { return errors.New("raft bind address not valid") } else if _, err := strconv.Atoi(rp); err != nil { return errors.New("raft bind port not valid") } radv, rp, err := net.SplitHostPort(c.RaftAdv) if err != nil { return errors.New("raft advertised address not valid") } if addr := net.ParseIP(radv); addr != nil && addr.IsUnspecified() { return fmt.Errorf("advertised Raft address is not routable (%s), specify it via -%s or -%s", radv, RaftAddrFlag, RaftAdvAddrFlag) } if _, err := strconv.Atoi(rp); err != nil { return errors.New("raft advertised port is not valid") } if c.RaftAdv == c.HTTPAdv { return errors.New("advertised HTTP and Raft addresses must differ") } // Enforce bootstrapping policies if c.BootstrapExpect > 0 && c.RaftNonVoter { return errors.New("bootstrapping only applicable to voting nodes") } // Join parameters OK? if c.JoinAddrs != "" { addrs := strings.Split(c.JoinAddrs, ",") for i := range addrs { if _, _, err := net.SplitHostPort(addrs[i]); err != nil { return fmt.Errorf("%s is an invalid join adddress", addrs[i]) } if c.BootstrapExpect == 0 { if addrs[i] == c.RaftAdv || addrs[i] == c.RaftAddr { return errors.New("node cannot join with itself unless bootstrapping") } if c.AutoRestoreFile != "" { return errors.New("auto-restoring cannot be used when joining a cluster") } } } } // Valid disco mode? switch c.DiscoMode { case "": case DiscoModeEtcdKV, DiscoModeConsulKV: if c.BootstrapExpect > 0 { return fmt.Errorf("bootstrapping not applicable when using %s", c.DiscoMode) } case DiscoModeDNS, DiscoModeDNSSRV: if c.BootstrapExpect == 0 { return fmt.Errorf("bootstrap-expect value required when using %s", c.DiscoMode) } default: return fmt.Errorf("disco mode must be one of %s, %s, %s, or %s", DiscoModeConsulKV, DiscoModeEtcdKV, DiscoModeDNS, DiscoModeDNSSRV) } return nil } // JoinAddresses returns the join addresses set at the command line. Returns nil // if no join addresses were set. func (c *Config) JoinAddresses() []string { if c.JoinAddrs == "" { return nil } return strings.Split(c.JoinAddrs, ",") } // HTTPURL returns the fully-formed, advertised HTTP API address for this config, including // protocol, host and port. func (c *Config) HTTPURL() string { apiProto := "http" if c.HTTPx509Cert != "" { apiProto = "https" } return fmt.Sprintf("%s://%s", apiProto, c.HTTPAdv) } // RaftPort returns the port on which the Raft system is listening. Validate must // have been called before calling this method. func (c *Config) RaftPort() int { _, port, err := net.SplitHostPort(c.RaftAddr) if err != nil { panic("RaftAddr not valid") } p, err := strconv.Atoi(port) if err != nil { panic("RaftAddr port not valid") } return p } // DiscoConfigReader returns a ReadCloser providing access to the Disco config. // The caller must call close on the ReadCloser when finished with it. If no // config was supplied, it returns nil. func (c *Config) DiscoConfigReader() io.ReadCloser { var rc io.ReadCloser if c.DiscoConfig == "" { return nil } // Open config file. If opening fails, assume string is the literal config. cfgFile, err := os.Open(c.DiscoConfig) if err != nil { rc = io.NopCloser(bytes.NewReader([]byte(c.DiscoConfig))) } else { rc = cfgFile } return rc } // CheckFilePaths checks that all file paths in the config exist. // Empy filepaths are ignored. func (c *Config) CheckFilePaths() error { v := reflect.ValueOf(c).Elem() // Iterate through the fields of the struct for i := 0; i < v.NumField(); i++ { field := v.Type().Field(i) fieldValue := v.Field(i) if fieldValue.Kind() != reflect.String { continue } if tagValue, ok := field.Tag.Lookup("filepath"); ok && tagValue == "true" { filePath := fieldValue.String() if filePath == "" { continue } _, err := os.Stat(filePath) if os.IsNotExist(err) { return fmt.Errorf("%s does not exist", filePath) } } } return nil } // BuildInfo is build information for display at command line. type BuildInfo struct { Version string Commit string Branch string SQLiteVersion string } // ParseFlags parses the command line, and returns the configuration. func ParseFlags(name, desc string, build *BuildInfo) (*Config, error) { if flag.Parsed() { return nil, fmt.Errorf("command-line flags already parsed") } config := &Config{} showVersion := false flag.StringVar(&config.NodeID, "node-id", "", "Unique ID for node. If not set, set to advertised Raft address") flag.StringVar(&config.HTTPAddr, HTTPAddrFlag, "localhost:4001", "HTTP server bind address. To enable HTTPS, set X.509 certificate and key") flag.StringVar(&config.HTTPAdv, HTTPAdvAddrFlag, "", "Advertised HTTP address. If not set, same as HTTP server bind address") flag.StringVar(&config.HTTPAllowOrigin, "http-allow-origin", "", "Value to set for Access-Control-Allow-Origin HTTP header") flag.StringVar(&config.HTTPx509CACert, "http-ca-cert", "", "Path to X.509 CA certificate for HTTPS") flag.StringVar(&config.HTTPx509Cert, HTTPx509CertFlag, "", "Path to HTTPS X.509 certificate") flag.StringVar(&config.HTTPx509Key, HTTPx509KeyFlag, "", "Path to HTTPS X.509 private key") flag.BoolVar(&config.NoHTTPVerify, "http-no-verify", false, "Skip verification of remote node's HTTPS certificate when joining a cluster") flag.BoolVar(&config.HTTPVerifyClient, "http-verify-client", false, "Enable mutual TLS for HTTPS") flag.StringVar(&config.NodeX509CACert, "node-ca-cert", "", "Path to X.509 CA certificate for node-to-node encryption") flag.StringVar(&config.NodeX509Cert, NodeX509CertFlag, "", "Path to X.509 certificate for node-to-node mutual authentication and encryption") flag.StringVar(&config.NodeX509Key, NodeX509KeyFlag, "", "Path to X.509 private key for node-to-node mutual authentication and encryption") flag.BoolVar(&config.NoNodeVerify, "node-no-verify", false, "Skip verification of any node-node certificate") flag.BoolVar(&config.NodeVerifyClient, "node-verify-client", false, "Enable mutual TLS for node-to-node communication") flag.StringVar(&config.AuthFile, "auth", "", "Path to authentication and authorization file. If not set, not enabled") flag.StringVar(&config.AutoBackupFile, "auto-backup", "", "Path to automatic backup configuration file. If not set, not enabled") flag.StringVar(&config.AutoRestoreFile, "auto-restore", "", "Path to automatic restore configuration file. If not set, not enabled") flag.StringVar(&config.RaftAddr, RaftAddrFlag, "localhost:4002", "Raft communication bind address") flag.StringVar(&config.RaftAdv, RaftAdvAddrFlag, "", "Advertised Raft communication address. If not set, same as Raft bind address") flag.StringVar(&config.JoinAddrs, "join", "", "Comma-delimited list of nodes, through which a cluster can be joined (proto://host:port)") flag.IntVar(&config.JoinAttempts, "join-attempts", 5, "Number of join attempts to make") flag.DurationVar(&config.JoinInterval, "join-interval", 3*time.Second, "Period between join attempts") flag.StringVar(&config.JoinAs, "join-as", "", "Username in authentication file to join as. If not set, joins anonymously") flag.IntVar(&config.BootstrapExpect, "bootstrap-expect", 0, "Minimum number of nodes required for a bootstrap") flag.DurationVar(&config.BootstrapExpectTimeout, "bootstrap-expect-timeout", 120*time.Second, "Maximum time for bootstrap process") flag.StringVar(&config.DiscoMode, "disco-mode", "", "Choose clustering discovery mode. If not set, no node discovery is performed") flag.StringVar(&config.DiscoKey, "disco-key", "rqlite", "Key prefix for cluster discovery service") flag.StringVar(&config.DiscoConfig, "disco-config", "", "Set discovery config, or path to cluster discovery config file") flag.StringVar(&config.OnDiskPath, "on-disk-path", "", "Path for SQLite on-disk database file. If not set, use a file in data directory") flag.BoolVar(&config.FKConstraints, "fk", false, "Enable SQLite foreign key constraints") flag.BoolVar(&showVersion, "version", false, "Show version information and exit") flag.BoolVar(&config.RaftNonVoter, "raft-non-voter", false, "Configure as non-voting node") flag.DurationVar(&config.RaftHeartbeatTimeout, "raft-timeout", time.Second, "Raft heartbeat timeout") flag.DurationVar(&config.RaftElectionTimeout, "raft-election-timeout", time.Second, "Raft election timeout") flag.DurationVar(&config.RaftApplyTimeout, "raft-apply-timeout", 10*time.Second, "Raft apply timeout") flag.Uint64Var(&config.RaftSnapThreshold, "raft-snap", 8192, "Number of outstanding log entries that trigger snapshot and Raft log compaction") flag.DurationVar(&config.RaftSnapInterval, "raft-snap-int", 30*time.Second, "Snapshot threshold check interval") flag.DurationVar(&config.RaftLeaderLeaseTimeout, "raft-leader-lease-timeout", 0, "Raft leader lease timeout. Use 0s for Raft default") flag.BoolVar(&config.RaftStepdownOnShutdown, "raft-shutdown-stepdown", true, "If leader, stepdown before shutting down. Enabled by default") flag.BoolVar(&config.RaftShutdownOnRemove, "raft-remove-shutdown", false, "Shutdown Raft if node removed from cluster") flag.BoolVar(&config.RaftClusterRemoveOnShutdown, "raft-cluster-remove-shutdown", false, "Node removes itself from cluster on graceful shutdown") flag.StringVar(&config.RaftLogLevel, "raft-log-level", "INFO", "Minimum log level for Raft module") flag.DurationVar(&config.RaftReapNodeTimeout, "raft-reap-node-timeout", 0*time.Hour, "Time after which a non-reachable voting node will be reaped. If not set, no reaping takes place") flag.DurationVar(&config.RaftReapReadOnlyNodeTimeout, "raft-reap-read-only-node-timeout", 0*time.Hour, "Time after which a non-reachable non-voting node will be reaped. If not set, no reaping takes place") flag.DurationVar(&config.ClusterConnectTimeout, "cluster-connect-timeout", 30*time.Second, "Timeout for initial connection to other nodes") flag.IntVar(&config.WriteQueueCap, "write-queue-capacity", 1024, "QueuedWrites queue capacity") flag.IntVar(&config.WriteQueueBatchSz, "write-queue-batch-size", 128, "QueuedWrites queue batch size") flag.DurationVar(&config.WriteQueueTimeout, "write-queue-timeout", 50*time.Millisecond, "QueuedWrites queue timeout") flag.BoolVar(&config.WriteQueueTx, "write-queue-tx", false, "Use a transaction when processing a queued write") flag.StringVar(&config.CPUProfile, "cpu-profile", "", "Path to file for CPU profiling information") flag.StringVar(&config.MemProfile, "mem-profile", "", "Path to file for memory profiling information") flag.Usage = func() { fmt.Fprintf(os.Stderr, "\n%s\n\n", desc) fmt.Fprintf(os.Stderr, "Usage: %s [flags] \n", name) flag.PrintDefaults() } flag.Parse() if showVersion { msg := fmt.Sprintf("%s %s %s %s %s sqlite%s (commit %s, branch %s, compiler %s)", name, build.Version, runtime.GOOS, runtime.GOARCH, runtime.Version(), build.SQLiteVersion, build.Commit, build.Branch, runtime.Compiler) errorExit(0, msg) } // Ensure, if set explicitly, that reap times are not too low. flag.Visit(func(f *flag.Flag) { if f.Name == "raft-reap-node-timeout" || f.Name == "raft-reap-read-only-node-timeout" { d, err := time.ParseDuration(f.Value.String()) if err != nil { errorExit(1, fmt.Sprintf("failed to parse duration: %s", err.Error())) } if d <= 0 { errorExit(1, fmt.Sprintf("-%s must be greater than 0", f.Name)) } } }) // Ensure the data path is set. if flag.NArg() < 1 { errorExit(1, "no data directory set") } config.DataPath = flag.Arg(0) // Ensure no args come after the data directory. if flag.NArg() > 1 { fmt.Fprintf(os.Stderr, "arguments after data directory (%s) are not accepted (%s)\n", config.DataPath, flag.Args()[1:]) os.Exit(1) } if err := config.Validate(); err != nil { errorExit(1, err.Error()) } return config, nil } func errorExit(code int, msg string) { if code != 0 { fmt.Fprintf(os.Stderr, "fatal: ") } fmt.Fprintf(os.Stderr, "%s\n", msg) os.Exit(code) } // bothUnsetSet returns true if both a and b are unset, or both are set. func bothUnsetSet(a, b string) bool { return (a == "" && b == "") || (a != "" && b != "") }