fluidB
/
fluidb-old
1
0
Fork 0
Code Pull Requests Activity

Use Protobuf for encoding Raft Log commands

Browse Source 
This PR changes Raft Log Entry encoding from JSON to Protobuf. Furthermore, larger Raft commands (which can result from batching SQL statements, or individually long SQL statements) are compressed before encoding.

This primary reason for this change is to reduce IO load since that is one of the largest performance bottlenecks. It will also reduce internode traffic.

Legacy JSON-encoded commands are still handled by this code, so this change is backwards-compatible with previous releases in the v5 series.
master
Philip O'Toole 4 years ago committed by GitHub
parent 7d4445d92d
commit 6575de779d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 2532 additions and 434 deletions
Show Stats Download Patch File Download Diff File

14
CONTRIBUTING.md
Unescape Escape View File

@ -40,6 +40,20 @@ $GOPATH/bin/rqlited ~/node.1
### Raspberry Pi
The process outlined above will work for Linux, OSX, and Windows. For Raspberry Pi, check out [this issue](https://github.com/rqlite/rqlite/issues/340).
### Protobuf code generation
_This step is not necessary unless you are making changes to protobuf definitions._
Ensure you have the required tools installed, and that `GOPATH` is set.
```bash
go get -u github.com/golang/protobuf/protoc-gen-go
go install github.com/golang/protobuf/protoc-gen-go
export GOBIN=$GOPATH/bin
export PATH=$PATH:$GOBIN
export SRC_DIR=$GOPATH/src/github.com/rqlite/rqlite/command
export DEST_DIR=$GOPATH/src
protoc -I=$SRC_DIR --go_out=$DEST_DIR $SRC_DIR/command.proto
```
### Speeding up the build process
It can be rather slow to rebuild rqlite, due to the repeated compilation of the SQLite source code. You can compile and install the SQLite libary once, so subsequent builds are much faster. To do so, execute the following commands:
```bash

5
cmd/rqlited/main.go
Unescape Escape View File

@ -78,6 +78,8 @@ var raftElectionTimeout string
var raftApplyTimeout string
var raftOpenTimeout string
var raftShutdownOnRemove bool
var compressionSize int
var compressionBatch int
var showVersion bool
var cpuProfile string
var memProfile string
@ -122,6 +124,8 @@ func init() {
flag.StringVar(&raftLeaderLeaseTimeout, "raft-leader-lease-timeout", "0s", "Raft leader lease timeout. Use 0s for Raft default")
flag.BoolVar(&raftShutdownOnRemove, "raft-remove-shutdown", false, "Shutdown Raft if node removed")
flag.StringVar(&raftLogLevel, "raft-log-level", "INFO", "Minimum log level for Raft module")
flag.IntVar(&compressionSize, "compression-size", 150, "Request query size for compression attempt")
flag.IntVar(&compressionBatch, "compression-batch", 5, "Request batch threshold for compression attempt")
flag.StringVar(&cpuProfile, "cpu-profile", "", "Path to file for CPU profiling information")
flag.StringVar(&memProfile, "mem-profile", "", "Path to file for memory profiling information")
flag.Usage = func() {
@ -187,6 +191,7 @@ func main() {
})
// Set optional parameters on store.
str.SetRequestCompression(compressionBatch, compressionSize)
str.RaftLogLevel = raftLogLevel
str.ShutdownOnRemove = raftShutdownOnRemove
str.SnapshotThreshold = raftSnapThreshold

905
command/command.pb.go
Unescape Escape View File

@ -0,0 +1,905 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.25.0
// protoc v3.13.0
// source: command.proto
package command
import (
proto "github.com/golang/protobuf/proto"
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
reflect "reflect"
sync "sync"
)
const (
// Verify that this generated code is sufficiently up-to-date.
_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
// Verify that runtime/protoimpl is sufficiently up-to-date.
_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
)
// This is a compile-time assertion that a sufficiently up-to-date version
// of the legacy proto package is being used.
const _ = proto.ProtoPackageIsVersion4
type QueryRequest_Level int32
const (
QueryRequest_QUERY_REQUEST_LEVEL_NONE QueryRequest_Level = 0
QueryRequest_QUERY_REQUEST_LEVEL_WEAK QueryRequest_Level = 1
QueryRequest_QUERY_REQUEST_LEVEL_STRONG QueryRequest_Level = 2
)
// Enum value maps for QueryRequest_Level.
var (
QueryRequest_Level_name = map[int32]string{
0: "QUERY_REQUEST_LEVEL_NONE",
1: "QUERY_REQUEST_LEVEL_WEAK",
2: "QUERY_REQUEST_LEVEL_STRONG",
}
QueryRequest_Level_value = map[string]int32{
"QUERY_REQUEST_LEVEL_NONE": 0,
"QUERY_REQUEST_LEVEL_WEAK": 1,
"QUERY_REQUEST_LEVEL_STRONG": 2,
}
)
func (x QueryRequest_Level) Enum() *QueryRequest_Level {
p := new(QueryRequest_Level)
*p = x
return p
}
func (x QueryRequest_Level) String() string {
return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
}
func (QueryRequest_Level) Descriptor() protoreflect.EnumDescriptor {
return file_command_proto_enumTypes[0].Descriptor()
}
func (QueryRequest_Level) Type() protoreflect.EnumType {
return &file_command_proto_enumTypes[0]
}
func (x QueryRequest_Level) Number() protoreflect.EnumNumber {
return protoreflect.EnumNumber(x)
}
// Deprecated: Use QueryRequest_Level.Descriptor instead.
func (QueryRequest_Level) EnumDescriptor() ([]byte, []int) {
return file_command_proto_rawDescGZIP(), []int{3, 0}
}
type Command_Type int32
const (
Command_COMMAND_TYPE_UNKNOWN Command_Type = 0
Command_COMMAND_TYPE_QUERY Command_Type = 1
Command_COMMAND_TYPE_EXECUTE Command_Type = 2
Command_COMMAND_TYPE_METADATA_SET Command_Type = 3
Command_COMMAND_TYPE_METADATA_DELETE Command_Type = 4
)
// Enum value maps for Command_Type.
var (
Command_Type_name = map[int32]string{
0: "COMMAND_TYPE_UNKNOWN",
1: "COMMAND_TYPE_QUERY",
2: "COMMAND_TYPE_EXECUTE",
3: "COMMAND_TYPE_METADATA_SET",
4: "COMMAND_TYPE_METADATA_DELETE",
}
Command_Type_value = map[string]int32{
"COMMAND_TYPE_UNKNOWN": 0,
"COMMAND_TYPE_QUERY": 1,
"COMMAND_TYPE_EXECUTE": 2,
"COMMAND_TYPE_METADATA_SET": 3,
"COMMAND_TYPE_METADATA_DELETE": 4,
}
)
func (x Command_Type) Enum() *Command_Type {
p := new(Command_Type)
*p = x
return p
}
func (x Command_Type) String() string {
return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
}
func (Command_Type) Descriptor() protoreflect.EnumDescriptor {
return file_command_proto_enumTypes[1].Descriptor()
}
func (Command_Type) Type() protoreflect.EnumType {
return &file_command_proto_enumTypes[1]
}
func (x Command_Type) Number() protoreflect.EnumNumber {
return protoreflect.EnumNumber(x)
}
// Deprecated: Use Command_Type.Descriptor instead.
func (Command_Type) EnumDescriptor() ([]byte, []int) {
return file_command_proto_rawDescGZIP(), []int{7, 0}
}
type Parameter struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
// Types that are assignable to Value:
// *Parameter_I
// *Parameter_D
// *Parameter_B
// *Parameter_Y
// *Parameter_S
Value isParameter_Value `protobuf_oneof:"value"`
}
func (x *Parameter) Reset() {
*x = Parameter{}
if protoimpl.UnsafeEnabled {
mi := &file_command_proto_msgTypes[0]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *Parameter) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*Parameter) ProtoMessage() {}
func (x *Parameter) ProtoReflect() protoreflect.Message {
mi := &file_command_proto_msgTypes[0]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use Parameter.ProtoReflect.Descriptor instead.
func (*Parameter) Descriptor() ([]byte, []int) {
return file_command_proto_rawDescGZIP(), []int{0}
}
func (m *Parameter) GetValue() isParameter_Value {
if m != nil {
return m.Value
}
return nil
}
func (x *Parameter) GetI() int64 {
if x, ok := x.GetValue().(*Parameter_I); ok {
return x.I
}
return 0
}
func (x *Parameter) GetD() float64 {
if x, ok := x.GetValue().(*Parameter_D); ok {
return x.D
}
return 0
}
func (x *Parameter) GetB() bool {
if x, ok := x.GetValue().(*Parameter_B); ok {
return x.B
}
return false
}
func (x *Parameter) GetY() []byte {
if x, ok := x.GetValue().(*Parameter_Y); ok {
return x.Y
}
return nil
}
func (x *Parameter) GetS() string {
if x, ok := x.GetValue().(*Parameter_S); ok {
return x.S
}
return ""
}
type isParameter_Value interface {
isParameter_Value()
}
type Parameter_I struct {
I int64 `protobuf:"zigzag64,1,opt,name=i,proto3,oneof"`
}
type Parameter_D struct {
D float64 `protobuf:"fixed64,2,opt,name=d,proto3,oneof"`
}
type Parameter_B struct {
B bool `protobuf:"varint,3,opt,name=b,proto3,oneof"`
}
type Parameter_Y struct {
Y []byte `protobuf:"bytes,4,opt,name=y,proto3,oneof"`
}
type Parameter_S struct {
S string `protobuf:"bytes,5,opt,name=s,proto3,oneof"`
}
func (*Parameter_I) isParameter_Value() {}
func (*Parameter_D) isParameter_Value() {}
func (*Parameter_B) isParameter_Value() {}
func (*Parameter_Y) isParameter_Value() {}
func (*Parameter_S) isParameter_Value() {}
type Statement struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Sql string `protobuf:"bytes,1,opt,name=sql,proto3" json:"sql,omitempty"`
Parameters []*Parameter `protobuf:"bytes,2,rep,name=parameters,proto3" json:"parameters,omitempty"`
}
func (x *Statement) Reset() {
*x = Statement{}
if protoimpl.UnsafeEnabled {
mi := &file_command_proto_msgTypes[1]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *Statement) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*Statement) ProtoMessage() {}
func (x *Statement) ProtoReflect() protoreflect.Message {
mi := &file_command_proto_msgTypes[1]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use Statement.ProtoReflect.Descriptor instead.
func (*Statement) Descriptor() ([]byte, []int) {
return file_command_proto_rawDescGZIP(), []int{1}
}
func (x *Statement) GetSql() string {
if x != nil {
return x.Sql
}
return ""
}
func (x *Statement) GetParameters() []*Parameter {
if x != nil {
return x.Parameters
}
return nil
}
type Request struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Transaction bool `protobuf:"varint,1,opt,name=transaction,proto3" json:"transaction,omitempty"`
Statements []*Statement `protobuf:"bytes,2,rep,name=statements,proto3" json:"statements,omitempty"`
}
func (x *Request) Reset() {
*x = Request{}
if protoimpl.UnsafeEnabled {
mi := &file_command_proto_msgTypes[2]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *Request) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*Request) ProtoMessage() {}
func (x *Request) ProtoReflect() protoreflect.Message {
mi := &file_command_proto_msgTypes[2]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use Request.ProtoReflect.Descriptor instead.
func (*Request) Descriptor() ([]byte, []int) {
return file_command_proto_rawDescGZIP(), []int{2}
}
func (x *Request) GetTransaction() bool {
if x != nil {
return x.Transaction
}
return false
}
func (x *Request) GetStatements() []*Statement {
if x != nil {
return x.Statements
}
return nil
}
type QueryRequest struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Request *Request `protobuf:"bytes,1,opt,name=request,proto3" json:"request,omitempty"`
Timings bool `protobuf:"varint,2,opt,name=timings,proto3" json:"timings,omitempty"`
Level QueryRequest_Level `protobuf:"varint,3,opt,name=level,proto3,enum=command.QueryRequest_Level" json:"level,omitempty"`
Freshness int64 `protobuf:"varint,4,opt,name=freshness,proto3" json:"freshness,omitempty"`
}
func (x *QueryRequest) Reset() {
*x = QueryRequest{}
if protoimpl.UnsafeEnabled {
mi := &file_command_proto_msgTypes[3]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *QueryRequest) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*QueryRequest) ProtoMessage() {}
func (x *QueryRequest) ProtoReflect() protoreflect.Message {
mi := &file_command_proto_msgTypes[3]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use QueryRequest.ProtoReflect.Descriptor instead.
func (*QueryRequest) Descriptor() ([]byte, []int) {
return file_command_proto_rawDescGZIP(), []int{3}
}
func (x *QueryRequest) GetRequest() *Request {
if x != nil {
return x.Request
}
return nil
}
func (x *QueryRequest) GetTimings() bool {
if x != nil {
return x.Timings
}
return false
}
func (x *QueryRequest) GetLevel() QueryRequest_Level {
if x != nil {
return x.Level
}
return QueryRequest_QUERY_REQUEST_LEVEL_NONE
}
func (x *QueryRequest) GetFreshness() int64 {
if x != nil {
return x.Freshness
}
return 0
}
type ExecuteRequest struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Request *Request `protobuf:"bytes,1,opt,name=request,proto3" json:"request,omitempty"`
Timings bool `protobuf:"varint,2,opt,name=timings,proto3" json:"timings,omitempty"`
}
func (x *ExecuteRequest) Reset() {
*x = ExecuteRequest{}
if protoimpl.UnsafeEnabled {
mi := &file_command_proto_msgTypes[4]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *ExecuteRequest) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*ExecuteRequest) ProtoMessage() {}
func (x *ExecuteRequest) ProtoReflect() protoreflect.Message {
mi := &file_command_proto_msgTypes[4]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use ExecuteRequest.ProtoReflect.Descriptor instead.
func (*ExecuteRequest) Descriptor() ([]byte, []int) {
return file_command_proto_rawDescGZIP(), []int{4}
}
func (x *ExecuteRequest) GetRequest() *Request {
if x != nil {
return x.Request
}
return nil
}
func (x *ExecuteRequest) GetTimings() bool {
if x != nil {
return x.Timings
}
return false
}
type MetadataSet struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
RaftId string `protobuf:"bytes,1,opt,name=raft_id,json=raftId,proto3" json:"raft_id,omitempty"`
Data map[string]string `protobuf:"bytes,2,rep,name=data,proto3" json:"data,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
}
func (x *MetadataSet) Reset() {
*x = MetadataSet{}
if protoimpl.UnsafeEnabled {
mi := &file_command_proto_msgTypes[5]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *MetadataSet) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*MetadataSet) ProtoMessage() {}
func (x *MetadataSet) ProtoReflect() protoreflect.Message {
mi := &file_command_proto_msgTypes[5]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use MetadataSet.ProtoReflect.Descriptor instead.
func (*MetadataSet) Descriptor() ([]byte, []int) {
return file_command_proto_rawDescGZIP(), []int{5}
}
func (x *MetadataSet) GetRaftId() string {
if x != nil {
return x.RaftId
}
return ""
}
func (x *MetadataSet) GetData() map[string]string {
if x != nil {
return x.Data
}
return nil
}
type MetadataDelete struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
RaftId string `protobuf:"bytes,1,opt,name=raft_id,json=raftId,proto3" json:"raft_id,omitempty"`
}
func (x *MetadataDelete) Reset() {
*x = MetadataDelete{}
if protoimpl.UnsafeEnabled {
mi := &file_command_proto_msgTypes[6]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *MetadataDelete) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*MetadataDelete) ProtoMessage() {}
func (x *MetadataDelete) ProtoReflect() protoreflect.Message {
mi := &file_command_proto_msgTypes[6]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use MetadataDelete.ProtoReflect.Descriptor instead.
func (*MetadataDelete) Descriptor() ([]byte, []int) {
return file_command_proto_rawDescGZIP(), []int{6}
}
func (x *MetadataDelete) GetRaftId() string {
if x != nil {
return x.RaftId
}
return ""
}
type Command struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Type Command_Type `protobuf:"varint,1,opt,name=type,proto3,enum=command.Command_Type" json:"type,omitempty"`
SubCommand []byte `protobuf:"bytes,2,opt,name=sub_command,json=subCommand,proto3" json:"sub_command,omitempty"`
Compressed bool `protobuf:"varint,3,opt,name=compressed,proto3" json:"compressed,omitempty"`
}
func (x *Command) Reset() {
*x = Command{}
if protoimpl.UnsafeEnabled {
mi := &file_command_proto_msgTypes[7]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *Command) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*Command) ProtoMessage() {}
func (x *Command) ProtoReflect() protoreflect.Message {
mi := &file_command_proto_msgTypes[7]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use Command.ProtoReflect.Descriptor instead.
func (*Command) Descriptor() ([]byte, []int) {
return file_command_proto_rawDescGZIP(), []int{7}
}
func (x *Command) GetType() Command_Type {
if x != nil {
return x.Type
}
return Command_COMMAND_TYPE_UNKNOWN
}
func (x *Command) GetSubCommand() []byte {
if x != nil {
return x.SubCommand
}
return nil
}
func (x *Command) GetCompressed() bool {
if x != nil {
return x.Compressed
}
return false
}
var File_command_proto protoreflect.FileDescriptor
var file_command_proto_rawDesc = []byte{
0x0a, 0x0d, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12,
0x07, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x22, 0x64, 0x0a, 0x09, 0x50, 0x61, 0x72, 0x61,
0x6d, 0x65, 0x74, 0x65, 0x72, 0x12, 0x0e, 0x0a, 0x01, 0x69, 0x18, 0x01, 0x20, 0x01, 0x28, 0x12,
0x48, 0x00, 0x52, 0x01, 0x69, 0x12, 0x0e, 0x0a, 0x01, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x01,
0x48, 0x00, 0x52, 0x01, 0x64, 0x12, 0x0e, 0x0a, 0x01, 0x62, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08,
0x48, 0x00, 0x52, 0x01, 0x62, 0x12, 0x0e, 0x0a, 0x01, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c,
0x48, 0x00, 0x52, 0x01, 0x79, 0x12, 0x0e, 0x0a, 0x01, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09,
0x48, 0x00, 0x52, 0x01, 0x73, 0x42, 0x07, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x51,
0x0a, 0x09, 0x53, 0x74, 0x61, 0x74, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x73,
0x71, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x73, 0x71, 0x6c, 0x12, 0x32, 0x0a,
0x0a, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28,
0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x2e, 0x50, 0x61, 0x72, 0x61,
0x6d, 0x65, 0x74, 0x65, 0x72, 0x52, 0x0a, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72,
0x73, 0x22, 0x5f, 0x0a, 0x07, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x20, 0x0a, 0x0b,
0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28,
0x08, 0x52, 0x0b, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x32,
0x0a, 0x0a, 0x73, 0x74, 0x61, 0x74, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03,
0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x2e, 0x53, 0x74, 0x61,
0x74, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x0a, 0x73, 0x74, 0x61, 0x74, 0x65, 0x6d, 0x65, 0x6e,
0x74, 0x73, 0x22, 0x8a, 0x02, 0x0a, 0x0c, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75,
0x65, 0x73, 0x74, 0x12, 0x2a, 0x0a, 0x07, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x18, 0x01,
0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x2e, 0x52,
0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x52, 0x07, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12,
0x18, 0x0a, 0x07, 0x74, 0x69, 0x6d, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08,
0x52, 0x07, 0x74, 0x69, 0x6d, 0x69, 0x6e, 0x67, 0x73, 0x12, 0x31, 0x0a, 0x05, 0x6c, 0x65, 0x76,
0x65, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1b, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x61,
0x6e, 0x64, 0x2e, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e,
0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x1c, 0x0a, 0x09,
0x66, 0x72, 0x65, 0x73, 0x68, 0x6e, 0x65, 0x73, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52,
0x09, 0x66, 0x72, 0x65, 0x73, 0x68, 0x6e, 0x65, 0x73, 0x73, 0x22, 0x63, 0x0a, 0x05, 0x4c, 0x65,
0x76, 0x65, 0x6c, 0x12, 0x1c, 0x0a, 0x18, 0x51, 0x55, 0x45, 0x52, 0x59, 0x5f, 0x52, 0x45, 0x51,
0x55, 0x45, 0x53, 0x54, 0x5f, 0x4c, 0x45, 0x56, 0x45, 0x4c, 0x5f, 0x4e, 0x4f, 0x4e, 0x45, 0x10,
0x00, 0x12, 0x1c, 0x0a, 0x18, 0x51, 0x55, 0x45, 0x52, 0x59, 0x5f, 0x52, 0x45, 0x51, 0x55, 0x45,
0x53, 0x54, 0x5f, 0x4c, 0x45, 0x56, 0x45, 0x4c, 0x5f, 0x57, 0x45, 0x41, 0x4b, 0x10, 0x01, 0x12,
0x1e, 0x0a, 0x1a, 0x51, 0x55, 0x45, 0x52, 0x59, 0x5f, 0x52, 0x45, 0x51, 0x55, 0x45, 0x53, 0x54,
0x5f, 0x4c, 0x45, 0x56, 0x45, 0x4c, 0x5f, 0x53, 0x54, 0x52, 0x4f, 0x4e, 0x47, 0x10, 0x02, 0x22,
0x56, 0x0a, 0x0e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
0x74, 0x12, 0x2a, 0x0a, 0x07, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01,
0x28, 0x0b, 0x32, 0x10, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x2e, 0x52, 0x65, 0x71,
0x75, 0x65, 0x73, 0x74, 0x52, 0x07, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x18, 0x0a,
0x07, 0x74, 0x69, 0x6d, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07,
0x74, 0x69, 0x6d, 0x69, 0x6e, 0x67, 0x73, 0x22, 0x93, 0x01, 0x0a, 0x0b, 0x4d, 0x65, 0x74, 0x61,
0x64, 0x61, 0x74, 0x61, 0x53, 0x65, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x72, 0x61, 0x66, 0x74, 0x5f,
0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x72, 0x61, 0x66, 0x74, 0x49, 0x64,
0x12, 0x32, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e,
0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74,
0x61, 0x53, 0x65, 0x74, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04,
0x64, 0x61, 0x74, 0x61, 0x1a, 0x37, 0x0a, 0x09, 0x44, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72,
0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03,
0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01,
0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x29, 0x0a,
0x0e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12,
0x17, 0x0a, 0x07, 0x72, 0x61, 0x66, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
0x52, 0x06, 0x72, 0x61, 0x66, 0x74, 0x49, 0x64, 0x22, 0x8b, 0x02, 0x0a, 0x07, 0x43, 0x6f, 0x6d,
0x6d, 0x61, 0x6e, 0x64, 0x12, 0x29, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01,
0x28, 0x0e, 0x32, 0x15, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x2e, 0x43, 0x6f, 0x6d,
0x6d, 0x61, 0x6e, 0x64, 0x2e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12,
0x1f, 0x0a, 0x0b, 0x73, 0x75, 0x62, 0x5f, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x18, 0x02,
0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x73, 0x75, 0x62, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64,
0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x65, 0x64, 0x18, 0x03,
0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x63, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x65, 0x64,
0x22, 0x93, 0x01, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x12, 0x18, 0x0a, 0x14, 0x43, 0x4f, 0x4d,
0x4d, 0x41, 0x4e, 0x44, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57,
0x4e, 0x10, 0x00, 0x12, 0x16, 0x0a, 0x12, 0x43, 0x4f, 0x4d, 0x4d, 0x41, 0x4e, 0x44, 0x5f, 0x54,
0x59, 0x50, 0x45, 0x5f, 0x51, 0x55, 0x45, 0x52, 0x59, 0x10, 0x01, 0x12, 0x18, 0x0a, 0x14, 0x43,
0x4f, 0x4d, 0x4d, 0x41, 0x4e, 0x44, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x45, 0x58, 0x45, 0x43,
0x55, 0x54, 0x45, 0x10, 0x02, 0x12, 0x1d, 0x0a, 0x19, 0x43, 0x4f, 0x4d, 0x4d, 0x41, 0x4e, 0x44,
0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4d, 0x45, 0x54, 0x41, 0x44, 0x41, 0x54, 0x41, 0x5f, 0x53,
0x45, 0x54, 0x10, 0x03, 0x12, 0x20, 0x0a, 0x1c, 0x43, 0x4f, 0x4d, 0x4d, 0x41, 0x4e, 0x44, 0x5f,
0x54, 0x59, 0x50, 0x45, 0x5f, 0x4d, 0x45, 0x54, 0x41, 0x44, 0x41, 0x54, 0x41, 0x5f, 0x44, 0x45,
0x4c, 0x45, 0x54, 0x45, 0x10, 0x04, 0x42, 0x22, 0x5a, 0x20, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62,
0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x72, 0x71, 0x6c, 0x69, 0x74, 0x65, 0x2f, 0x72, 0x71, 0x6c, 0x69,
0x74, 0x65, 0x2f, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74,
0x6f, 0x33,
}
var (
file_command_proto_rawDescOnce sync.Once
file_command_proto_rawDescData = file_command_proto_rawDesc
)
func file_command_proto_rawDescGZIP() []byte {
file_command_proto_rawDescOnce.Do(func() {
file_command_proto_rawDescData = protoimpl.X.CompressGZIP(file_command_proto_rawDescData)
})
return file_command_proto_rawDescData
}
var file_command_proto_enumTypes = make([]protoimpl.EnumInfo, 2)
var file_command_proto_msgTypes = make([]protoimpl.MessageInfo, 9)
var file_command_proto_goTypes = []interface{}{
(QueryRequest_Level)(0), // 0: command.QueryRequest.Level
(Command_Type)(0), // 1: command.Command.Type
(*Parameter)(nil), // 2: command.Parameter
(*Statement)(nil), // 3: command.Statement
(*Request)(nil), // 4: command.Request
(*QueryRequest)(nil), // 5: command.QueryRequest
(*ExecuteRequest)(nil), // 6: command.ExecuteRequest
(*MetadataSet)(nil), // 7: command.MetadataSet
(*MetadataDelete)(nil), // 8: command.MetadataDelete
(*Command)(nil), // 9: command.Command
nil, // 10: command.MetadataSet.DataEntry
}
var file_command_proto_depIdxs = []int32{
2, // 0: command.Statement.parameters:type_name -> command.Parameter
3, // 1: command.Request.statements:type_name -> command.Statement
4, // 2: command.QueryRequest.request:type_name -> command.Request
0, // 3: command.QueryRequest.level:type_name -> command.QueryRequest.Level
4, // 4: command.ExecuteRequest.request:type_name -> command.Request
10, // 5: command.MetadataSet.data:type_name -> command.MetadataSet.DataEntry
1, // 6: command.Command.type:type_name -> command.Command.Type
7, // [7:7] is the sub-list for method output_type
7, // [7:7] is the sub-list for method input_type
7, // [7:7] is the sub-list for extension type_name
7, // [7:7] is the sub-list for extension extendee
0, // [0:7] is the sub-list for field type_name
}
func init() { file_command_proto_init() }
func file_command_proto_init() {
if File_command_proto != nil {
return
}
if !protoimpl.UnsafeEnabled {
file_command_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*Parameter); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_command_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*Statement); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_command_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*Request); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_command_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*QueryRequest); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_command_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*ExecuteRequest); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_command_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*MetadataSet); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_command_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*MetadataDelete); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_command_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*Command); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
}
file_command_proto_msgTypes[0].OneofWrappers = []interface{}{
(*Parameter_I)(nil),
(*Parameter_D)(nil),
(*Parameter_B)(nil),
(*Parameter_Y)(nil),
(*Parameter_S)(nil),
}
type x struct{}
out := protoimpl.TypeBuilder{
File: protoimpl.DescBuilder{
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_command_proto_rawDesc,
NumEnums: 2,
NumMessages: 9,
NumExtensions: 0,
NumServices: 0,
},
GoTypes: file_command_proto_goTypes,
DependencyIndexes: file_command_proto_depIdxs,
EnumInfos: file_command_proto_enumTypes,
MessageInfos: file_command_proto_msgTypes,
}.Build()
File_command_proto = out.File
file_command_proto_rawDesc = nil
file_command_proto_goTypes = nil
file_command_proto_depIdxs = nil
}

63
command/command.proto
Unescape Escape View File

@ -0,0 +1,63 @@
syntax = "proto3";
package command;
option go_package = "github.com/rqlite/rqlite/command";
message Parameter {
oneof value {
sint64 i = 1;
double d = 2;
bool b = 3;
bytes y = 4;
string s = 5;
}
}
message Statement {
string sql = 1;
repeated Parameter parameters = 2;
}
message Request {
bool transaction = 1;
repeated Statement statements = 2;
}
message QueryRequest {
Request request = 1;
bool timings = 2;
enum Level {
QUERY_REQUEST_LEVEL_NONE = 0;
QUERY_REQUEST_LEVEL_WEAK = 1;
QUERY_REQUEST_LEVEL_STRONG = 2;
}
Level level = 3;
int64 freshness = 4;
}
message ExecuteRequest {
Request request = 1;
bool timings = 2;
}
message MetadataSet {
string raft_id = 1;
map<string, string> data = 2;
}
message MetadataDelete {
string raft_id = 1;
}
message Command {
enum Type {
COMMAND_TYPE_UNKNOWN = 0;
COMMAND_TYPE_QUERY = 1;
COMMAND_TYPE_EXECUTE = 2;
COMMAND_TYPE_METADATA_SET = 3;
COMMAND_TYPE_METADATA_DELETE = 4;
}
Type type = 1;
bytes sub_command = 2;
bool compressed = 3;
}

189
command/legacy/command.go
Unescape Escape View File

@ -0,0 +1,189 @@
package legacy
import (
"encoding/json"
"errors"
"github.com/golang/protobuf/proto"
"github.com/rqlite/rqlite/command"
)
const (
execute commandType = iota // Commands which modify the database.
query // Commands which query the database.
metadataSet // Commands which sets Store metadata
metadataDelete // Commands which deletes Store metadata
)
var (
// ErrNotLegacyCommand is returned when a command is not legacy encoded.
ErrNotLegacyCommand = errors.New("not legacy command")
// ErrUnknownType is returned when an unknown command type is encountered.
ErrUnknownCommandType = errors.New("unknown command type")
// ErrUnsupportedType is returned when a request contains an unsupported type.
ErrUnsupportedType = errors.New("unsupported type")
)
// commandType are commands that affect the state of the cluster, and must go through Raft.
type commandType int
// Value is the type for parameters passed to a parameterized SQL statement.
type Value interface{}
type Command struct {
Typ commandType `json:"typ,omitempty"`
Sub json.RawMessage `json:"sub,omitempty"`
}
// databaseSub is a command sub which involves interaction with the database.
// Queries and Parameters are separate fields, for backwards-compatibility
// reasons. Unless Parameters is nil, it should be the same length as Queries.
type databaseSub struct {
Tx bool `json:"tx,omitempty"`
SQLs []string `json:"queries,omitempty"`
Parameters [][]Value `json:"Parameters,omitempty`
Timings bool `json:"timings,omitempty"`
}
type metadataSetSub struct {
RaftID string `json:"raft_id,omitempty"`
Data map[string]string `json:"data,omitempty"`
}
func Unmarshal(b []byte, c *command.Command) error {
if b == nil || len(b) == 0 || b[0] != '{' {
return ErrNotLegacyCommand
}
var lc Command
if err := json.Unmarshal(b, &lc); err != nil {
return err
}
var m proto.Message
switch lc.Typ {
case execute, query:
var d databaseSub
if err := json.Unmarshal(lc.Sub, &d); err != nil {
return err
}
stmts, err := subCommandToStatements(&d)
if err != nil {
return err
}
if lc.Typ == execute {
c.Type = command.Command_COMMAND_TYPE_EXECUTE
m = &command.ExecuteRequest{
Request: &command.Request{
Transaction: d.Tx,
Statements: stmts,
},
Timings: d.Timings,
}
} else {
c.Type = command.Command_COMMAND_TYPE_QUERY
m = &command.QueryRequest{
Request: &command.Request{
Transaction: d.Tx,
Statements: stmts,
},
Timings: d.Timings,
}
}
case metadataSet:
var d metadataSetSub
if err := json.Unmarshal(lc.Sub, &d); err != nil {
return err
}
c.Type = command.Command_COMMAND_TYPE_METADATA_SET
m = &command.MetadataSet{
RaftId: d.RaftID,
Data: d.Data,
}
case metadataDelete:
var d string
if err := json.Unmarshal(lc.Sub, &d); err != nil {
return err
}
c.Type = command.Command_COMMAND_TYPE_METADATA_DELETE
m = &command.MetadataDelete{
RaftId: d,
}
default:
return ErrUnknownCommandType
}
// Just marshal it, forget about compression, this will
// never go to disk after all.
b, err := proto.Marshal(m)
if err != nil {
return err
}
c.SubCommand = b
c.Compressed = false
return nil
}
func subCommandToStatements(d *databaseSub) ([]*command.Statement, error) {
stmts := make([]*command.Statement, len(d.SQLs))
for i := range d.SQLs {
stmts[i] = &command.Statement{
Sql: d.SQLs[i],
Parameters: nil,
}
// Support backwards-compatibility, since old versions didn't
// have any Parameters in legacy Raft commands
if len(d.Parameters) == 0 {
continue
}
stmts[i].Parameters = make([]*command.Parameter, len(d.Parameters[i]))
for j := range d.Parameters[i] {
switch v := d.Parameters[i][j].(type) {
case int:
case int64:
stmts[i].Parameters[j] = &command.Parameter{
Value: &command.Parameter_I{
I: v,
},
}
case float64:
stmts[i].Parameters[j] = &command.Parameter{
Value: &command.Parameter_D{
D: v,
},
}
case bool:
stmts[i].Parameters[j] = &command.Parameter{
Value: &command.Parameter_B{
B: v,
},
}
case []byte:
stmts[i].Parameters[j] = &command.Parameter{
Value: &command.Parameter_Y{
Y: v,
},
}
case string:
stmts[i].Parameters[j] = &command.Parameter{
Value: &command.Parameter_S{
S: v,
},
}
default:
return nil, ErrUnsupportedType
}
}
}
return stmts, nil
}

208
command/legacy/command_test.go
Unescape Escape View File

@ -0,0 +1,208 @@
package legacy
import (
"reflect"
"testing"
"github.com/rqlite/rqlite/command"
)
func Test_SimpleExecute(t *testing.T) {
// "CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT, age INTEGER)" timings=true
b := []byte{123, 34, 115, 117, 98, 34, 58, 123, 34, 113, 117, 101, 114, 105, 101, 115, 34, 58, 91, 34, 67, 82, 69, 65, 84, 69, 32, 84, 65, 66, 76, 69, 32, 102, 111, 111, 32, 40, 105, 100, 32, 73, 78, 84, 69, 71, 69, 82, 32, 78, 79, 84, 32, 78, 85, 76, 76, 32, 80, 82, 73, 77, 65, 82, 89, 32, 75, 69, 89, 44, 32, 110, 97, 109, 101, 32, 84, 69, 88, 84, 44, 32, 97, 103, 101, 32, 73, 78, 84, 69, 71, 69, 82, 41, 34, 93, 44, 34, 80, 97, 114, 97, 109, 101, 116, 101, 114, 115, 34, 58, 91, 110, 117, 108, 108, 93, 44, 34, 116, 105, 109, 105, 110, 103, 115, 34, 58, 116, 114, 117, 101, 125, 125}
var c command.Command
var er command.ExecuteRequest
if err := Unmarshal(b, &c); err != nil {
t.Fatalf("failed to Unmarshal: %s", err)
}
if c.Type != command.Command_COMMAND_TYPE_EXECUTE {
t.Fatalf("incorrect command type: %s", c.Type)
}
if err := command.UnmarshalSubCommand(&c, &er); err != nil {
t.Fatalf("failed to Unmarshal subcommand: %s", err)
}
if !er.Timings {
t.Fatalf("timings not set")
}
if er.Request.Transaction {
t.Fatalf("transaction set")
}
if n := len(er.Request.Statements); n != 1 {
t.Fatalf("incorrect number of statments: %d", n)
}
if s := er.Request.Statements[0].Sql; s != "CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT, age INTEGER)" {
t.Fatalf("incorrect SQL: %s", s)
}
}
func Test_SimpleQuery(t *testing.T) {
// "SELECT * FROM foo", timings=true
b := []byte{123, 34, 116, 121, 112, 34, 58, 49, 44, 34, 115, 117, 98, 34, 58, 123, 34, 113, 117, 101, 114, 105, 101, 115, 34, 58, 91, 34, 83, 69, 76, 69, 67, 84, 32, 42, 32, 70, 82, 79, 77, 32, 102, 111, 111, 34, 93, 44, 34, 80, 97, 114, 97, 109, 101, 116, 101, 114, 115, 34, 58, 91, 110, 117, 108, 108, 93, 44, 34, 116, 105, 109, 105, 110, 103, 115, 34, 58, 116, 114, 117, 101, 125, 125}
var c command.Command
var qr command.QueryRequest
if err := Unmarshal(b, &c); err != nil {
t.Fatalf("failed to Unmarshal: %s", err)
}
if c.Type != command.Command_COMMAND_TYPE_QUERY {
t.Fatalf("incorrect command type: %s", c.Type)
}
if err := command.UnmarshalSubCommand(&c, &qr); err != nil {
t.Fatalf("failed to Unmarshal subcommand: %s", err)
}
if !qr.Timings {
t.Fatalf("timings not set")
}
if qr.Request.Transaction {
t.Fatalf("transaction set")
}
if n := len(qr.Request.Statements); n != 1 {
t.Fatalf("incorrect number of statments: %d", n)
}
if s := qr.Request.Statements[0].Sql; s != "SELECT * FROM foo" {
t.Fatalf("incorrect SQL: %s", s)
}
}
func Test_SingleParameterized(t *testing.T) {
// ["INSERT INTO foo(name, age) VALUES(?, ?)", "fiona", 20], timings=true
b := []byte{123, 34, 115, 117, 98, 34, 58, 123, 34, 113, 117, 101, 114, 105, 101, 115, 34, 58, 91, 34, 73, 78, 83, 69, 82, 84, 32, 73, 78, 84, 79, 32, 102, 111, 111, 40, 110, 97, 109, 101, 44, 32, 97, 103, 101, 41, 32, 86, 65, 76, 85, 69, 83, 40, 63, 44, 32, 63, 41, 34, 93, 44, 34, 80, 97, 114, 97, 109, 101, 116, 101, 114, 115, 34, 58, 91, 91, 34, 102, 105, 111, 110, 97, 34, 44, 50, 48, 93, 93, 44, 34, 116, 105, 109, 105, 110, 103, 115, 34, 58, 116, 114, 117, 101, 125, 125}
var c command.Command
var er command.ExecuteRequest
if err := Unmarshal(b, &c); err != nil {
t.Fatalf("failed to Unmarshal: %s", err)
}
if c.Type != command.Command_COMMAND_TYPE_EXECUTE {
t.Fatalf("incorrect command type: %s", c.Type)
}
if err := command.UnmarshalSubCommand(&c, &er); err != nil {
t.Fatalf("failed to Unmarshal subcommand: %s", err)
}
if !er.Timings {
t.Fatalf("timings not set")
}
if er.Request.Transaction {
t.Fatalf("transaction set")
}
if n := len(er.Request.Statements); n != 1 {
t.Fatalf("incorrect number of statments: %d", n)
}
if s := er.Request.Statements[0].Sql; s != "INSERT INTO foo(name, age) VALUES(?, ?)" {
t.Fatalf("incorrect SQL: %s", s)
}
if l := len(er.Request.Statements[0].Parameters); l != 2 {
t.Fatalf("incorrect number of parameters: %d", l)
}
if v := er.Request.Statements[0].Parameters[0].GetS(); v != "fiona" {
t.Fatalf("incorrect value for 1st parameter: %s", v)
}
if v := er.Request.Statements[0].Parameters[1].GetD(); v != 20 {
t.Fatalf("incorrect value for 2nd parameter: %f", v)
}
}
func Test_MultipleParameterized(t *testing.T) {
// ["INSERT INTO foo(name, age) VALUES(?, ?)", "fiona", 20], ["INSERT INTO bar(name, age, address) VALUES(?, ?, ?)", "declan", 5, "galway"], timings=true
b := []byte{123, 34, 115, 117, 98, 34, 58, 123, 34, 113, 117, 101, 114, 105, 101, 115, 34, 58, 91, 34, 73, 78, 83, 69, 82, 84, 32, 73, 78, 84, 79, 32, 102, 111, 111, 40, 110, 97, 109, 101, 44, 32, 97, 103, 101, 41, 32, 86, 65, 76, 85, 69, 83, 40, 63, 44, 32, 63, 41, 34, 44, 34, 73, 78, 83, 69, 82, 84, 32, 73, 78, 84, 79, 32, 98, 97, 114, 40, 110, 97, 109, 101, 44, 32, 97, 103, 101, 44, 32, 97, 100, 100, 114, 101, 115, 115, 41, 32, 86, 65, 76, 85, 69, 83, 40, 63, 44, 32, 63, 44, 32, 63, 41, 34, 93, 44, 34, 80, 97, 114, 97, 109, 101, 116, 101, 114, 115, 34, 58, 91, 91, 34, 102, 105, 111, 110, 97, 34, 44, 50, 48, 93, 44, 91, 34, 100, 101, 99, 108, 97, 110, 34, 44, 53, 44, 34, 103, 97, 108, 119, 97, 121, 34, 93, 93, 44, 34, 116, 105, 109, 105, 110, 103, 115, 34, 58, 116, 114, 117, 101, 125, 125}
var c command.Command
var er command.ExecuteRequest
if err := Unmarshal(b, &c); err != nil {
t.Fatalf("failed to Unmarshal: %s", err)
}
if c.Type != command.Command_COMMAND_TYPE_EXECUTE {
t.Fatalf("incorrect command type: %s", c.Type)
}
if err := command.UnmarshalSubCommand(&c, &er); err != nil {
t.Fatalf("failed to Unmarshal subcommand: %s", err)
}
if !er.Timings {
t.Fatalf("timings not set")
}
if er.Request.Transaction {
t.Fatalf("transaction set")
}
if n := len(er.Request.Statements); n != 2 {
t.Fatalf("incorrect number of statments: %d", n)
}
if s := er.Request.Statements[0].Sql; s != "INSERT INTO foo(name, age) VALUES(?, ?)" {
t.Fatalf("incorrect SQL: %s", s)
}
if l := len(er.Request.Statements[0].Parameters); l != 2 {
t.Fatalf("incorrect number of parameters: %d", l)
}
if v := er.Request.Statements[0].Parameters[0].GetS(); v != "fiona" {
t.Fatalf("incorrect value for 1st parameter: %s", v)
}
if v := er.Request.Statements[0].Parameters[1].GetD(); v != 20 {
t.Fatalf("incorrect value for 2nd parameter: %f", v)
}
if s := er.Request.Statements[1].Sql; s != "INSERT INTO bar(name, age, address) VALUES(?, ?, ?)" {
t.Fatalf("incorrect SQL: %s", s)
}
if l := len(er.Request.Statements[1].Parameters); l != 3 {
t.Fatalf("incorrect number of parameters: %d", l)
}
if v := er.Request.Statements[1].Parameters[0].GetS(); v != "declan" {
t.Fatalf("incorrect value for 1st parameter: %s", v)
}
if v := er.Request.Statements[1].Parameters[1].GetD(); v != 5 {
t.Fatalf("incorrect value for 2nd parameter: %f", v)
}
if v := er.Request.Statements[1].Parameters[2].GetS(); v != "galway" {
t.Fatalf("incorrect value for 1st parameter: %s", v)
}
}
func Test_MetadataSet(t *testing.T) {
b := []byte{123, 34, 116, 121, 112, 34, 58, 50, 44, 34, 115, 117, 98, 34, 58, 123, 34, 114, 97, 102, 116, 95, 105, 100, 34, 58, 34, 108, 111, 99, 97, 108, 104, 111, 115, 116, 58, 52, 48, 48, 50, 34, 44, 34, 100, 97, 116, 97, 34, 58, 123, 34, 97, 112, 105, 95, 97, 100, 100, 114, 34, 58, 34, 108, 111, 99, 97, 108, 104, 111, 115, 116, 58, 52, 48, 48, 49, 34, 44, 34, 97, 112, 105, 95, 112, 114, 111, 116, 111, 34, 58, 34, 104, 116, 116, 112, 34, 125, 125, 125}
var c command.Command
var ms command.MetadataSet
if err := Unmarshal(b, &c); err != nil {
t.Fatalf("failed to Unmarshal: %s", err)
}
if c.Type != command.Command_COMMAND_TYPE_METADATA_SET {
t.Fatalf("incorrect command type: %s", c.Type)
}
if err := command.UnmarshalSubCommand(&c, &ms); err != nil {
t.Fatalf("failed to Unmarshal subcommand: %s", err)
}
if id := ms.RaftId; id != "localhost:4002" {
t.Fatalf("incorrect Raft ID: %s", id)
}
if !reflect.DeepEqual(ms.Data, map[string]string{"api_addr": "localhost:4001", "api_proto": "http"}) {
t.Fatalf("map is incorrect: %s", ms.Data)
}
}
func Test_MetadataDelete(t *testing.T) {
b := []byte{123, 34, 116, 121, 112, 34, 58, 51, 44, 34, 115, 117, 98, 34, 58, 34, 108, 111, 99, 97, 108, 104, 111, 115, 116, 58, 52, 48, 48, 52, 34, 125}
var c command.Command
var md command.MetadataDelete
if err := Unmarshal(b, &c); err != nil {
t.Fatalf("failed to Unmarshal: %s", err)
}
if c.Type != command.Command_COMMAND_TYPE_METADATA_DELETE {
t.Fatalf("incorrect command type: %s", c.Type)
}
if err := command.UnmarshalSubCommand(&c, &md); err != nil {
t.Fatalf("failed to Unmarshal subcommand: %s", err)
}
if id := md.RaftId; id != "localhost:4004" {
t.Fatalf("incorrect Raft ID: %s", id)
}
}

174
command/marshal.go
Unescape Escape View File

@ -0,0 +1,174 @@
package command
import (
"bytes"
"compress/gzip"
"expvar"
"fmt"
"io/ioutil"
"github.com/golang/protobuf/proto"
)
const (
DefaultBatchThreshold = 5
DefaultSizeThreshold = 150
)
type Requester interface {
proto.Message
GetRequest() *Request
}
type RequestMarshaler struct {
BatchThreshold int
SizeThreshold int
ForceCompression bool
gz *gzip.Writer
}
const (
numRequests = "num_requests"
numCompressedRequests = "num_compressed_requests"
numUncompressedRequests = "num_uncompressed_requests"
numCompressedBytes = "num_compressed_bytes"
numPrecompressedBytes = "num_precompressed_bytes"
numUncompressedBytes = "num_uncompressed_bytes"
numCompressionMisses = "num_compression_misses"
)
// stats captures stats for the Proto marshaler.
var stats *expvar.Map
func init() {
stats = expvar.NewMap("proto")
stats.Add(numRequests, 0)
stats.Add(numCompressedRequests, 0)
stats.Add(numUncompressedRequests, 0)
stats.Add(numCompressedBytes, 0)
stats.Add(numUncompressedBytes, 0)
stats.Add(numCompressionMisses, 0)
stats.Add(numPrecompressedBytes, 0)
}
func NewRequestMarshaler() *RequestMarshaler {
w, err := gzip.NewWriterLevel(nil, gzip.BestCompression)
if err != nil {
panic(fmt.Sprintf("failed to create GZIP writer: %s", err.Error()))
}
return &RequestMarshaler{
BatchThreshold: DefaultBatchThreshold,
SizeThreshold: DefaultSizeThreshold,
gz: w,
}
}
func (m *RequestMarshaler) Marshal(r Requester) ([]byte, bool, error) {
stats.Add(numRequests, 0)
compress := false
stmts := r.GetRequest().GetStatements()
if len(stmts) >= m.BatchThreshold {
compress = true
} else {
for i := range stmts {
if len(stmts[i].Sql) >= m.SizeThreshold {
compress = true
break
}
}
}
b, err := proto.Marshal(r)
if err != nil {
return nil, false, err
}
ubz := len(b)
stats.Add(numPrecompressedBytes, int64(ubz))
if compress {
// Let's try compression.
var buf bytes.Buffer
m.gz.Reset(&buf)
if _, err := m.gz.Write(b); err != nil {
return nil, false, err
}
if err := m.gz.Close(); err != nil {
return nil, false, err
}
// Is compression better?
if ubz > len(buf.Bytes()) || m.ForceCompression {
// Yes! Let's keep it.
b = buf.Bytes()
stats.Add(numCompressedRequests, 1)
stats.Add(numCompressedBytes, int64(len(b)))
} else {
// No. :-( Dump it.
compress = false
stats.Add(numCompressionMisses, 1)
}
} else {
stats.Add(numUncompressedRequests, 1)
stats.Add(numUncompressedBytes, int64(len(b)))
}
return b, compress, nil
}
func (m *RequestMarshaler) Stats() map[string]interface{} {
return map[string]interface{}{
"compression_size": m.SizeThreshold,
"compression_batch": m.BatchThreshold,
"force_compression": m.ForceCompression,
}
}
func Marshal(c *Command) ([]byte, error) {
return proto.Marshal(c)
}
func Unmarshal(b []byte, c *Command) error {
return proto.Unmarshal(b, c)
}
func MarshalMetadataSet(c *MetadataSet) ([]byte, error) {
return proto.Marshal(c)
}
func UnMarshalMetadataSet(b []byte, c *MetadataSet) error {
return proto.Unmarshal(b, c)
}
func MarshalMetadataDelete(c *MetadataDelete) ([]byte, error) {
return proto.Marshal(c)
}
func UnMarshalMetadataDelete(b []byte, c *MetadataDelete) error {
return proto.Unmarshal(b, c)
}
// Assumes m is the is the right type....caller must use c.Type
func UnmarshalSubCommand(c *Command, m proto.Message) error {
b := c.SubCommand
if c.Compressed {
gz, err := gzip.NewReader(bytes.NewReader(b))
if err != nil {
return err
}
ub, err := ioutil.ReadAll(gz)
if err != nil {
return err
}
if err := gz.Close(); err != nil {
return err
}
b = ub
}
return proto.Unmarshal(b, m)
}

238
command/marshal_test.go
Unescape Escape View File

@ -0,0 +1,238 @@
package command
import (
"testing"
"github.com/golang/protobuf/proto"
)
func Test_NewRequestMarshaler(t *testing.T) {
r := NewRequestMarshaler()
if r == nil {
t.Fatal("failed to create Request marshaler")
}
}
func Test_MarshalUncompressed(t *testing.T) {
rm := NewRequestMarshaler()
r := &QueryRequest{
Request: &Request{
Statements: []*Statement{
{
Sql: `INSERT INTO "names" VALUES(1,'bob','123-45-678')`,
},
},
},
Timings: true,
Freshness: 100,
}
b, comp, err := rm.Marshal(r)
if err != nil {
t.Fatalf("failed to marshal QueryRequest: %s", err)
}
if comp {
t.Fatal("Marshaled QueryRequest incorrectly compressed")
}
c := &Command{
Type: Command_COMMAND_TYPE_QUERY,
SubCommand: b,
Compressed: comp,
}
b, err = Marshal(c)
if err != nil {
t.Fatalf("failed to marshal Command: %s", err)
}
var nc Command
if err := Unmarshal(b, &nc); err != nil {
t.Fatalf("failed to unmarshal Command: %s", err)
}
if nc.Type != Command_COMMAND_TYPE_QUERY {
t.Fatalf("unmarshaled command has wrong type: %s", nc.Type)
}
if nc.Compressed {
t.Fatal("Unmarshaled QueryRequest incorrectly marked as compressed")
}
var nr QueryRequest
if err := UnmarshalSubCommand(&nc, &nr); err != nil {
t.Fatalf("failed to unmarshal sub command: %s", err)
}
if nr.Timings != r.Timings {
t.Fatalf("unmarshaled timings incorrect")
}
if nr.Freshness != r.Freshness {
t.Fatalf("unmarshaled Freshness incorrect")
}
if len(nr.Request.Statements) != 1 {
t.Fatalf("unmarshaled number of statements incorrect")
}
if nr.Request.Statements[0].Sql != `INSERT INTO "names" VALUES(1,'bob','123-45-678')` {
t.Fatalf("unmarshaled SQL incorrect")
}
}
func Test_MarshalCompressedBatch(t *testing.T) {
rm := NewRequestMarshaler()
rm.BatchThreshold = 1
rm.ForceCompression = true
r := &QueryRequest{
Request: &Request{
Statements: []*Statement{
{
Sql: `INSERT INTO "names" VALUES(1,'bob','123-45-678')`,
},
},
},
Timings: true,
Freshness: 100,
}
b, comp, err := rm.Marshal(r)
if err != nil {
t.Fatalf("failed to marshal QueryRequest: %s", err)
}
if !comp {
t.Fatal("Marshaled QueryRequest wasn't compressed")
}
c := &Command{
Type: Command_COMMAND_TYPE_QUERY,
SubCommand: b,
Compressed: comp,
}
b, err = Marshal(c)
if err != nil {
t.Fatalf("failed to marshal Command: %s", err)
}
var nc Command
if err := Unmarshal(b, &nc); err != nil {
t.Fatalf("failed to unmarshal Command: %s", err)
}
if nc.Type != Command_COMMAND_TYPE_QUERY {
t.Fatalf("unmarshaled command has wrong type: %s", nc.Type)
}
if !nc.Compressed {
t.Fatal("Unmarshaled QueryRequest incorrectly marked as uncompressed")
}
var nr QueryRequest
if err := UnmarshalSubCommand(&nc, &nr); err != nil {
t.Fatalf("failed to unmarshal sub command: %s", err)
}
if !proto.Equal(&nr, r) {
t.Fatal("Original and unmarshaled Query Request are not equal")
}
}
func Test_MarshalCompressedSize(t *testing.T) {
rm := NewRequestMarshaler()
rm.SizeThreshold = 1
rm.ForceCompression = true
r := &QueryRequest{
Request: &Request{
Statements: []*Statement{
{
Sql: `INSERT INTO "names" VALUES(1,'bob','123-45-678')`,
},
},
},
Timings: true,
Freshness: 100,
}
b, comp, err := rm.Marshal(r)
if err != nil {
t.Fatalf("failed to marshal QueryRequest: %s", err)
}
if !comp {
t.Fatal("Marshaled QueryRequest wasn't compressed")
}
c := &Command{
Type: Command_COMMAND_TYPE_QUERY,
SubCommand: b,
Compressed: comp,
}
b, err = Marshal(c)
if err != nil {
t.Fatalf("failed to marshal Command: %s", err)
}
var nc Command
if err := Unmarshal(b, &nc); err != nil {
t.Fatalf("failed to unmarshal Command: %s", err)
}
if nc.Type != Command_COMMAND_TYPE_QUERY {
t.Fatalf("unmarshaled command has wrong type: %s", nc.Type)
}
if !nc.Compressed {
t.Fatal("Unmarshaled QueryRequest incorrectly marked as uncompressed")
}
var nr QueryRequest
if err := UnmarshalSubCommand(&nc, &nr); err != nil {
t.Fatalf("failed to unmarshal sub command: %s", err)
}
if !proto.Equal(&nr, r) {
t.Fatal("Original and unmarshaled Query Request are not equal")
}
}
func Test_MarshalWontCompressBatch(t *testing.T) {
rm := NewRequestMarshaler()
rm.BatchThreshold = 1
r := &QueryRequest{
Request: &Request{
Statements: []*Statement{
{
Sql: `INSERT INTO "names" VALUES(1,'bob','123-45-678')`,
},
},
},
Timings: true,
Freshness: 100,
}
_, comp, err := rm.Marshal(r)
if err != nil {
t.Fatalf("failed to marshal QueryRequest: %s", err)
}
if comp {
t.Fatal("Marshaled QueryRequest was compressed")
}
}
func Test_MarshalWontCompressSize(t *testing.T) {
rm := NewRequestMarshaler()
rm.SizeThreshold = 1
r := &QueryRequest{
Request: &Request{
Statements: []*Statement{
{
Sql: `INSERT INTO "names" VALUES(1,'bob','123-45-678')`,
},
},
},
Timings: true,
Freshness: 100,
}
_, comp, err := rm.Marshal(r)
if err != nil {
t.Fatalf("failed to marshal QueryRequest: %s", err)
}
if comp {
t.Fatal("Marshaled QueryRequest was compressed")
}
}

93
db/db.go
Unescape Escape View File

@ -11,6 +11,7 @@ import (
"time"
"github.com/mattn/go-sqlite3"
"github.com/rqlite/rqlite/command"
)
const bkDelay = 250
@ -69,13 +70,6 @@ type Rows struct {
Time float64 `json:"time,omitempty"`
}
// Statement represents a single parameterized statement for processing
// by the database layer.
type Statement struct {
SQL string
Parameters []driver.Value
}
// Open opens a file-based database, creating it if it does not exist.
func Open(dbPath string) (*DB, error) {
return open(fqdsn(dbPath, ""))
@ -186,12 +180,21 @@ func (db *DB) AbortTransaction() error {
// ExecuteStringStmt executes a single query that modifies the database. This is
// primarily a convenience function.
func (db *DB) ExecuteStringStmt(query string) ([]*Result, error) {
return db.Execute([]Statement{{query, nil}}, false, false)
r := &command.Request{
Statements: []*command.Statement{
{
Sql: query,
},
},
}
return db.Execute(r, false)
}
// Execute executes queries that modify the database.
func (db *DB) Execute(stmts []Statement, tx, xTime bool) ([]*Result, error) {
stats.Add(numExecutions, int64(len(stmts)))
func (db *DB) Execute(req *command.Request, xTime bool) ([]*Result, error) {
stats.Add(numExecutions, int64(len(req.Statements)))
tx := req.Transaction
if tx {
stats.Add(numETx, 1)
}
@ -244,15 +247,24 @@ func (db *DB) Execute(stmts []Statement, tx, xTime bool) ([]*Result, error) {
}
// Execute each statement.
for _, stmt := range stmts {
if stmt.SQL == "" {
for _, stmt := range req.Statements {
sql := stmt.Sql
if sql == "" {
continue
}
result := &Result{}
start := time.Now()
r, err := execer.Exec(stmt.SQL, stmt.Parameters)
parameters, err := parametersToValues(stmt.Parameters)
if err != nil {
if handleError(result, err) {
continue
}
break
}
r, err := execer.Exec(sql, parameters)
if err != nil {
if handleError(result, err) {
continue
@ -294,12 +306,21 @@ func (db *DB) Execute(stmts []Statement, tx, xTime bool) ([]*Result, error) {
// QueryStringStmt executes a single query that return rows, but don't modify database.
func (db *DB) QueryStringStmt(query string) ([]*Rows, error) {
return db.Query([]Statement{{query, nil}}, false, false)
r := &command.Request{
Statements: []*command.Statement{
{
Sql: query,
},
},
}
return db.Query(r, false)
}
// Query executes queries that return rows, but don't modify the database.
func (db *DB) Query(stmts []Statement, tx, xTime bool) ([]*Rows, error) {
stats.Add(numQueries, int64(len(stmts)))
func (db *DB) Query(req *command.Request, xTime bool) ([]*Rows, error) {
stats.Add(numQueries, int64(len(req.Statements)))
tx := req.Transaction
if tx {
stats.Add(numQTx, 1)
}
@ -334,15 +355,23 @@ func (db *DB) Query(stmts []Statement, tx, xTime bool) ([]*Rows, error) {
}
}
for _, stmt := range stmts {
if stmt.SQL == "" {
for _, stmt := range req.Statements {
sql := stmt.Sql
if sql == "" {
continue
}
rows := &Rows{}
start := time.Now()
rs, err := queryer.Query(stmt.SQL, stmt.Parameters)
parameters, err := parametersToValues(stmt.Parameters)
if err != nil {
rows.Error = err.Error()
allRows = append(allRows, rows)
continue
}
rs, err := queryer.Query(sql, parameters)
if err != nil {
rows.Error = err.Error()
allRows = append(allRows, rows)
@ -519,6 +548,32 @@ func copyDatabase(dst *sqlite3.SQLiteConn, src *sqlite3.SQLiteConn) error {
return nil
}
// parametersToValues maps values in the proto params to SQL driver values.
func parametersToValues(parameters []*command.Parameter) ([]driver.Value, error) {
if parameters == nil {
return nil, nil
}
values := make([]driver.Value, len(parameters))
for i := range parameters {
switch w := parameters[i].GetValue().(type) {
case *command.Parameter_I:
values[i] = w.I
case *command.Parameter_D:
values[i] = w.D
case *command.Parameter_B:
values[i] = w.B
case *command.Parameter_Y:
values[i] = w.Y
case *command.Parameter_S:
values[i] = w.S
default:
return nil, fmt.Errorf("unsupported type: %T", w)
}
}
return values, nil
}
// normalizeRowValues performs some normalization of values in the returned rows.
// Text values come over (from sqlite-go) as []byte instead of strings
// for some reason, so we have explicitly convert (but only when type

260
db/db_test.go
Unescape Escape View File

@ -1,7 +1,6 @@
package db
import (
"database/sql/driver"
"encoding/json"
"fmt"
"io/ioutil"
@ -10,6 +9,7 @@ import (
"strings"
"testing"
"github.com/rqlite/rqlite/command"
"github.com/rqlite/rqlite/testdata/chinook"
)
@ -110,11 +110,11 @@ func Test_EmptyStatements(t *testing.T) {
defer db.Close()
defer os.Remove(path)
_, err := db.Execute([]Statement{{"", nil}}, false, false)
_, err := db.ExecuteStringStmt("")
if err != nil {
t.Fatalf("failed to execute empty statement: %s", err.Error())
}
_, err = db.Execute([]Statement{{";", nil}}, false, false)
_, err = db.ExecuteStringStmt(";")
if err != nil {
t.Fatalf("failed to execute empty statement with semicolon: %s", err.Error())
}
@ -215,11 +215,20 @@ func Test_SimpleJoinStatements(t *testing.T) {
t.Fatalf("failed to create table: %s", err.Error())
}
_, err = db.Execute([]Statement{
{`INSERT INTO "names" VALUES(1,'bob','123-45-678')`, nil},
{`INSERT INTO "names" VALUES(2,'tom','111-22-333')`, nil},
{`INSERT INTO "names" VALUES(3,'matt','222-22-333')`, nil},
}, false, false)
req := &command.Request{
Statements: []*command.Statement{
{
Sql: `INSERT INTO "names" VALUES(1,'bob','123-45-678')`,
},
{
Sql: `INSERT INTO "names" VALUES(2,'tom','111-22-333')`,
},
{
Sql: `INSERT INTO "names" VALUES(3,'matt','222-22-333')`,
},
},
}
_, err = db.Execute(req, false)
if err != nil {
t.Fatalf("failed to insert record: %s", err.Error())
}
@ -277,10 +286,17 @@ func Test_SimpleMultiStatements(t *testing.T) {
t.Fatalf("failed to create table: %s", err.Error())
}
re, err := db.Execute([]Statement{
{`INSERT INTO foo(name) VALUES("fiona")`, nil},
{`INSERT INTO foo(name) VALUES("dana")`, nil},
}, false, false)
req := &command.Request{
Statements: []*command.Statement{
{
Sql: `INSERT INTO foo(name) VALUES("fiona")`,
},
{
Sql: `INSERT INTO foo(name) VALUES("dana")`,
},
},
}
re, err := db.Execute(req, false)
if err != nil {
t.Fatalf("failed to insert record: %s", err.Error())
}
@ -288,10 +304,17 @@ func Test_SimpleMultiStatements(t *testing.T) {
t.Fatalf("unexpected results for query\nexp: %s\ngot: %s", exp, got)
}
ro, err := db.Query([]Statement{
{`SELECT * FROM foo`, nil},
{`SELECT * FROM foo`, nil},
}, false, false)
req = &command.Request{
Statements: []*command.Statement{
{
Sql: `SELECT * FROM foo`,
},
{
Sql: `SELECT * FROM foo`,
},
},
}
ro, err := db.Query(req, false)
if err != nil {
t.Fatalf("failed to query empty table: %s", err.Error())
}
@ -305,21 +328,33 @@ func Test_SimpleSingleMultiLineStatements(t *testing.T) {
defer db.Close()
defer os.Remove(path)
_, err := db.Execute([]Statement{
{`
req := &command.Request{
Statements: []*command.Statement{
{
Sql: `
CREATE TABLE foo (
id INTEGER NOT NULL PRIMARY KEY,
name TEXT
)`, nil}}, false, false)
id INTEGER NOT NULL PRIMARY KEY,
name TEXT
)`,
},
},
}
_, err := db.Execute(req, false)
if err != nil {
t.Fatalf("failed to create table: %s", err.Error())
}
re, err := db.Execute(
[]Statement{
{`INSERT INTO foo(name) VALUES("fiona")`, nil},
{`INSERT INTO foo(name) VALUES("dana")`, nil},
}, false, false)
req = &command.Request{
Statements: []*command.Statement{
{
Sql: `INSERT INTO foo(name) VALUES("fiona")`,
},
{
Sql: `INSERT INTO foo(name) VALUES("dana")`,
},
},
}
re, err := db.Execute(req, false)
if err != nil {
t.Fatalf("failed to insert record: %s", err.Error())
}
@ -439,17 +474,31 @@ func Test_SimpleParameterizedStatements(t *testing.T) {
t.Fatalf("failed to create table: %s", err.Error())
}
s := Statement{
SQL: "INSERT INTO foo(name) VALUES(?)",
Parameters: []driver.Value{"fiona"},
req := &command.Request{
Statements: []*command.Statement{
{
Sql: "INSERT INTO foo(name) VALUES(?)",
Parameters: []*command.Parameter{
{
Value: &command.Parameter_S{
S: "fiona",
},
},
},
},
},
}
_, err = db.Execute([]Statement{s}, false, false)
_, err = db.Execute(req, false)
if err != nil {
t.Fatalf("failed to insert record: %s", err.Error())
}
s.Parameters = []driver.Value{"aoife"}
_, err = db.Execute([]Statement{s}, false, false)
req.Statements[0].Parameters[0] = &command.Parameter{
Value: &command.Parameter_S{
S: "aoife",
},
}
_, err = db.Execute(req, false)
if err != nil {
t.Fatalf("failed to insert record: %s", err.Error())
}
@ -462,9 +511,13 @@ func Test_SimpleParameterizedStatements(t *testing.T) {
t.Fatalf("unexpected results for query\nexp: %s\ngot: %s", exp, got)
}
s.SQL = "SELECT * FROM foo WHERE name=?"
s.Parameters = []driver.Value{"aoife"}
r, err = db.Query([]Statement{s}, false, false)
req.Statements[0].Sql = "SELECT * FROM foo WHERE name=?"
req.Statements[0].Parameters[0] = &command.Parameter{
Value: &command.Parameter_S{
S: "aoife",
},
}
r, err = db.Query(req, false)
if err != nil {
t.Fatalf("failed to query table: %s", err.Error())
}
@ -472,8 +525,12 @@ func Test_SimpleParameterizedStatements(t *testing.T) {
t.Fatalf("unexpected results for query\nexp: %s\ngot: %s", exp, got)
}
s.Parameters = []driver.Value{"fiona"}
r, err = db.Query([]Statement{s}, false, false)
req.Statements[0].Parameters[0] = &command.Parameter{
Value: &command.Parameter_S{
S: "fiona",
},
}
r, err = db.Query(req, false)
if err != nil {
t.Fatalf("failed to query table: %s", err.Error())
}
@ -481,11 +538,31 @@ func Test_SimpleParameterizedStatements(t *testing.T) {
t.Fatalf("unexpected results for query\nexp: %s\ngot: %s", exp, got)
}
stmts := []Statement{
{"SELECT * FROM foo WHERE NAME=?", []driver.Value{"fiona"}},
{"SELECT * FROM foo WHERE NAME=?", []driver.Value{"aoife"}},
}
r, err = db.Query(stmts, false, false)
req = &command.Request{
Statements: []*command.Statement{
{
Sql: "SELECT * FROM foo WHERE NAME=?",
Parameters: []*command.Parameter{
{
Value: &command.Parameter_S{
S: "fiona",
},
},
},
},
{
Sql: "SELECT * FROM foo WHERE NAME=?",
Parameters: []*command.Parameter{
{
Value: &command.Parameter_S{
S: "aoife",
},
},
},
},
},
}
r, err = db.Query(req, false)
if err != nil {
t.Fatalf("failed to query table: %s", err.Error())
}
@ -687,13 +764,23 @@ func Test_PartialFail(t *testing.T) {
t.Fatalf("failed to create table: %s", err.Error())
}
stmts := []Statement{
{`INSERT INTO foo(id, name) VALUES(1, "fiona")`, nil},
{`INSERT INTO foo(id, name) VALUES(2, "fiona")`, nil},
{`INSERT INTO foo(id, name) VALUES(1, "fiona")`, nil},
{`INSERT INTO foo(id, name) VALUES(4, "fiona")`, nil},
}
r, err := db.Execute(stmts, false, false)
req := &command.Request{
Statements: []*command.Statement{
{
Sql: `INSERT INTO foo(id, name) VALUES(1, "fiona")`,
},
{
Sql: `INSERT INTO foo(id, name) VALUES(2, "fiona")`,
},
{
Sql: `INSERT INTO foo(id, name) VALUES(1, "fiona")`,
},
{
Sql: `INSERT INTO foo(id, name) VALUES(4, "fiona")`,
},
},
}
r, err := db.Execute(req, false)
if err != nil {
t.Fatalf("failed to insert records: %s", err.Error())
}
@ -719,13 +806,24 @@ func Test_SimpleTransaction(t *testing.T) {
t.Fatalf("failed to create table: %s", err.Error())
}
stmts := []Statement{
{`INSERT INTO foo(id, name) VALUES(1, "fiona")`, nil},
{`INSERT INTO foo(id, name) VALUES(2, "fiona")`, nil},
{`INSERT INTO foo(id, name) VALUES(3, "fiona")`, nil},
{`INSERT INTO foo(id, name) VALUES(4, "fiona")`, nil},
}
r, err := db.Execute(stmts, true, false)
req := &command.Request{
Transaction: true,
Statements: []*command.Statement{
{
Sql: `INSERT INTO foo(id, name) VALUES(1, "fiona")`,
},
{
Sql: `INSERT INTO foo(id, name) VALUES(2, "fiona")`,
},
{
Sql: `INSERT INTO foo(id, name) VALUES(3, "fiona")`,
},
{
Sql: `INSERT INTO foo(id, name) VALUES(4, "fiona")`,
},
},
}
r, err := db.Execute(req, false)
if err != nil {
t.Fatalf("failed to insert records: %s", err.Error())
}
@ -751,13 +849,24 @@ func Test_PartialFailTransaction(t *testing.T) {
t.Fatalf("failed to create table: %s", err.Error())
}
stmts := []Statement{
{`INSERT INTO foo(id, name) VALUES(1, "fiona")`, nil},
{`INSERT INTO foo(id, name) VALUES(2, "fiona")`, nil},
{`INSERT INTO foo(id, name) VALUES(1, "fiona")`, nil},
{`INSERT INTO foo(id, name) VALUES(4, "fiona")`, nil},
}
r, err := db.Execute(stmts, true, false)
req := &command.Request{
Transaction: true,
Statements: []*command.Statement{
{
Sql: `INSERT INTO foo(id, name) VALUES(1, "fiona")`,
},
{
Sql: `INSERT INTO foo(id, name) VALUES(2, "fiona")`,
},
{
Sql: `INSERT INTO foo(id, name) VALUES(1, "fiona")`,
},
{
Sql: `INSERT INTO foo(id, name) VALUES(4, "fiona")`,
},
},
}
r, err := db.Execute(req, false)
if err != nil {
t.Fatalf("failed to insert records: %s", err.Error())
}
@ -783,13 +892,24 @@ func Test_Backup(t *testing.T) {
t.Fatalf("failed to create table: %s", err.Error())
}
stmts := []Statement{
{`INSERT INTO foo(id, name) VALUES(1, "fiona")`, nil},
{`INSERT INTO foo(id, name) VALUES(2, "fiona")`, nil},
{`INSERT INTO foo(id, name) VALUES(3, "fiona")`, nil},
{`INSERT INTO foo(id, name) VALUES(4, "fiona")`, nil},
}
_, err = db.Execute(stmts, true, false)
req := &command.Request{
Transaction: true,
Statements: []*command.Statement{
{
Sql: `INSERT INTO foo(id, name) VALUES(1, "fiona")`,
},
{
Sql: `INSERT INTO foo(id, name) VALUES(2, "fiona")`,
},
{
Sql: `INSERT INTO foo(id, name) VALUES(3, "fiona")`,
},
{
Sql: `INSERT INTO foo(id, name) VALUES(4, "fiona")`,
},
},
}
_, err = db.Execute(req, false)
if err != nil {
t.Fatalf("failed to insert records: %s", err.Error())
}

2
go.mod
Unescape Escape View File

@ -6,6 +6,7 @@ require (
github.com/Bowery/prompt v0.0.0-20190916142128-fa8279994f75
github.com/armon/go-metrics v0.3.4 // indirect
github.com/fatih/color v1.10.0 // indirect
github.com/golang/protobuf v1.4.3
github.com/hashicorp/go-hclog v0.15.0 // indirect
github.com/hashicorp/go-immutable-radix v1.3.0 // indirect
github.com/hashicorp/go-msgpack v1.1.5 // indirect
@ -19,4 +20,5 @@ require (
golang.org/x/crypto v0.0.0-20201116153603-4be66e5b6582
golang.org/x/sys v0.0.0-20201116194326-cc9327a14d48 // indirect
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221 // indirect
google.golang.org/protobuf v1.25.0
)

34
go.sum
Unescape Escape View File

@ -22,6 +22,7 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
@ -34,6 +35,8 @@ github.com/denisenkom/go-mssqldb v0.0.0-20190707035753-2be1aa521ff4/go.mod h1:zA
github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg=
@ -55,12 +58,23 @@ github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
@ -162,6 +176,7 @@ github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3O
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
@ -200,6 +215,7 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@ -252,8 +268,10 @@ golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxb
golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190424220101-1e8e1cfdf96b/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
@ -262,8 +280,23 @@ google.golang.org/appengine v1.6.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@ -278,5 +311,6 @@ gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
xorm.io/builder v0.3.6/go.mod h1:LEFAPISnRzG+zxaxj2vPicRwz67BdhFreKg8yv8/TgU=
xorm.io/core v0.7.2-0.20190928055935-90aeac8d08eb/go.mod h1:jJfd0UAEzZ4t87nbQYtVjmqpIODugN6PD2D9E+dJvdM=

61
http/request_parser.go
Unescape Escape View File

@ -4,7 +4,7 @@ import (
"encoding/json"
"errors"
"github.com/rqlite/rqlite/store"
"github.com/rqlite/rqlite/command"
)
var (
@ -13,10 +13,13 @@ var (
// ErrInvalidRequest is returned when a request cannot be parsed.
ErrInvalidRequest = errors.New("invalid request")
// ErrUnsupportedType is returned when a request contains an unsupported type.
ErrUnsupportedType = errors.New("unsupported type")
)
// ParseRequest generates a set of Statements for a given byte slice.
func ParseRequest(b []byte) ([]store.Statement, error) {
func ParseRequest(b []byte) ([]*command.Statement, error) {
if b == nil {
return nil, ErrNoStatements
}
@ -31,9 +34,11 @@ func ParseRequest(b []byte) ([]store.Statement, error) {
return nil, ErrNoStatements
}
stmts := make([]store.Statement, len(simple))
stmts := make([]*command.Statement, len(simple))
for i := range simple {
stmts[i].SQL = simple[i]
stmts[i] = &command.Statement{
Sql: simple[i],
}
}
return stmts, nil
}
@ -42,26 +47,64 @@ func ParseRequest(b []byte) ([]store.Statement, error) {
if err := json.Unmarshal(b, &parameterized); err != nil {
return nil, ErrInvalidRequest
}
stmts := make([]store.Statement, len(parameterized))
stmts := make([]*command.Statement, len(parameterized))
for i := range parameterized {
if len(parameterized[i]) == 0 {
return nil, ErrNoStatements
}
var ok bool
stmts[i].SQL, ok = parameterized[i][0].(string)
sql, ok := parameterized[i][0].(string)
if !ok {
return nil, ErrInvalidRequest
}
stmts[i] = &command.Statement{
Sql: sql,
Parameters: nil,
}
if len(parameterized[i]) == 1 {
// No actual parameters after the SQL string
continue
}
stmts[i].Parameters = make([]store.Value, len(parameterized[i])-1)
stmts[i].Parameters = make([]*command.Parameter, len(parameterized[i])-1)
for j := range parameterized[i][1:] {
stmts[i].Parameters[j] = parameterized[i][j+1]
switch v := parameterized[i][j+1].(type) {
case int:
case int64:
stmts[i].Parameters[j] = &command.Parameter{
Value: &command.Parameter_I{
I: v,
},
}
case float64:
stmts[i].Parameters[j] = &command.Parameter{
Value: &command.Parameter_D{
D: v,
},
}
case bool:
stmts[i].Parameters[j] = &command.Parameter{
Value: &command.Parameter_B{
B: v,
},
}
case []byte:
stmts[i].Parameters[j] = &command.Parameter{
Value: &command.Parameter_Y{
Y: v,
},
}
case string:
stmts[i].Parameters[j] = &command.Parameter{
Value: &command.Parameter_S{
S: v,
},
}
default:
return nil, ErrUnsupportedType
}
}
}
return stmts, nil

40
http/request_parser_test.go
Unescape Escape View File

@ -38,8 +38,8 @@ func Test_SingleSimpleRequest(t *testing.T) {
if len(stmts) != 1 {
t.Fatalf("incorrect number of statements returned: %d", len(stmts))
}
if stmts[0].SQL != s {
t.Fatalf("incorrect statement parsed, exp %s, got %s", s, stmts[0].SQL)
if stmts[0].Sql != s {
t.Fatalf("incorrect statement parsed, exp %s, got %s", s, stmts[0].Sql)
}
if stmts[0].Parameters != nil {
t.Fatal("statement parameters are not nil")
@ -69,14 +69,14 @@ func Test_DoubleSimpleRequest(t *testing.T) {
if len(stmts) != 2 {
t.Fatalf("incorrect number of statements returned: %d", len(stmts))
}
if stmts[0].SQL != s0 {
t.Fatalf("incorrect statement parsed, exp %s, got %s", s0, stmts[0].SQL)
if stmts[0].Sql != s0 {
t.Fatalf("incorrect statement parsed, exp %s, got %s", s0, stmts[0].Sql)
}
if stmts[0].Parameters != nil {
t.Fatal("statement parameters are not nil")
}
if stmts[1].SQL != s1 {
t.Fatalf("incorrect statement parsed, exp %s, got %s", s1, stmts[1].SQL)
if stmts[1].Sql != s1 {
t.Fatalf("incorrect statement parsed, exp %s, got %s", s1, stmts[1].Sql)
}
if stmts[1].Parameters != nil {
t.Fatal("statement parameters are not nil")
@ -97,18 +97,18 @@ func Test_SingleParameterizedRequest(t *testing.T) {
if len(stmts) != 1 {
t.Fatalf("incorrect number of statements returned: %d", len(stmts))
}
if stmts[0].SQL != s {
t.Fatalf("incorrect statement parsed, exp %s, got %s", s, stmts[0].SQL)
if stmts[0].Sql != s {
t.Fatalf("incorrect statement parsed, exp %s, got %s", s, stmts[0].Sql)
}
if len(stmts[0].Parameters) != 2 {
t.Fatalf("incorrect number of parameters returned: %d", len(stmts[0].Parameters))
}
if stmts[0].Parameters[0] != p0 {
if stmts[0].Parameters[0].GetS() != p0 {
t.Fatalf("incorrect paramter, exp %s, got %s", p0, stmts[0].Parameters[0])
}
if int(stmts[0].Parameters[1].(float64)) != p1 {
t.Fatalf("incorrect paramter, exp %d, got %d", p1, stmts[0].Parameters[1])
if int(stmts[0].Parameters[1].GetD()) != p1 {
t.Fatalf("incorrect paramter, exp %d, got %d", p1, int(stmts[0].Parameters[1].GetI()))
}
}
@ -124,8 +124,8 @@ func Test_SingleParameterizedRequestNoParams(t *testing.T) {
if len(stmts) != 1 {
t.Fatalf("incorrect number of statements returned: %d", len(stmts))
}
if stmts[0].SQL != s {
t.Fatalf("incorrect statement parsed, exp %s, got %s", s, stmts[0].SQL)
if stmts[0].Sql != s {
t.Fatalf("incorrect statement parsed, exp %s, got %s", s, stmts[0].Sql)
}
if len(stmts[0].Parameters) != 0 {
@ -147,20 +147,20 @@ func Test_SingleParameterizedRequestNoParamsMixed(t *testing.T) {
if len(stmts) != 2 {
t.Fatalf("incorrect number of statements returned: %d", len(stmts))
}
if stmts[0].SQL != s1 {
t.Fatalf("incorrect statement parsed, exp %s, got %s", s1, stmts[0].SQL)
if stmts[0].Sql != s1 {
t.Fatalf("incorrect statement parsed, exp %s, got %s", s1, stmts[0].Sql)
}
if len(stmts[0].Parameters) != 0 {
t.Fatalf("incorrect number of parameters returned: %d", len(stmts[0].Parameters))
}
if stmts[1].SQL != s2 {
t.Fatalf("incorrect statement parsed, exp %s, got %s", s2, stmts[0].SQL)
if stmts[1].Sql != s2 {
t.Fatalf("incorrect statement parsed, exp %s, got %s", s2, stmts[0].Sql)
}
if len(stmts[1].Parameters) != 1 {
t.Fatalf("incorrect number of parameters returned: %d", len(stmts[0].Parameters))
}
if stmts[1].Parameters[0] != p2 {
if stmts[1].Parameters[0].GetS() != p2 {
t.Fatalf("incorrect parameter, exp %s, got %s", p2, stmts[1].Parameters[0])
}
}
@ -177,8 +177,8 @@ func Test_SingleSimpleParameterizedRequest(t *testing.T) {
if len(stmts) != 1 {
t.Fatalf("incorrect number of statements returned: %d", len(stmts))
}
if stmts[0].SQL != s {
t.Fatalf("incorrect statement parsed, exp %s, got %s", s, stmts[0].SQL)
if stmts[0].Sql != s {
t.Fatalf("incorrect statement parsed, exp %s, got %s", s, stmts[0].Sql)
}
if stmts[0].Parameters != nil {

90
http/service.go
Unescape Escape View File

@ -22,6 +22,7 @@ import (
"sync"
"time"
"github.com/rqlite/rqlite/command"
sql "github.com/rqlite/rqlite/db"
"github.com/rqlite/rqlite/store"
)
@ -32,17 +33,17 @@ type Database interface {
// to return rows. If timings is true, then timing information will
// be return. If tx is true, then either all queries will be executed
// successfully or it will as though none executed.
Execute(er *store.ExecuteRequest) ([]*sql.Result, error)
Execute(er *command.ExecuteRequest) ([]*sql.Result, error)
// ExecuteOrAbort performs the same function as Execute(), but ensures
// any transactions are aborted in case of any error.
ExecuteOrAbort(er *store.ExecuteRequest) ([]*sql.Result, error)
ExecuteOrAbort(er *command.ExecuteRequest) ([]*sql.Result, error)
// Query executes a slice of queries, each of which returns rows. If
// timings is true, then timing information will be returned. If tx
// is true, then all queries will take place while a read transaction
// is held on the database.
Query(qr *store.QueryRequest) ([]*sql.Rows, error)
Query(qr *command.QueryRequest) ([]*sql.Rows, error)
}
// Store is the interface the Raft-based database must implement.
@ -473,12 +474,9 @@ func (s *Service) handleLoad(w http.ResponseWriter, r *http.Request) {
// No JSON structure expected for this API.
queries := []string{string(b)}
stmts := make([]store.Statement, len(queries))
for i := range queries {
stmts[i].SQL = queries[i]
}
er := executeRequestFromStrings(queries, timings, false)
results, err := s.store.ExecuteOrAbort(&store.ExecuteRequest{stmts, timings, false})
results, err := s.store.ExecuteOrAbort(er)
if err != nil {
if err == store.ErrNotLeader {
leaderAPIAddr := s.LeaderAPIAddr()
@ -626,7 +624,15 @@ func (s *Service) handleExecute(w http.ResponseWriter, r *http.Request) {
return
}
results, err := s.store.Execute(&store.ExecuteRequest{stmts, timings, isTx})
er := &command.ExecuteRequest{
Request: &command.Request{
Transaction: isTx,
Statements: stmts,
},
Timings: timings,
}
results, err := s.store.Execute(er)
if err != nil {
if err == store.ErrNotLeader {
leaderAPIAddr := s.LeaderAPIAddr()
@ -695,7 +701,17 @@ func (s *Service) handleQuery(w http.ResponseWriter, r *http.Request) {
return
}
results, err := s.store.Query(&store.QueryRequest{queries, timings, isTx, lvl, frsh})
qr := &command.QueryRequest{
Request: &command.Request{
Transaction: isTx,
Statements: queries,
},
Timings: timings,
Level: lvl,
Freshness: frsh.Nanoseconds(),
}
results, err := s.store.Query(qr)
if err != nil {
if err == store.ErrNotLeader {
leaderAPIAddr := s.LeaderAPIAddr()
@ -855,14 +871,16 @@ func (s *Service) writeResponse(w http.ResponseWriter, r *http.Request, j *Respo
}
}
func requestQueries(r *http.Request) ([]store.Statement, error) {
func requestQueries(r *http.Request) ([]*command.Statement, error) {
if r.Method == "GET" {
query, err := stmtParam(r)
if err != nil || query == "" {
return nil, errors.New("bad query GET request")
}
return []store.Statement{
{query, nil},
return []*command.Statement{
&command.Statement{
Sql: query,
},
}, nil
}
@ -944,19 +962,19 @@ func timings(req *http.Request) (bool, error) {
}
// level returns the requested consistency level for a query
func level(req *http.Request) (store.ConsistencyLevel, error) {
func level(req *http.Request) (command.QueryRequest_Level, error) {
q := req.URL.Query()
lvl := strings.TrimSpace(q.Get("level"))
switch strings.ToLower(lvl) {
case "none":
return store.None, nil
return command.QueryRequest_QUERY_REQUEST_LEVEL_NONE, nil
case "weak":
return store.Weak, nil
return command.QueryRequest_QUERY_REQUEST_LEVEL_WEAK, nil
case "strong":
return store.Strong, nil
return command.QueryRequest_QUERY_REQUEST_LEVEL_STRONG, nil
default:
return store.Weak, nil
return command.QueryRequest_QUERY_REQUEST_LEVEL_WEAK, nil
}
}
@ -1018,3 +1036,39 @@ func EnsureHTTPS(addr string) string {
func CheckHTTPS(addr string) bool {
return strings.HasPrefix(addr, "https://")
}
// queryRequestFromStrings converts a slice of strings into a command.QueryRequest
func executeRequestFromStrings(s []string, timings, tx bool) *command.ExecuteRequest {
stmts := make([]*command.Statement, len(s))
for i := range s {
stmts[i] = &command.Statement{
Sql: s[i],
}
}
return &command.ExecuteRequest{
Request: &command.Request{
Statements: stmts,
Transaction: tx,
},
Timings: timings,
}
}
// queryRequestFromStrings converts a slice of strings into a command.QueryRequest
func queryRequestFromStrings(s []string, timings, tx bool) *command.QueryRequest {
stmts := make([]*command.Statement, len(s))
for i := range s {
stmts[i] = &command.Statement{
Sql: s[i],
}
}
return &command.QueryRequest{
Request: &command.Request{
Statements: stmts,
Transaction: tx,
},
Timings: timings,
}
}

7
http/service_test.go
Unescape Escape View File

@ -6,6 +6,7 @@ import (
"net/http"
"testing"
"github.com/rqlite/rqlite/command"
sql "github.com/rqlite/rqlite/db"
"github.com/rqlite/rqlite/store"
)
@ -611,18 +612,18 @@ type MockStore struct {
metadata map[string]string
}
func (m *MockStore) Execute(er *store.ExecuteRequest) ([]*sql.Result, error) {
func (m *MockStore) Execute(er *command.ExecuteRequest) ([]*sql.Result, error) {
if m.executeFn == nil {
return nil, nil
}
return nil, nil
}
func (m *MockStore) ExecuteOrAbort(er *store.ExecuteRequest) ([]*sql.Result, error) {
func (m *MockStore) ExecuteOrAbort(er *command.ExecuteRequest) ([]*sql.Result, error) {
return nil, nil
}
func (m *MockStore) Query(qr *store.QueryRequest) ([]*sql.Rows, error) {
func (m *MockStore) Query(qr *command.QueryRequest) ([]*sql.Rows, error) {
if m.queryFn == nil {
return nil, nil
}

54
store/command.go
Unescape Escape View File

@ -1,54 +0,0 @@
package store
import (
"encoding/json"
)
// commandType are commands that affect the state of the cluster, and must go through Raft.
type commandType int
const (
execute commandType = iota // Commands which modify the database.
query // Commands which query the database.
metadataSet // Commands which sets Store metadata
metadataDelete // Commands which deletes Store metadata
)
type command struct {
Typ commandType `json:"typ,omitempty"`
Sub json.RawMessage `json:"sub,omitempty"`
}
func newCommand(t commandType, d interface{}) (*command, error) {
b, err := json.Marshal(d)
if err != nil {
return nil, err
}
return &command{
Typ: t,
Sub: b,
}, nil
}
func newMetadataSetCommand(id string, md map[string]string) (*command, error) {
m := metadataSetSub{
RaftID: id,
Data: md,
}
return newCommand(metadataSet, m)
}
// databaseSub is a command sub which involves interaction with the database.
// Queries and Parameters are separate fields, for backwards-compatibility
// reasons. Unless Parameters is nil, it should be the same length as Queries.
type databaseSub struct {
Tx bool `json:"tx,omitempty"`
SQLs []string `json:"queries,omitempty"`
Parameters [][]Value `json:"Parameters,omitempty`
Timings bool `json:"timings,omitempty"`
}
type metadataSetSub struct {
RaftID string `json:"raft_id,omitempty"`
Data map[string]string `json:"data,omitempty"`
}

284
store/store.go
Unescape Escape View File

@ -5,7 +5,6 @@ package store
import (
"bytes"
gosql "database/sql/driver"
"encoding/binary"
"encoding/json"
"errors"
@ -23,6 +22,8 @@ import (
"github.com/hashicorp/raft"
"github.com/hashicorp/raft-boltdb"
"github.com/rqlite/rqlite/command"
legacy "github.com/rqlite/rqlite/command/legacy"
sql "github.com/rqlite/rqlite/db"
)
@ -57,9 +58,11 @@ const (
)
const (
numSnaphots = "num_snapshots"
numBackups = "num_backups"
numRestores = "num_restores"
numSnaphots = "num_snapshots"
numBackups = "num_backups"
numRestores = "num_restores"
numUncompressedCommands = "num_uncompressed_commands"
numCompressedCommands = "num_compressed_commands"
)
// BackupFormat represents the format of database backup.
@ -81,85 +84,10 @@ func init() {
stats.Add(numSnaphots, 0)
stats.Add(numBackups, 0)
stats.Add(numRestores, 0)
stats.Add(numUncompressedCommands, 0)
stats.Add(numCompressedCommands, 0)
}
// Value is the type for parameters passed to a parameterized SQL statement.
type Value interface{}
// Statement represent a parameterized SQL statement.
type Statement struct {
SQL string
Parameters []Value
}
// QueryRequest represents a query that returns rows, and does not modify
// the database.
type QueryRequest struct {
Stmts []Statement
Timings bool
Tx bool
Lvl ConsistencyLevel
Freshness time.Duration
}
func (q *QueryRequest) statements() []sql.Statement {
stmts := make([]sql.Statement, len(q.Stmts))
for i, s := range q.Stmts {
stmts[i].SQL = s.SQL
stmts[i].Parameters = make([]gosql.Value, len(s.Parameters))
for j := range s.Parameters {
stmts[i].Parameters[j] = s.Parameters[j]
}
}
return stmts
}
func (q *QueryRequest) command() *databaseSub {
c := databaseSub{
Tx: q.Tx,
SQLs: make([]string, len(q.Stmts)),
Parameters: make([][]Value, len(q.Stmts)),
Timings: q.Timings,
}
for i, s := range q.Stmts {
c.SQLs[i] = s.SQL
c.Parameters[i] = s.Parameters
}
return &c
}
// ExecuteRequest represents a query that returns no rows, but does modify
// the database.
type ExecuteRequest struct {
Stmts []Statement
Timings bool
Tx bool
}
func (e *ExecuteRequest) command() *databaseSub {
c := databaseSub{
Tx: e.Tx,
SQLs: make([]string, len(e.Stmts)),
Parameters: make([][]Value, len(e.Stmts)),
Timings: e.Timings,
}
for i, s := range e.Stmts {
c.SQLs[i] = s.SQL
c.Parameters[i] = s.Parameters
}
return &c
}
// ConsistencyLevel represents the available read consistency levels.
type ConsistencyLevel int
// Represents the available consistency levels.
const (
None ConsistencyLevel = iota
Weak
Strong
)
// ClusterState defines the possible Raft states the current node can be in
type ClusterState int
@ -186,9 +114,10 @@ type Store struct {
dbPath string // Path to underlying SQLite file, if not in-memory.
db *sql.DB // The underlying SQLite store.
raftLog raft.LogStore // Persistent log store.
raftStable raft.StableStore // Persistent k-v store.
boltStore *raftboltdb.BoltStore // Physical store.
reqMarshaller *command.RequestMarshaler // Request marshaler for writing to log.
raftLog raft.LogStore // Persistent log store.
raftStable raft.StableStore // Persistent k-v store.
boltStore *raftboltdb.BoltStore // Physical store.
metaMu sync.RWMutex
meta map[string]map[string]string
@ -222,14 +151,15 @@ func New(ln Listener, c *StoreConfig) *Store {
}
return &Store{
ln: ln,
raftDir: c.Dir,
raftID: c.ID,
dbConf: c.DBConf,
dbPath: filepath.Join(c.Dir, sqliteFile),
meta: make(map[string]map[string]string),
logger: logger,
ApplyTimeout: applyTimeout,
ln: ln,
raftDir: c.Dir,
raftID: c.ID,
dbConf: c.DBConf,
dbPath: filepath.Join(c.Dir, sqliteFile),
reqMarshaller: command.NewRequestMarshaler(),
meta: make(map[string]map[string]string),
logger: logger,
ApplyTimeout: applyTimeout,
}
}
@ -239,7 +169,8 @@ func (s *Store) Open(enableSingle bool) error {
s.logger.Printf("opening store with node ID %s", s.raftID)
s.logger.Printf("ensuring directory at %s exists", s.raftDir)
if err := os.MkdirAll(s.raftDir, 0755); err != nil {
err := os.MkdirAll(s.raftDir, 0755)
if err != nil {
return err
}
@ -430,6 +361,13 @@ func (s *Store) WaitForLeader(timeout time.Duration) (string, error) {
}
}
// SetRequestCompression allows low-level control over the compression threshold
// for the request marshaler.
func (s *Store) SetRequestCompression(batch, size int) {
s.reqMarshaller.BatchThreshold = batch
s.reqMarshaller.SizeThreshold = size
}
// WaitForAppliedIndex blocks until a given log index has been applied,
// or the timeout expires.
func (s *Store) WaitForAppliedIndex(idx uint64, timeout time.Duration) error {
@ -502,6 +440,7 @@ func (s *Store) Stats() (map[string]interface{}, error) {
"election_timeout": s.ElectionTimeout.String(),
"snapshot_threshold": s.SnapshotThreshold,
"snapshot_interval": s.SnapshotInterval,
"request_marshaler": s.reqMarshaller.Stats(),
"metadata": s.meta,
"nodes": nodes,
"dir": s.raftDir,
@ -512,7 +451,7 @@ func (s *Store) Stats() (map[string]interface{}, error) {
}
// Execute executes queries that return no rows, but do modify the database.
func (s *Store) Execute(ex *ExecuteRequest) ([]*sql.Result, error) {
func (s *Store) Execute(ex *command.ExecuteRequest) ([]*sql.Result, error) {
if s.raft.State() != raft.Leader {
return nil, ErrNotLeader
}
@ -521,7 +460,7 @@ func (s *Store) Execute(ex *ExecuteRequest) ([]*sql.Result, error) {
// ExecuteOrAbort executes the requests, but aborts any active transaction
// on the underlying database in the case of any error.
func (s *Store) ExecuteOrAbort(ex *ExecuteRequest) (results []*sql.Result, retErr error) {
func (s *Store) ExecuteOrAbort(ex *command.ExecuteRequest) (results []*sql.Result, retErr error) {
defer func() {
var errored bool
if results != nil {
@ -541,12 +480,24 @@ func (s *Store) ExecuteOrAbort(ex *ExecuteRequest) (results []*sql.Result, retEr
return s.execute(ex)
}
func (s *Store) execute(ex *ExecuteRequest) ([]*sql.Result, error) {
c, err := newCommand(execute, ex.command())
func (s *Store) execute(ex *command.ExecuteRequest) ([]*sql.Result, error) {
b, compressed, err := s.reqMarshaller.Marshal(ex)
if err != nil {
return nil, err
}
b, err := json.Marshal(c)
if compressed {
stats.Add(numCompressedCommands, 1)
} else {
stats.Add(numUncompressedCommands, 1)
}
c := &command.Command{
Type: command.Command_COMMAND_TYPE_EXECUTE,
SubCommand: b,
Compressed: compressed,
}
b, err = command.Marshal(c)
if err != nil {
return nil, err
}
@ -590,17 +541,29 @@ func (s *Store) Backup(leader bool, fmt BackupFormat, dst io.Writer) error {
}
// Query executes queries that return rows, and do not modify the database.
func (s *Store) Query(qr *QueryRequest) ([]*sql.Rows, error) {
func (s *Store) Query(qr *command.QueryRequest) ([]*sql.Rows, error) {
// Allow concurrent queries.
s.mu.RLock()
defer s.mu.RUnlock()
if qr.Lvl == Strong {
c, err := newCommand(query, qr.command())
if qr.Level == command.QueryRequest_QUERY_REQUEST_LEVEL_STRONG {
b, compressed, err := s.reqMarshaller.Marshal(qr)
if err != nil {
return nil, err
}
b, err := json.Marshal(c)
if compressed {
stats.Add(numCompressedCommands, 1)
} else {
stats.Add(numUncompressedCommands, 1)
}
c := &command.Command{
Type: command.Command_COMMAND_TYPE_QUERY,
SubCommand: b,
Compressed: compressed,
}
b, err = command.Marshal(c)
if err != nil {
return nil, err
}
@ -617,16 +580,17 @@ func (s *Store) Query(qr *QueryRequest) ([]*sql.Rows, error) {
return r.rows, r.error
}
if qr.Lvl == Weak && s.raft.State() != raft.Leader {
if qr.Level == command.QueryRequest_QUERY_REQUEST_LEVEL_WEAK && s.raft.State() != raft.Leader {
return nil, ErrNotLeader
}
if qr.Lvl == None && qr.Freshness > 0 && time.Since(s.raft.LastContact()) > qr.Freshness {
if qr.Level == command.QueryRequest_QUERY_REQUEST_LEVEL_NONE && qr.Freshness > 0 &&
time.Since(s.raft.LastContact()).Nanoseconds() > qr.Freshness {
return nil, ErrStaleRead
}
// Read straight from database.
return s.db.Query(qr.statements(), qr.Tx, qr.Timings)
return s.db.Query(qr.Request, qr.Timings)
}
// Join joins a node, identified by id and located at addr, to this store.
@ -738,15 +702,25 @@ func (s *Store) setMetadata(id string, md map[string]string) error {
return nil
}
c, err := newMetadataSetCommand(id, md)
ms := &command.MetadataSet{
RaftId: id,
Data: md,
}
bms, err := command.MarshalMetadataSet(ms)
if err != nil {
return err
}
b, err := json.Marshal(c)
c := &command.Command{
Type: command.Command_COMMAND_TYPE_METADATA_SET,
SubCommand: bms,
}
bc, err := command.Marshal(c)
if err != nil {
return err
}
f := s.raft.Apply(b, s.ApplyTimeout)
f := s.raft.Apply(bc, s.ApplyTimeout)
if e := f.(raft.Future); e.Error() != nil {
if e.Error() == raft.ErrNotLeader {
return ErrNotLeader
@ -796,15 +770,24 @@ func (s *Store) remove(id string) error {
return f.Error()
}
c, err := newCommand(metadataDelete, id)
md := command.MetadataDelete{
RaftId: id,
}
bmd, err := command.MarshalMetadataDelete(&md)
if err != nil {
return err
}
b, err := json.Marshal(c)
c := &command.Command{
Type: command.Command_COMMAND_TYPE_METADATA_DELETE,
SubCommand: bmd,
}
bc, err := command.Marshal(c)
if err != nil {
return err
}
f = s.raft.Apply(b, s.ApplyTimeout)
f = s.raft.Apply(bc, s.ApplyTimeout)
if e := f.(raft.Future); e.Error() != nil {
if e.Error() == raft.ErrNotLeader {
return ErrNotLeader
@ -854,54 +837,58 @@ type fsmGenericResponse struct {
// Apply applies a Raft log entry to the database.
func (s *Store) Apply(l *raft.Log) interface{} {
var c command
if err := json.Unmarshal(l.Data, &c); err != nil {
panic(fmt.Sprintf("failed to unmarshal cluster command: %s", err.Error()))
}
var c command.Command
switch c.Typ {
case execute, query:
var d databaseSub
if err := json.Unmarshal(c.Sub, &d); err != nil {
return &fsmGenericResponse{error: err}
if err := legacy.Unmarshal(l.Data, &c); err != nil {
if err = command.Unmarshal(l.Data, &c); err != nil {
panic(fmt.Sprintf("failed to unmarshal cluster command: %s", err.Error()))
}
stmts := subCommandToStatements(&d)
}
if c.Typ == execute {
r, err := s.db.Execute(stmts, d.Tx, d.Timings)
return &fsmExecuteResponse{results: r, error: err}
switch c.Type {
case command.Command_COMMAND_TYPE_QUERY:
var qr command.QueryRequest
if err := command.UnmarshalSubCommand(&c, &qr); err != nil {
panic(fmt.Sprintf("failed to unmarshal query subcommand: %s", err.Error()))
}
r, err := s.db.Query(stmts, d.Tx, d.Timings)
r, err := s.db.Query(qr.Request, qr.Timings)
return &fsmQueryResponse{rows: r, error: err}
case metadataSet:
var d metadataSetSub
if err := json.Unmarshal(c.Sub, &d); err != nil {
return &fsmGenericResponse{error: err}
case command.Command_COMMAND_TYPE_EXECUTE:
var er command.ExecuteRequest
if err := command.UnmarshalSubCommand(&c, &er); err != nil {
panic(fmt.Sprintf("failed to unmarshal execute subcommand: %s", err.Error()))
}
r, err := s.db.Execute(er.Request, er.Timings)
return &fsmExecuteResponse{results: r, error: err}
case command.Command_COMMAND_TYPE_METADATA_SET:
var ms command.MetadataSet
if err := command.UnmarshalSubCommand(&c, &ms); err != nil {
panic(fmt.Sprintf("failed to unmarshal metadata set subcommand: %s", err.Error()))
}
func() {
s.metaMu.Lock()
defer s.metaMu.Unlock()
if _, ok := s.meta[d.RaftID]; !ok {
s.meta[d.RaftID] = make(map[string]string)
if _, ok := s.meta[ms.RaftId]; !ok {
s.meta[ms.RaftId] = make(map[string]string)
}
for k, v := range d.Data {
s.meta[d.RaftID][k] = v
for k, v := range ms.Data {
s.meta[ms.RaftId][k] = v
}
}()
return &fsmGenericResponse{}
case metadataDelete:
var d string
if err := json.Unmarshal(c.Sub, &d); err != nil {
return &fsmGenericResponse{error: err}
case command.Command_COMMAND_TYPE_METADATA_DELETE:
var md command.MetadataDelete
if err := command.UnmarshalSubCommand(&c, &md); err != nil {
panic(fmt.Sprintf("failed to unmarshal metadata delete subcommand: %s", err.Error()))
}
func() {
s.metaMu.Lock()
defer s.metaMu.Unlock()
delete(s.meta, d)
delete(s.meta, md.RaftId)
}()
return &fsmGenericResponse{}
default:
return &fsmGenericResponse{error: fmt.Errorf("unknown command: %v", c.Typ)}
return &fsmGenericResponse{error: fmt.Errorf("unhandled command: %v", c.Type)}
}
}
@ -1122,25 +1109,6 @@ func (s *Store) database(leader bool, dst io.Writer) error {
// Release is a no-op.
func (f *fsmSnapshot) Release() {}
func subCommandToStatements(d *databaseSub) []sql.Statement {
stmts := make([]sql.Statement, len(d.SQLs))
for i := range d.SQLs {
stmts[i].SQL = d.SQLs[i]
// Support backwards-compatibility, since previous versions didn't
// have Parameters in Raft commands.
if len(d.Parameters) == 0 {
stmts[i].Parameters = make([]gosql.Value, 0)
} else {
stmts[i].Parameters = make([]gosql.Value, len(d.Parameters[i]))
for j := range d.Parameters[i] {
stmts[i].Parameters[j] = d.Parameters[i][j]
}
}
}
return stmts
}
// enabledFromBool converts bool to "enabled" or "disabled".
func enabledFromBool(b bool) string {
if b {

245
store/store_test.go
Unescape Escape View File

@ -11,6 +11,7 @@ import (
"testing"
"time"
"github.com/rqlite/rqlite/command"
sql "github.com/rqlite/rqlite/db"
"github.com/rqlite/rqlite/testdata/chinook"
)
@ -59,17 +60,18 @@ func Test_SingleNodeInMemExecuteQuery(t *testing.T) {
defer s.Close(true)
s.WaitForLeader(10 * time.Second)
queries := stmtsFromStrings([]string{
er := executeRequestFromStrings([]string{
`CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)`,
`INSERT INTO foo(id, name) VALUES(1, "fiona")`,
})
_, err := s.Execute(&ExecuteRequest{queries, false, false})
}, false, false)
_, err := s.Execute(er)
if err != nil {
t.Fatalf("failed to execute on single node: %s", err.Error())
}
queries = stmtsFromString("SELECT * FROM foo")
r, err := s.Query(&QueryRequest{queries, false, false, None, 0})
qr := queryRequestFromString("SELECT * FROM foo", false, false)
qr.Level = command.QueryRequest_QUERY_REQUEST_LEVEL_NONE
r, err := s.Query(qr)
if err != nil {
t.Fatalf("failed to query single node: %s", err.Error())
}
@ -92,10 +94,10 @@ func Test_SingleNodeInMemExecuteQueryFail(t *testing.T) {
defer s.Close(true)
s.WaitForLeader(10 * time.Second)
queries := stmtsFromStrings([]string{
er := executeRequestFromStrings([]string{
`INSERT INTO foo(id, name) VALUES(1, "fiona")`,
})
r, err := s.Execute(&ExecuteRequest{queries, false, false})
}, false, false)
r, err := s.Execute(er)
if err != nil {
t.Fatalf("failed to execute on single node: %s", err.Error())
}
@ -114,11 +116,11 @@ func Test_SingleNodeFileExecuteQuery(t *testing.T) {
defer s.Close(true)
s.WaitForLeader(10 * time.Second)
queries := stmtsFromStrings([]string{
er := executeRequestFromStrings([]string{
`CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)`,
`INSERT INTO foo(id, name) VALUES(1, "fiona")`,
})
_, err := s.Execute(&ExecuteRequest{queries, false, false})
}, false, false)
_, err := s.Execute(er)
if err != nil {
t.Fatalf("failed to execute on single node: %s", err.Error())
}
@ -133,31 +135,43 @@ func Test_SingleNodeFileExecuteQuery(t *testing.T) {
}
}
r, err := s.Query(&QueryRequest{stmtsFromString("SELECT * FROM foo"), false, false, None, 0})
qr := queryRequestFromString("SELECT * FROM foo", false, false)
qr.Level = command.QueryRequest_QUERY_REQUEST_LEVEL_NONE
r, err := s.Query(qr)
if err != nil {
t.Fatalf("failed to query single node: %s", err.Error())
}
check(r)
r, err = s.Query(&QueryRequest{stmtsFromString("SELECT * FROM foo"), false, false, Weak, 0})
qr = queryRequestFromString("SELECT * FROM foo", false, false)
qr.Level = command.QueryRequest_QUERY_REQUEST_LEVEL_WEAK
r, err = s.Query(qr)
if err != nil {
t.Fatalf("failed to query single node: %s", err.Error())
}
check(r)
r, err = s.Query(&QueryRequest{stmtsFromString("SELECT * FROM foo"), false, false, Strong, 0})
qr = queryRequestFromString("SELECT * FROM foo", false, false)
qr.Level = command.QueryRequest_QUERY_REQUEST_LEVEL_STRONG
r, err = s.Query(qr)
if err != nil {
t.Fatalf("failed to query single node: %s", err.Error())
}
check(r)
r, err = s.Query(&QueryRequest{stmtsFromString("SELECT * FROM foo"), false, true, None, 0})
qr = queryRequestFromString("SELECT * FROM foo", false, true)
qr.Timings = true
qr.Level = command.QueryRequest_QUERY_REQUEST_LEVEL_NONE
r, err = s.Query(qr)
if err != nil {
t.Fatalf("failed to query single node: %s", err.Error())
}
check(r)
r, err = s.Query(&QueryRequest{stmtsFromString("SELECT * FROM foo"), true, false, None, 0})
qr = queryRequestFromString("SELECT * FROM foo", true, false)
qr.Request.Transaction = true
qr.Level = command.QueryRequest_QUERY_REQUEST_LEVEL_NONE
r, err = s.Query(qr)
if err != nil {
t.Fatalf("failed to query single node: %s", err.Error())
}
@ -174,23 +188,31 @@ func Test_SingleNodeExecuteQueryTx(t *testing.T) {
defer s.Close(true)
s.WaitForLeader(10 * time.Second)
queries := stmtsFromStrings([]string{
er := executeRequestFromStrings([]string{
`CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)`,
`INSERT INTO foo(id, name) VALUES(1, "fiona")`,
})
_, err := s.Execute(&ExecuteRequest{queries, false, true})
}, false, true)
_, err := s.Execute(er)
if err != nil {
t.Fatalf("failed to execute on single node: %s", err.Error())
}
r, err := s.Query(&QueryRequest{stmtsFromString("SELECT * FROM foo"), false, true, None, 0})
qr := queryRequestFromString("SELECT * FROM foo", false, true)
qr.Level = command.QueryRequest_QUERY_REQUEST_LEVEL_NONE
r, err := s.Query(qr)
if err != nil {
t.Fatalf("failed to query single node: %s", err.Error())
}
r, err = s.Query(&QueryRequest{stmtsFromString("SELECT * FROM foo"), false, true, Weak, 0})
qr.Level = command.QueryRequest_QUERY_REQUEST_LEVEL_WEAK
r, err = s.Query(qr)
if err != nil {
t.Fatalf("failed to query single node: %s", err.Error())
}
r, err = s.Query(&QueryRequest{stmtsFromString("SELECT * FROM foo"), false, true, Strong, 0})
qr.Level = command.QueryRequest_QUERY_REQUEST_LEVEL_STRONG
r, err = s.Query(qr)
if err != nil {
t.Fatalf("failed to query single node: %s", err.Error())
}
@ -200,10 +222,6 @@ func Test_SingleNodeExecuteQueryTx(t *testing.T) {
if exp, got := `[[1,"fiona"]]`, asJSON(r[0].Values); exp != got {
t.Fatalf("unexpected results for query\nexp: %s\ngot: %s", exp, got)
}
_, err = s.Execute(&ExecuteRequest{queries, false, true})
if err != nil {
t.Fatalf("failed to execute on single node: %s", err.Error())
}
}
func Test_SingleNodeBackupBinary(t *testing.T) {
@ -224,7 +242,7 @@ CREATE TABLE foo (id integer not null primary key, name text);
INSERT INTO "foo" VALUES(1,'fiona');
COMMIT;
`
_, err := s.Execute(&ExecuteRequest{stmtsFromString(dump), false, false})
_, err := s.Execute(executeRequestFromString(dump, false, false))
if err != nil {
t.Fatalf("failed to load simple dump: %s", err.Error())
}
@ -272,7 +290,7 @@ CREATE TABLE foo (id integer not null primary key, name text);
INSERT INTO "foo" VALUES(1,'fiona');
COMMIT;
`
_, err := s.Execute(&ExecuteRequest{stmtsFromString(dump), false, false})
_, err := s.Execute(executeRequestFromString(dump, false, false))
if err != nil {
t.Fatalf("failed to load simple dump: %s", err.Error())
}
@ -311,13 +329,15 @@ CREATE TABLE foo (id integer not null primary key, name text);
INSERT INTO "foo" VALUES(1,'fiona');
COMMIT;
`
_, err := s.Execute(&ExecuteRequest{stmtsFromString(dump), false, false})
_, err := s.Execute(executeRequestFromString(dump, false, false))
if err != nil {
t.Fatalf("failed to load simple dump: %s", err.Error())
}
// Check that data were loaded correctly.
r, err := s.Query(&QueryRequest{stmtsFromString("SELECT * FROM foo"), false, true, Strong, 0})
qr := queryRequestFromString("SELECT * FROM foo", false, true)
qr.Level = command.QueryRequest_QUERY_REQUEST_LEVEL_STRONG
r, err := s.Query(qr)
if err != nil {
t.Fatalf("failed to query single node: %s", err.Error())
}
@ -353,13 +373,14 @@ CREATE VIEW foobar as select name as Person, Age as age from foo inner join bar
CREATE TRIGGER new_foobar instead of insert on foobar begin insert into foo (name) values (new.Person); insert into bar (nameid, age) values ((select id from foo where name == new.Person), new.Age); end;
COMMIT;
`
_, err := s.Execute(&ExecuteRequest{stmtsFromString(dump), false, false})
_, err := s.Execute(executeRequestFromString(dump, false, false))
if err != nil {
t.Fatalf("failed to load dump with trigger: %s", err.Error())
}
// Check that the VIEW and TRIGGER are OK by using both.
r, err := s.Execute(&ExecuteRequest{stmtsFromString("INSERT INTO foobar VALUES('jason', 16)"), false, true})
er := executeRequestFromString("INSERT INTO foobar VALUES('jason', 16)", false, true)
r, err := s.Execute(er)
if err != nil {
t.Fatalf("failed to insert into view on single node: %s", err.Error())
}
@ -382,7 +403,7 @@ func Test_SingleNodeLoadNoStatements(t *testing.T) {
BEGIN TRANSACTION;
COMMIT;
`
_, err := s.Execute(&ExecuteRequest{stmtsFromString(dump), false, false})
_, err := s.Execute(executeRequestFromString(dump, false, false))
if err != nil {
t.Fatalf("failed to load dump with no commands: %s", err.Error())
}
@ -399,7 +420,7 @@ func Test_SingleNodeLoadEmpty(t *testing.T) {
s.WaitForLeader(10 * time.Second)
dump := ``
_, err := s.Execute(&ExecuteRequest{stmtsFromString(dump), false, false})
_, err := s.Execute(executeRequestFromString(dump, false, false))
if err != nil {
t.Fatalf("failed to load empty dump: %s", err.Error())
}
@ -422,7 +443,7 @@ BEGIN TRANSACTION;
CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT);
COMMIT;
`
r, err := s.Execute(&ExecuteRequest{stmtsFromString(dump), false, false})
r, err := s.Execute(executeRequestFromString(dump, false, false))
if err != nil {
t.Fatalf("failed to load commands: %s", err.Error())
}
@ -430,7 +451,7 @@ COMMIT;
t.Fatalf("error received creating table: %s", r[0].Error)
}
r, err = s.Execute(&ExecuteRequest{stmtsFromString(dump), false, false})
r, err = s.Execute(executeRequestFromString(dump, false, false))
if err != nil {
t.Fatalf("failed to load commands: %s", err.Error())
}
@ -438,7 +459,7 @@ COMMIT;
t.Fatalf("received wrong error message: %s", r[0].Error)
}
r, err = s.Execute(&ExecuteRequest{stmtsFromString(dump), false, false})
r, err = s.Execute(executeRequestFromString(dump, false, false))
if err != nil {
t.Fatalf("failed to load commands: %s", err.Error())
}
@ -446,7 +467,7 @@ COMMIT;
t.Fatalf("received wrong error message: %s", r[0].Error)
}
r, err = s.ExecuteOrAbort(&ExecuteRequest{stmtsFromString(dump), false, false})
r, err = s.ExecuteOrAbort(executeRequestFromString(dump, false, false))
if err != nil {
t.Fatalf("failed to load commands: %s", err.Error())
}
@ -454,7 +475,7 @@ COMMIT;
t.Fatalf("received wrong error message: %s", r[0].Error)
}
r, err = s.Execute(&ExecuteRequest{stmtsFromString(dump), false, false})
r, err = s.Execute(executeRequestFromString(dump, false, false))
if err != nil {
t.Fatalf("failed to load commands: %s", err.Error())
}
@ -473,14 +494,16 @@ func Test_SingleNodeLoadChinook(t *testing.T) {
defer s.Close(true)
s.WaitForLeader(10 * time.Second)
_, err := s.Execute(&ExecuteRequest{stmtsFromString(chinook.DB), false, false})
_, err := s.Execute(executeRequestFromString(chinook.DB, false, false))
if err != nil {
t.Fatalf("failed to load chinook dump: %s", err.Error())
}
// Check that data were loaded correctly.
r, err := s.Query(&QueryRequest{stmtsFromString("SELECT count(*) FROM track"), false, true, Strong, 0})
qr := queryRequestFromString("SELECT count(*) FROM track", false, true)
qr.Level = command.QueryRequest_QUERY_REQUEST_LEVEL_STRONG
r, err := s.Query(qr)
if err != nil {
t.Fatalf("failed to query single node: %s", err.Error())
}
@ -491,7 +514,9 @@ func Test_SingleNodeLoadChinook(t *testing.T) {
t.Fatalf("unexpected results for query\nexp: %s\ngot: %s", exp, got)
}
r, err = s.Query(&QueryRequest{stmtsFromString("SELECT count(*) FROM album"), false, true, Strong, 0})
qr = queryRequestFromString("SELECT count(*) FROM album", false, true)
qr.Level = command.QueryRequest_QUERY_REQUEST_LEVEL_STRONG
r, err = s.Query(qr)
if err != nil {
t.Fatalf("failed to query single node: %s", err.Error())
}
@ -502,7 +527,9 @@ func Test_SingleNodeLoadChinook(t *testing.T) {
t.Fatalf("unexpected results for query\nexp: %s\ngot: %s", exp, got)
}
r, err = s.Query(&QueryRequest{stmtsFromString("SELECT count(*) FROM artist"), false, true, Strong, 0})
qr = queryRequestFromString("SELECT count(*) FROM artist", false, true)
qr.Level = command.QueryRequest_QUERY_REQUEST_LEVEL_STRONG
r, err = s.Query(qr)
if err != nil {
t.Fatalf("failed to query single node: %s", err.Error())
}
@ -684,15 +711,17 @@ func Test_MultiNodeExecuteQuery(t *testing.T) {
t.Fatalf("failed to join to node at %s: %s", s0.Addr(), err.Error())
}
queries := stmtsFromStrings([]string{
er := executeRequestFromStrings([]string{
`CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)`,
`INSERT INTO foo(id, name) VALUES(1, "fiona")`,
})
_, err := s0.Execute(&ExecuteRequest{queries, false, false})
}, false, false)
_, err := s0.Execute(er)
if err != nil {
t.Fatalf("failed to execute on single node: %s", err.Error())
}
r, err := s0.Query(&QueryRequest{stmtsFromString("SELECT * FROM foo"), false, false, None, 0})
qr := queryRequestFromString("SELECT * FROM foo", false, false)
qr.Level = command.QueryRequest_QUERY_REQUEST_LEVEL_NONE
r, err := s0.Query(qr)
if err != nil {
t.Fatalf("failed to query leader node: %s", err.Error())
}
@ -708,15 +737,18 @@ func Test_MultiNodeExecuteQuery(t *testing.T) {
if err := s1.WaitForAppliedIndex(3, 5*time.Second); err != nil {
t.Fatalf("error waiting for follower to apply index: %s:", err.Error())
}
r, err = s1.Query(&QueryRequest{stmtsFromString("SELECT * FROM foo"), false, false, Weak, 0})
qr.Level = command.QueryRequest_QUERY_REQUEST_LEVEL_WEAK
r, err = s1.Query(qr)
if err == nil {
t.Fatalf("successfully queried non-leader node")
}
r, err = s1.Query(&QueryRequest{stmtsFromString("SELECT * FROM foo"), false, false, Strong, 0})
qr.Level = command.QueryRequest_QUERY_REQUEST_LEVEL_STRONG
r, err = s1.Query(qr)
if err == nil {
t.Fatalf("successfully queried non-leader node")
}
r, err = s1.Query(&QueryRequest{stmtsFromString("SELECT * FROM foo"), false, false, None, 0})
qr.Level = command.QueryRequest_QUERY_REQUEST_LEVEL_NONE
r, err = s1.Query(qr)
if err != nil {
t.Fatalf("failed to query follower node: %s", err.Error())
}
@ -732,15 +764,18 @@ func Test_MultiNodeExecuteQuery(t *testing.T) {
if err := s2.WaitForAppliedIndex(3, 5*time.Second); err != nil {
t.Fatalf("error waiting for follower to apply index: %s:", err.Error())
}
r, err = s2.Query(&QueryRequest{stmtsFromString("SELECT * FROM foo"), false, false, Weak, 0})
qr.Level = command.QueryRequest_QUERY_REQUEST_LEVEL_WEAK
r, err = s1.Query(qr)
if err == nil {
t.Fatalf("successfully queried non-voting node with Weak")
}
r, err = s2.Query(&QueryRequest{stmtsFromString("SELECT * FROM foo"), false, false, Strong, 0})
qr.Level = command.QueryRequest_QUERY_REQUEST_LEVEL_STRONG
r, err = s1.Query(qr)
if err == nil {
t.Fatalf("successfully queried non-voting node with Strong")
}
r, err = s2.Query(&QueryRequest{stmtsFromString("SELECT * FROM foo"), false, false, None, 0})
qr.Level = command.QueryRequest_QUERY_REQUEST_LEVEL_NONE
r, err = s1.Query(qr)
if err != nil {
t.Fatalf("failed to query non-voting node: %s", err.Error())
}
@ -773,15 +808,18 @@ func Test_MultiNodeExecuteQueryFreshness(t *testing.T) {
t.Fatalf("failed to join to node at %s: %s", s0.Addr(), err.Error())
}
queries := stmtsFromStrings([]string{
er := executeRequestFromStrings([]string{
`CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)`,
`INSERT INTO foo(id, name) VALUES(1, "fiona")`,
})
_, err := s0.Execute(&ExecuteRequest{queries, false, false})
}, false, false)
_, err := s0.Execute(er)
if err != nil {
t.Fatalf("failed to execute on single node: %s", err.Error())
}
r, err := s0.Query(&QueryRequest{stmtsFromString("SELECT * FROM foo"), false, false, None, 0})
qr := queryRequestFromString("SELECT * FROM foo", false, false)
qr.Level = command.QueryRequest_QUERY_REQUEST_LEVEL_NONE
r, err := s0.Query(qr)
if err != nil {
t.Fatalf("failed to query leader node: %s", err.Error())
}
@ -800,13 +838,16 @@ func Test_MultiNodeExecuteQueryFreshness(t *testing.T) {
// "Weak" consistency queries with 1 nanosecond freshness should pass, because freshness
// is ignored in this case.
r, err = s0.Query(&QueryRequest{stmtsFromString("SELECT * FROM foo"), false, false, Weak, mustParseDuration("1ns")})
qr.Level = command.QueryRequest_QUERY_REQUEST_LEVEL_WEAK
qr.Freshness = mustParseDuration("1ns").Nanoseconds()
r, err = s0.Query(qr)
if err != nil {
t.Fatalf("Failed to ignore freshness if level is Weak: %s", err.Error())
}
qr.Level = command.QueryRequest_QUERY_REQUEST_LEVEL_STRONG
// "Strong" consistency queries with 1 nanosecond freshness should pass, because freshness
// is ignored in this case.
r, err = s0.Query(&QueryRequest{stmtsFromString("SELECT * FROM foo"), false, false, Strong, mustParseDuration("1ns")})
r, err = s0.Query(qr)
if err != nil {
t.Fatalf("Failed to ignore freshness if level is Strong: %s", err.Error())
}
@ -815,7 +856,9 @@ func Test_MultiNodeExecuteQueryFreshness(t *testing.T) {
s0.Close(true)
// "None" consistency queries should still work.
r, err = s1.Query(&QueryRequest{stmtsFromString("SELECT * FROM foo"), false, false, None, 0})
qr.Level = command.QueryRequest_QUERY_REQUEST_LEVEL_NONE
qr.Freshness = 0
r, err = s1.Query(qr)
if err != nil {
t.Fatalf("failed to query follower node: %s", err.Error())
}
@ -831,7 +874,9 @@ func Test_MultiNodeExecuteQueryFreshness(t *testing.T) {
// "None" consistency queries with 1 nanosecond freshness should fail, because at least
// one nanosecond *should* have passed since leader died (surely!).
r, err = s1.Query(&QueryRequest{stmtsFromString("SELECT * FROM foo"), false, false, None, mustParseDuration("1ns")})
qr.Level = command.QueryRequest_QUERY_REQUEST_LEVEL_NONE
qr.Freshness = mustParseDuration("1ns").Nanoseconds()
r, err = s1.Query(qr)
if err == nil {
t.Fatalf("freshness violating query didn't return an error")
}
@ -840,7 +885,8 @@ func Test_MultiNodeExecuteQueryFreshness(t *testing.T) {
}
// Freshness of 0 is ignored.
r, err = s1.Query(&QueryRequest{stmtsFromString("SELECT * FROM foo"), false, false, None, 0})
qr.Freshness = 0
r, err = s1.Query(qr)
if err != nil {
t.Fatalf("failed to query follower node: %s", err.Error())
}
@ -853,7 +899,8 @@ func Test_MultiNodeExecuteQueryFreshness(t *testing.T) {
// "None" consistency queries with 1 hour freshness should pass, because it should
// not be that long since the leader died.
r, err = s1.Query(&QueryRequest{stmtsFromString("SELECT * FROM foo"), false, false, None, mustParseDuration("1h")})
qr.Freshness = mustParseDuration("1h").Nanoseconds()
r, err = s1.Query(qr)
if err != nil {
t.Fatalf("failed to query follower node: %s", err.Error())
}
@ -888,7 +935,7 @@ func Test_StoreLogTruncationMultinode(t *testing.T) {
`INSERT INTO foo(id, name) VALUES(5, "fiona")`,
}
for i := range queries {
_, err := s0.Execute(&ExecuteRequest{stmtsFromString(queries[i]), false, false})
_, err := s0.Execute(executeRequestFromString(queries[i], false, false))
if err != nil {
t.Fatalf("failed to execute on single node: %s", err.Error())
}
@ -918,7 +965,8 @@ func Test_StoreLogTruncationMultinode(t *testing.T) {
if err := s1.WaitForAppliedIndex(8, 5*time.Second); err != nil {
t.Fatalf("error waiting for follower to apply index: %s:", err.Error())
}
r, err := s1.Query(&QueryRequest{stmtsFromString("SELECT count(*) FROM foo"), false, true, None, 0})
qr := queryRequestFromString("SELECT count(*) FROM foo", false, true)
r, err := s1.Query(qr)
if err != nil {
t.Fatalf("failed to query single node: %s", err.Error())
}
@ -940,15 +988,15 @@ func Test_SingleNodeSnapshotOnDisk(t *testing.T) {
defer s.Close(true)
s.WaitForLeader(10 * time.Second)
queries := stmtsFromStrings([]string{
queries := []string{
`CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)`,
`INSERT INTO foo(id, name) VALUES(1, "fiona")`,
})
_, err := s.Execute(&ExecuteRequest{queries, false, false})
}
_, err := s.Execute(executeRequestFromStrings(queries, false, false))
if err != nil {
t.Fatalf("failed to execute on single node: %s", err.Error())
}
_, err = s.Query(&QueryRequest{stmtsFromString("SELECT * FROM foo"), false, false, None, 0})
_, err = s.Query(queryRequestFromString("SELECT * FROM foo", false, false))
if err != nil {
t.Fatalf("failed to query single node: %s", err.Error())
}
@ -980,7 +1028,7 @@ func Test_SingleNodeSnapshotOnDisk(t *testing.T) {
}
// Ensure database is back in the correct state.
r, err := s.Query(&QueryRequest{stmtsFromString("SELECT * FROM foo"), false, false, None, 0})
r, err := s.Query(queryRequestFromString("SELECT * FROM foo", false, false))
if err != nil {
t.Fatalf("failed to query single node: %s", err.Error())
}
@ -1002,15 +1050,15 @@ func Test_SingleNodeSnapshotInMem(t *testing.T) {
defer s.Close(true)
s.WaitForLeader(10 * time.Second)
queries := stmtsFromStrings([]string{
queries := []string{
`CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)`,
`INSERT INTO foo(id, name) VALUES(1, "fiona")`,
})
_, err := s.Execute(&ExecuteRequest{queries, false, false})
}
_, err := s.Execute(executeRequestFromStrings(queries, false, false))
if err != nil {
t.Fatalf("failed to execute on single node: %s", err.Error())
}
_, err = s.Query(&QueryRequest{stmtsFromString("SELECT * FROM foo"), false, false, None, 0})
_, err = s.Query(queryRequestFromString("SELECT * FROM foo", false, false))
if err != nil {
t.Fatalf("failed to query single node: %s", err.Error())
}
@ -1042,7 +1090,7 @@ func Test_SingleNodeSnapshotInMem(t *testing.T) {
}
// Ensure database is back in the correct state.
r, err := s.Query(&QueryRequest{stmtsFromString("SELECT * FROM foo"), false, false, None, 0})
r, err := s.Query(queryRequestFromString("SELECT * FROM foo", false, false))
if err != nil {
t.Fatalf("failed to query single node: %s", err.Error())
}
@ -1218,17 +1266,48 @@ func mustParseDuration(t string) time.Duration {
return d
}
func stmtsFromString(s string) []Statement {
return stmtsFromStrings([]string{s})
func executeRequestFromString(s string, timings, tx bool) *command.ExecuteRequest {
return executeRequestFromStrings([]string{s}, timings, tx)
}
// stmtsFromStrings converts a slice of strings into unparameterized DB statements
func stmtsFromStrings(s []string) []Statement {
stmts := make([]Statement, len(s))
for i, ss := range s {
stmts[i] = Statement{ss, nil}
// queryRequestFromStrings converts a slice of strings into a command.QueryRequest
func executeRequestFromStrings(s []string, timings, tx bool) *command.ExecuteRequest {
stmts := make([]*command.Statement, len(s))
for i := range s {
stmts[i] = &command.Statement{
Sql: s[i],
}
}
return &command.ExecuteRequest{
Request: &command.Request{
Statements: stmts,
Transaction: tx,
},
Timings: timings,
}
}
func queryRequestFromString(s string, timings, tx bool) *command.QueryRequest {
return queryRequestFromStrings([]string{s}, timings, tx)
}
// queryRequestFromStrings converts a slice of strings into a command.QueryRequest
func queryRequestFromStrings(s []string, timings, tx bool) *command.QueryRequest {
stmts := make([]*command.Statement, len(s))
for i := range s {
stmts[i] = &command.Statement{
Sql: s[i],
}
}
return &command.QueryRequest{
Request: &command.Request{
Statements: stmts,
Transaction: tx,
},
Timings: timings,
}
return stmts
}
func asJSON(v interface{}) string {

Write Preview
Loading…
Cancel
Save