diff --git a/api/instance_service.go b/api/instance_service.go index f143e445..0c4b177f 100644 --- a/api/instance_service.go +++ b/api/instance_service.go @@ -23,7 +23,8 @@ type InstanceAPI struct { func (s *InstanceAPI) Create(ctx context.Context, in *CreateRequest) (*CreateReply, error) { inst, err := model.Instances(ctx).Create(&model.Instance{ - Template: in.GetTemplate(), + Template: in.GetTemplate(), + AutoRecovery: in.GetAutoRecovery(), }) if err != nil { log.WithError(err).Error() @@ -94,7 +95,10 @@ func (s *InstanceAPI) Run(ctx context.Context, in *CreateRequest) (*RunReply, er if err := checkSupportAPI(in.GetTemplate(), ctx); err != nil { return nil, err } - res1, err := s.Create(ctx, &CreateRequest{Template: in.GetTemplate()}) + res1, err := s.Create(ctx, &CreateRequest{ + Template: in.GetTemplate(), + AutoRecovery: in.GetAutoRecovery(), + }) if err != nil { log.WithError(err).Error("Failed InstanceAPI.Run at Create") return nil, err diff --git a/api/v1.pb.go b/api/v1.pb.go index 32b75a20..1982a3db 100644 --- a/api/v1.pb.go +++ b/api/v1.pb.go @@ -280,7 +280,8 @@ func (m *ConsoleReply) GetAddress() string { type CreateRequest struct { // string resource_id = 1; // Obsolete - Template *model.Template `protobuf:"bytes,2,opt,name=template" json:"template,omitempty"` + Template *model.Template `protobuf:"bytes,2,opt,name=template" json:"template,omitempty"` + AutoRecovery bool `protobuf:"varint,3,opt,name=auto_recovery,json=autoRecovery" json:"auto_recovery,omitempty"` } func (m *CreateRequest) Reset() { *m = CreateRequest{} } @@ -295,6 +296,13 @@ func (m *CreateRequest) GetTemplate() *model.Template { return nil } +func (m *CreateRequest) GetAutoRecovery() bool { + if m != nil { + return m.AutoRecovery + } + return false +} + type CreateReply struct { InstanceId string `protobuf:"bytes,1,opt,name=instance_id" json:"instance_id,omitempty"` } @@ -1402,71 +1410,72 @@ var _Instance_serviceDesc = grpc.ServiceDesc{ func init() { proto.RegisterFile("v1.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ - // 1048 bytes of a gzipped FileDescriptorProto + // 1068 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x56, 0xdb, 0x6e, 0xdb, 0x46, - 0x10, 0x95, 0x48, 0x89, 0x96, 0x46, 0x96, 0x2c, 0xaf, 0x5d, 0x5b, 0x65, 0x51, 0xd4, 0x21, 0x8a, - 0xc6, 0x48, 0x1c, 0x4a, 0x51, 0x8a, 0x5e, 0x92, 0x16, 0x41, 0x64, 0x37, 0x88, 0x50, 0x23, 0x6d, - 0xd7, 0x7a, 0xea, 0x4b, 0xb0, 0x96, 0xd6, 0x14, 0x0b, 0x8a, 0xcb, 0x92, 0x4b, 0x37, 0x7a, 0xec, - 0x27, 0xf5, 0x43, 0xfa, 0x31, 0xfd, 0x83, 0x82, 0x7b, 0xa1, 0x48, 0x47, 0x06, 0xd8, 0xbe, 0x08, - 0x9c, 0xd9, 0x39, 0xe7, 0xec, 0xce, 0xce, 0xce, 0x08, 0x5a, 0xb7, 0x4f, 0xdd, 0x28, 0x66, 0x9c, - 0x21, 0x93, 0x44, 0xbe, 0xdd, 0x59, 0xb1, 0x05, 0x0d, 0xa4, 0xc7, 0xee, 0xce, 0x83, 0x34, 0xe1, - 0x34, 0x56, 0xe6, 0x0b, 0xcf, 0xe7, 0xcb, 0xf4, 0xda, 0x9d, 0xb3, 0xd5, 0xd0, 0x63, 0x01, 0x09, - 0xbd, 0xa1, 0x58, 0xb8, 0x4e, 0x6f, 0x86, 0x11, 0x5f, 0x47, 0x34, 0x19, 0x72, 0x7f, 0x45, 0x13, - 0x4e, 0x56, 0xd1, 0xe6, 0x4b, 0x82, 0x9d, 0x97, 0xd0, 0x3e, 0x67, 0x61, 0xc2, 0x02, 0x3a, 0x0d, - 0x51, 0x1f, 0xcc, 0xeb, 0xf4, 0x66, 0x50, 0x3f, 0xa9, 0x9f, 0xb6, 0x71, 0xf6, 0x89, 0x4e, 0xa0, - 0xe3, 0x87, 0x09, 0x27, 0xe1, 0x9c, 0xbe, 0xf3, 0x17, 0x03, 0x43, 0xac, 0x14, 0x5d, 0xce, 0x6f, - 0x00, 0x8a, 0xe0, 0xa7, 0x94, 0xa3, 0x87, 0x60, 0xcc, 0x97, 0x82, 0xa0, 0x37, 0x3e, 0x76, 0x49, - 0xe4, 0xbb, 0x9b, 0x45, 0xf7, 0x7c, 0x49, 0xc2, 0x90, 0x06, 0xd8, 0x98, 0x2f, 0xb5, 0x94, 0x91, - 0x4b, 0x39, 0x0f, 0x60, 0x47, 0x05, 0x20, 0x00, 0x2b, 0xe1, 0x0b, 0x96, 0xf2, 0x7e, 0x4d, 0x7d, - 0xd3, 0x38, 0xee, 0xd7, 0x9d, 0x33, 0x68, 0xe1, 0x34, 0xc4, 0x34, 0x0a, 0xd6, 0x77, 0x77, 0x56, - 0xff, 0x70, 0x67, 0x43, 0xe8, 0x5c, 0x71, 0x16, 0x61, 0xfa, 0x7b, 0x4a, 0x13, 0x5e, 0x01, 0xf0, - 0x04, 0xda, 0x12, 0x50, 0x8d, 0x7f, 0x0c, 0xbd, 0x0b, 0x9a, 0xf0, 0x98, 0xad, 0xab, 0x4b, 0x8c, - 0x60, 0x37, 0xc7, 0x54, 0x56, 0x51, 0x29, 0xac, 0xae, 0xf2, 0x1e, 0x76, 0x73, 0x4c, 0x25, 0x15, - 0x74, 0x06, 0x8d, 0xac, 0x52, 0xc4, 0x7d, 0xf4, 0xc6, 0x03, 0x57, 0x96, 0x9b, 0x22, 0x71, 0x67, - 0x31, 0x09, 0x93, 0x88, 0xc5, 0x1c, 0x8b, 0x28, 0x34, 0x80, 0x1d, 0xb2, 0x58, 0xc4, 0x34, 0x49, - 0x06, 0xa6, 0xe0, 0xd2, 0xa6, 0xf3, 0x1d, 0x74, 0xcf, 0x63, 0x4a, 0x78, 0xbe, 0xd9, 0xc7, 0xd0, - 0xe2, 0x74, 0x15, 0x05, 0x84, 0x4b, 0xf2, 0xce, 0x78, 0x4f, 0x91, 0xcf, 0x94, 0x1b, 0xe7, 0x01, - 0xd9, 0x8d, 0x69, 0x74, 0xb5, 0xe4, 0x8c, 0x60, 0xf7, 0x8a, 0x93, 0x98, 0x57, 0x4f, 0x8d, 0x0b, - 0xa0, 0x10, 0xd5, 0x14, 0x9e, 0x42, 0x17, 0xd3, 0x6b, 0xc6, 0xfe, 0x83, 0xc4, 0x10, 0x3a, 0x1a, - 0x52, 0x4d, 0x63, 0x02, 0xfb, 0x53, 0x65, 0x4e, 0x2f, 0xb4, 0x4e, 0x1f, 0x8c, 0xe9, 0x85, 0x8c, - 0x7e, 0x53, 0xc3, 0xc6, 0xf4, 0x02, 0x1d, 0x42, 0xe3, 0x2d, 0x59, 0xc9, 0x34, 0x66, 0x3e, 0x61, - 0x4d, 0x9a, 0x60, 0xfe, 0x48, 0xd7, 0xce, 0x25, 0x74, 0x35, 0x87, 0x94, 0xed, 0x6d, 0xf0, 0x02, - 0xfd, 0x18, 0x5a, 0x5a, 0xf3, 0xce, 0x45, 0xe4, 0xb8, 0x3c, 0xc0, 0xf9, 0xd3, 0x80, 0x03, 0xed, - 0xbe, 0xf4, 0x93, 0xfc, 0xf0, 0xdf, 0x40, 0x23, 0x22, 0x1e, 0x15, 0xb4, 0x9d, 0xf1, 0xe7, 0xe2, - 0x81, 0x6f, 0x89, 0x73, 0x7f, 0x26, 0x9e, 0xae, 0x00, 0x2c, 0x10, 0xe8, 0x6b, 0xb0, 0x6e, 0xfc, - 0x80, 0xd3, 0x58, 0x89, 0x7f, 0x76, 0x2f, 0xf6, 0xb5, 0x08, 0xc3, 0x2a, 0xdc, 0x7e, 0x01, 0x9d, - 0x02, 0x1b, 0x3a, 0x02, 0x8b, 0xdd, 0xdc, 0x24, 0x94, 0x8b, 0x3d, 0x34, 0xb1, 0xb2, 0xd0, 0x21, - 0x34, 0x03, 0x7f, 0xe5, 0x73, 0x41, 0xdf, 0xc4, 0xd2, 0xb0, 0x9f, 0x83, 0x25, 0xe9, 0xd0, 0x08, - 0x9a, 0x09, 0xcf, 0x8a, 0x50, 0xf6, 0x26, 0xfb, 0xce, 0xd9, 0xaf, 0xb2, 0x35, 0x57, 0xfc, 0x62, - 0x19, 0xe8, 0xfc, 0x65, 0x6c, 0xae, 0x45, 0xee, 0x2f, 0x4b, 0xeb, 0x97, 0xa5, 0x0c, 0x9c, 0x6c, - 0x39, 0x45, 0x14, 0xac, 0xd5, 0xf9, 0xa3, 0x60, 0xad, 0x4e, 0xff, 0x3d, 0x34, 0x7d, 0x4e, 0x57, - 0xc9, 0xc0, 0x38, 0x31, 0x4f, 0x3b, 0xe3, 0x87, 0xf7, 0xc0, 0x8a, 0x9e, 0x29, 0xa7, 0x2b, 0x2c, - 0x51, 0xf6, 0x2f, 0xd0, 0xce, 0x19, 0xb3, 0x93, 0x72, 0xc6, 0x49, 0xa0, 0x12, 0x20, 0x8d, 0xec, - 0x49, 0xce, 0xd3, 0x38, 0xa6, 0xa1, 0xce, 0x80, 0x36, 0x37, 0x99, 0x31, 0x8b, 0x99, 0x99, 0x41, - 0xff, 0xae, 0x5a, 0x56, 0x32, 0x79, 0x81, 0x1a, 0xfe, 0x62, 0x93, 0x33, 0xa3, 0x6a, 0xce, 0x2e, - 0x00, 0xe5, 0xac, 0xcc, 0xd3, 0x77, 0xe6, 0x82, 0x35, 0x23, 0xb1, 0xa7, 0xee, 0xac, 0x33, 0x3e, - 0x2a, 0x1d, 0x3f, 0x2f, 0x79, 0xac, 0xa2, 0x9c, 0x2f, 0x0a, 0x7b, 0xcb, 0x58, 0xb2, 0x53, 0x23, - 0x68, 0x5c, 0xfa, 0x61, 0x96, 0x77, 0xf3, 0xb4, 0x8d, 0xc5, 0xb7, 0xf3, 0x1a, 0x0e, 0x75, 0xdc, - 0x0f, 0xb7, 0x34, 0xe4, 0xff, 0x57, 0xef, 0x9f, 0xfa, 0x66, 0xdb, 0x8a, 0x28, 0x93, 0x7c, 0x05, - 0x40, 0x33, 0xeb, 0x9d, 0xe8, 0x8c, 0xb2, 0x6e, 0x1e, 0x94, 0xa8, 0x36, 0xc1, 0xae, 0xf8, 0x9c, - 0xad, 0x23, 0x8a, 0x0b, 0x20, 0xf4, 0x15, 0xb4, 0xa4, 0x45, 0xb8, 0xaa, 0x7b, 0xdb, 0xf5, 0x18, - 0xf3, 0x02, 0xea, 0xea, 0x11, 0xed, 0xce, 0xf4, 0x44, 0xc6, 0x79, 0x2c, 0x3a, 0xd3, 0x99, 0x37, - 0x05, 0xe8, 0x70, 0x5b, 0xe6, 0xdf, 0xd4, 0x74, 0xd6, 0x1f, 0x41, 0x3b, 0x97, 0x47, 0x5d, 0x65, - 0xbc, 0x65, 0x21, 0xed, 0xd7, 0x50, 0x0f, 0x40, 0x98, 0x02, 0xd2, 0xaf, 0x4f, 0x2c, 0x68, 0x4c, - 0xd8, 0x62, 0x3d, 0x9e, 0xc0, 0x9e, 0x66, 0x53, 0x5d, 0x1e, 0x0d, 0xc1, 0x7a, 0xc5, 0x39, 0x99, - 0x2f, 0x51, 0xaf, 0x38, 0xb9, 0xa7, 0xa1, 0xbd, 0x77, 0x67, 0x92, 0x3b, 0xb5, 0xd3, 0xfa, 0xa8, - 0x3e, 0xfe, 0xbb, 0x01, 0x2d, 0x4d, 0x82, 0x46, 0x60, 0xc9, 0xde, 0x8d, 0x90, 0x8c, 0x2e, 0x8e, - 0x01, 0xbb, 0x5f, 0xf2, 0x45, 0xc1, 0xda, 0xa9, 0xa1, 0x27, 0xd0, 0x14, 0xad, 0x18, 0xed, 0x8b, - 0xc5, 0x62, 0x23, 0x57, 0x8a, 0x9b, 0x4e, 0xed, 0xd4, 0xd0, 0x23, 0x30, 0x71, 0x1a, 0x6e, 0x65, - 0xef, 0x0a, 0x9f, 0xfe, 0x6b, 0x20, 0x62, 0x1b, 0xd9, 0x24, 0x47, 0x7d, 0x45, 0x93, 0xff, 0x0b, - 0xb0, 0x7b, 0x05, 0x8f, 0x8c, 0x7d, 0x06, 0x3b, 0x6a, 0x24, 0xa3, 0x03, 0xb1, 0x58, 0x1e, 0xea, - 0xf6, 0x7e, 0xd9, 0x29, 0x41, 0x23, 0xb0, 0x64, 0x8f, 0x57, 0xfb, 0x29, 0xcd, 0x08, 0x75, 0xda, - 0xc2, 0x10, 0x90, 0x32, 0x3a, 0xd1, 0x07, 0xc5, 0x74, 0x96, 0x65, 0x8a, 0x63, 0xdb, 0xa9, 0x65, - 0xdd, 0xe6, 0x6a, 0xc9, 0xfe, 0x40, 0xf7, 0x54, 0xb0, 0x8d, 0x4a, 0x7e, 0x8d, 0x7a, 0x9e, 0xbd, - 0x95, 0x84, 0xa3, 0xc1, 0x7d, 0x3d, 0xd6, 0x3e, 0xda, 0xde, 0x80, 0x9c, 0x1a, 0xfa, 0x16, 0xcc, - 0x4b, 0xe6, 0xa1, 0xe3, 0x72, 0x40, 0xfe, 0x96, 0xed, 0x8f, 0x3e, 0x5c, 0x10, 0xc0, 0x51, 0x1d, - 0xbd, 0x84, 0xa6, 0x28, 0x35, 0xf4, 0xf1, 0xb6, 0x47, 0x22, 0xe1, 0xc7, 0xf7, 0xbc, 0x9f, 0x8c, - 0x60, 0xf2, 0xe9, 0xaf, 0x9f, 0x14, 0xfe, 0xca, 0x92, 0xf7, 0xc9, 0x72, 0xc8, 0x22, 0x1a, 0xde, - 0x2e, 0xe6, 0x43, 0x12, 0xf9, 0xd7, 0x96, 0x78, 0x32, 0xcf, 0xfe, 0x0d, 0x00, 0x00, 0xff, 0xff, - 0xed, 0x7c, 0xc9, 0xf5, 0x1b, 0x0b, 0x00, 0x00, + 0x10, 0x95, 0xa8, 0x4b, 0xa4, 0x91, 0x25, 0xcb, 0x6b, 0xd7, 0x56, 0x59, 0x14, 0x75, 0xd8, 0xa2, + 0x31, 0x12, 0x87, 0x52, 0x94, 0xa2, 0x97, 0x04, 0x45, 0x10, 0xd9, 0x0d, 0x22, 0xd4, 0x48, 0xdb, + 0xb5, 0x9e, 0xfa, 0x62, 0xac, 0xa5, 0xb5, 0xc4, 0x82, 0xe2, 0xb2, 0xe4, 0xd2, 0x0d, 0x1f, 0xfb, + 0x49, 0xfd, 0x90, 0x7e, 0x4c, 0xff, 0xa0, 0xd8, 0x1b, 0x45, 0x3a, 0x32, 0xc0, 0xf6, 0x45, 0xe0, + 0xcc, 0xce, 0x39, 0x67, 0x77, 0x76, 0x76, 0x46, 0xd0, 0xba, 0x7d, 0xe6, 0x86, 0x11, 0xe3, 0x0c, + 0xd5, 0x48, 0xe8, 0xd9, 0x9d, 0x35, 0x5b, 0x50, 0x5f, 0x79, 0xec, 0xee, 0xdc, 0x4f, 0x62, 0x4e, + 0x23, 0x6d, 0xbe, 0x5c, 0x7a, 0x7c, 0x95, 0x5c, 0xbb, 0x73, 0xb6, 0x1e, 0x2e, 0x99, 0x4f, 0x82, + 0xe5, 0x50, 0x2e, 0x5c, 0x27, 0x37, 0xc3, 0x90, 0xa7, 0x21, 0x8d, 0x87, 0xdc, 0x5b, 0xd3, 0x98, + 0x93, 0x75, 0xb8, 0xf9, 0x52, 0x60, 0xe7, 0x15, 0xb4, 0xcf, 0x58, 0x10, 0x33, 0x9f, 0x4e, 0x03, + 0xd4, 0x87, 0xda, 0x75, 0x72, 0x33, 0xa8, 0x1e, 0x57, 0x4f, 0xda, 0x58, 0x7c, 0xa2, 0x63, 0xe8, + 0x78, 0x41, 0xcc, 0x49, 0x30, 0xa7, 0x57, 0xde, 0x62, 0x60, 0xc9, 0x95, 0xbc, 0xcb, 0xf9, 0x0d, + 0x40, 0x13, 0xfc, 0x94, 0x70, 0xf4, 0x08, 0xac, 0xf9, 0x4a, 0x12, 0xf4, 0xc6, 0x47, 0x2e, 0x09, + 0x3d, 0x77, 0xb3, 0xe8, 0x9e, 0xad, 0x48, 0x10, 0x50, 0x1f, 0x5b, 0xf3, 0x95, 0x91, 0xb2, 0x32, + 0x29, 0xe7, 0x21, 0x3c, 0xd0, 0x01, 0x08, 0xa0, 0x19, 0xf3, 0x05, 0x4b, 0x78, 0xbf, 0xa2, 0xbf, + 0x69, 0x14, 0xf5, 0xab, 0xce, 0x29, 0xb4, 0x70, 0x12, 0x60, 0x1a, 0xfa, 0xe9, 0xdd, 0x9d, 0x55, + 0x3f, 0xdc, 0xd9, 0x10, 0x3a, 0x97, 0x9c, 0x85, 0x98, 0xfe, 0x9e, 0xd0, 0x98, 0x97, 0x00, 0x3c, + 0x85, 0xb6, 0x02, 0x94, 0xe3, 0x1f, 0x43, 0xef, 0x9c, 0xc6, 0x3c, 0x62, 0x69, 0x79, 0x89, 0x11, + 0xec, 0x64, 0x98, 0xd2, 0x2a, 0x3a, 0x85, 0xe5, 0x55, 0xde, 0xc3, 0x4e, 0x86, 0x29, 0xa5, 0x82, + 0x4e, 0xa1, 0x2e, 0x2a, 0x45, 0xde, 0x47, 0x6f, 0x3c, 0x70, 0x55, 0xb9, 0x69, 0x12, 0x77, 0x16, + 0x91, 0x20, 0x0e, 0x59, 0xc4, 0xb1, 0x8c, 0x42, 0x03, 0x78, 0x40, 0x16, 0x8b, 0x88, 0xc6, 0xf1, + 0xa0, 0x26, 0xb9, 0x8c, 0xe9, 0x10, 0xe8, 0x9e, 0x45, 0x94, 0xf0, 0x6c, 0xb3, 0x4f, 0xa0, 0xc5, + 0xe9, 0x3a, 0xf4, 0x09, 0x57, 0xe4, 0x9d, 0xf1, 0xae, 0x26, 0x9f, 0x69, 0x37, 0xce, 0x02, 0xd0, + 0xe7, 0xd0, 0x25, 0x09, 0x67, 0x57, 0x11, 0x9d, 0xb3, 0x5b, 0x1a, 0xa5, 0x92, 0xbd, 0x85, 0x77, + 0x84, 0x13, 0x6b, 0x9f, 0xb8, 0x56, 0x23, 0x51, 0x2e, 0x83, 0x23, 0xd8, 0xb9, 0xe4, 0x24, 0xe2, + 0xe5, 0xf3, 0xe7, 0x02, 0x68, 0x44, 0x39, 0x85, 0x67, 0xd0, 0xc5, 0xf4, 0x9a, 0xb1, 0xff, 0x20, + 0x31, 0x84, 0x8e, 0x81, 0x94, 0xd3, 0x98, 0xc0, 0xde, 0x54, 0x9b, 0xd3, 0x73, 0xa3, 0xd3, 0x07, + 0x6b, 0x7a, 0xae, 0xa2, 0xdf, 0x56, 0xb0, 0x35, 0x3d, 0x47, 0x07, 0x50, 0x7f, 0x47, 0xd6, 0x2a, + 0xd7, 0xc2, 0x27, 0xad, 0x49, 0x03, 0x6a, 0x3f, 0xd2, 0xd4, 0xb9, 0x80, 0xae, 0xe1, 0x50, 0xb2, + 0xbd, 0x0d, 0x5e, 0xa2, 0x9f, 0x40, 0xcb, 0x68, 0xde, 0xb9, 0xad, 0x0c, 0x97, 0x05, 0x38, 0x7f, + 0x5a, 0xb0, 0x6f, 0xdc, 0x17, 0x5e, 0x9c, 0x1d, 0xfe, 0x5b, 0xa8, 0x87, 0x64, 0x49, 0x25, 0x6d, + 0x67, 0xfc, 0x85, 0xec, 0x02, 0x5b, 0xe2, 0xdc, 0x9f, 0xc9, 0xd2, 0x94, 0x09, 0x96, 0x08, 0xf4, + 0x0d, 0x34, 0x6f, 0x3c, 0x9f, 0xd3, 0x48, 0x8b, 0x7f, 0x76, 0x2f, 0xf6, 0x8d, 0x0c, 0xc3, 0x3a, + 0xdc, 0x7e, 0x09, 0x9d, 0x1c, 0x1b, 0x3a, 0x84, 0x26, 0xbb, 0xb9, 0x89, 0x29, 0x97, 0x7b, 0x68, + 0x60, 0x6d, 0xa1, 0x03, 0x68, 0xf8, 0xde, 0xda, 0xe3, 0x92, 0xbe, 0x81, 0x95, 0x61, 0xbf, 0x80, + 0xa6, 0xa2, 0x43, 0x23, 0x68, 0xc4, 0x5c, 0x54, 0xaa, 0x6a, 0x60, 0xf6, 0x9d, 0xb3, 0x5f, 0x8a, + 0x35, 0x57, 0xfe, 0x62, 0x15, 0xe8, 0xfc, 0x65, 0x6d, 0xae, 0x45, 0xed, 0x4f, 0xa4, 0xf5, 0xab, + 0x42, 0x06, 0x8e, 0xb7, 0x9c, 0x22, 0xf4, 0x53, 0x7d, 0xfe, 0xd0, 0x4f, 0xf5, 0xe9, 0xbf, 0x87, + 0x86, 0xc7, 0xe9, 0x3a, 0x1e, 0x58, 0xc7, 0xb5, 0x93, 0xce, 0xf8, 0xd1, 0x3d, 0xb0, 0xbc, 0x67, + 0xca, 0xe9, 0x1a, 0x2b, 0x94, 0xfd, 0x0b, 0xb4, 0x33, 0x46, 0x71, 0x52, 0xce, 0x38, 0xf1, 0x75, + 0x02, 0x94, 0x21, 0xde, 0xed, 0x3c, 0x89, 0x22, 0x1a, 0x98, 0x0c, 0x18, 0x73, 0x93, 0x99, 0x5a, + 0x3e, 0x33, 0x33, 0xe8, 0xdf, 0x55, 0x13, 0x25, 0x93, 0x15, 0xa8, 0xe5, 0x2d, 0x36, 0x39, 0xb3, + 0xca, 0xe6, 0xec, 0x1c, 0x50, 0xc6, 0xca, 0x96, 0xe6, 0xce, 0x5c, 0x68, 0xce, 0x48, 0xb4, 0xd4, + 0x77, 0xd6, 0x19, 0x1f, 0x16, 0x8e, 0x9f, 0x95, 0x3c, 0xd6, 0x51, 0xce, 0x97, 0xb9, 0xbd, 0x09, + 0x16, 0x71, 0x6a, 0x04, 0xf5, 0x0b, 0x2f, 0x10, 0x79, 0xaf, 0x9d, 0xb4, 0xb1, 0xfc, 0x76, 0xde, + 0xc0, 0x81, 0x89, 0xfb, 0xe1, 0x96, 0x06, 0xfc, 0xff, 0xea, 0xfd, 0x53, 0xdd, 0x6c, 0x5b, 0x13, + 0x09, 0xc9, 0xd7, 0x00, 0x54, 0x58, 0x57, 0xb2, 0x7d, 0xaa, 0xba, 0x79, 0x58, 0xa0, 0xda, 0x04, + 0xbb, 0xf2, 0x73, 0x96, 0x86, 0x14, 0xe7, 0x40, 0xe8, 0x6b, 0x68, 0x29, 0x8b, 0x70, 0x5d, 0xf7, + 0xb6, 0xbb, 0x64, 0x6c, 0xe9, 0x53, 0xd7, 0xcc, 0x71, 0x77, 0x66, 0xc6, 0x36, 0xce, 0x62, 0xd1, + 0xa9, 0xc9, 0x7c, 0x4d, 0x82, 0x0e, 0xb6, 0x65, 0xfe, 0x6d, 0xc5, 0x64, 0xfd, 0x31, 0xb4, 0x33, + 0x79, 0xd4, 0xd5, 0xc6, 0x3b, 0x16, 0xd0, 0x7e, 0x05, 0xf5, 0x00, 0xa4, 0x29, 0x21, 0xfd, 0xea, + 0xa4, 0x09, 0xf5, 0x09, 0x5b, 0xa4, 0xe3, 0x09, 0xec, 0x1a, 0x36, 0x3d, 0x0a, 0xd0, 0x10, 0x9a, + 0xaf, 0x39, 0x27, 0xf3, 0x15, 0xea, 0xe5, 0xc7, 0xfb, 0x34, 0xb0, 0x77, 0xef, 0x8c, 0x7b, 0xa7, + 0x72, 0x52, 0x1d, 0x55, 0xc7, 0x7f, 0xd7, 0xa1, 0x65, 0x48, 0xd0, 0x08, 0x9a, 0xaa, 0x77, 0x23, + 0xa4, 0xa2, 0xf3, 0xb3, 0xc2, 0xee, 0x17, 0x7c, 0xa1, 0x9f, 0x3a, 0x15, 0xf4, 0x14, 0x1a, 0xb2, + 0x15, 0xa3, 0x3d, 0xb9, 0x98, 0x6f, 0xe4, 0x5a, 0x71, 0xd3, 0xa9, 0x9d, 0x0a, 0x7a, 0x0c, 0x35, + 0x9c, 0x04, 0x5b, 0xd9, 0xbb, 0xd2, 0x67, 0xfe, 0x3f, 0xc8, 0xd8, 0xba, 0x18, 0xf7, 0xa8, 0xaf, + 0x69, 0xb2, 0xbf, 0x0a, 0x76, 0x2f, 0xe7, 0x51, 0xb1, 0xcf, 0xe1, 0x81, 0x9e, 0xdb, 0x68, 0x5f, + 0x2e, 0x16, 0x27, 0xbf, 0xbd, 0x57, 0x74, 0x2a, 0xd0, 0x08, 0x9a, 0xaa, 0xc7, 0xeb, 0xfd, 0x14, + 0x66, 0x84, 0x3e, 0x6d, 0x6e, 0x08, 0x28, 0x19, 0x93, 0xe8, 0xfd, 0x7c, 0x3a, 0x8b, 0x32, 0xf9, + 0xd9, 0xee, 0x54, 0x44, 0xb7, 0xb9, 0x5c, 0xb1, 0x3f, 0xd0, 0x3d, 0x15, 0x6c, 0xa3, 0x82, 0xdf, + 0xa0, 0x5e, 0x88, 0xb7, 0x12, 0x73, 0x34, 0xb8, 0xaf, 0xc7, 0xda, 0x87, 0xdb, 0x1b, 0x90, 0x53, + 0x41, 0xdf, 0x41, 0xed, 0x82, 0x2d, 0xd1, 0x51, 0x31, 0x20, 0x7b, 0xcb, 0xf6, 0x47, 0x1f, 0x2e, + 0x48, 0xe0, 0xa8, 0x8a, 0x5e, 0x41, 0x43, 0x96, 0x1a, 0xfa, 0x78, 0xdb, 0x23, 0x51, 0xf0, 0xa3, + 0x7b, 0xde, 0x8f, 0x20, 0x98, 0x7c, 0xfa, 0xeb, 0x27, 0xb9, 0xff, 0xbb, 0xe4, 0x7d, 0xbc, 0x1a, + 0xb2, 0x90, 0x06, 0xb7, 0x8b, 0xf9, 0x90, 0x84, 0xde, 0x75, 0x53, 0x3e, 0x99, 0xe7, 0xff, 0x06, + 0x00, 0x00, 0xff, 0xff, 0xb4, 0xcf, 0x4d, 0xf9, 0x40, 0x0b, 0x00, 0x00, } diff --git a/ci/citest/acceptance-test/multibox/10.0.100.13-vdc-executor-null/guestroot/etc/mesos-slave/attributes b/ci/citest/acceptance-test/multibox/10.0.100.13-vdc-executor-null/guestroot/etc/mesos-slave/attributes index 868b31a4..9e69cd27 100644 --- a/ci/citest/acceptance-test/multibox/10.0.100.13-vdc-executor-null/guestroot/etc/mesos-slave/attributes +++ b/ci/citest/acceptance-test/multibox/10.0.100.13-vdc-executor-null/guestroot/etc/mesos-slave/attributes @@ -1 +1 @@ -hypervisor:null +hypervisor:null;openvdc-node-id:null1 diff --git a/ci/citest/acceptance-test/multibox/10.0.100.13-vdc-executor-null/guestroot/etc/openvdc/executor.toml b/ci/citest/acceptance-test/multibox/10.0.100.13-vdc-executor-null/guestroot/etc/openvdc/executor.toml index ae87aa9c..575ceae9 100644 --- a/ci/citest/acceptance-test/multibox/10.0.100.13-vdc-executor-null/guestroot/etc/openvdc/executor.toml +++ b/ci/citest/acceptance-test/multibox/10.0.100.13-vdc-executor-null/guestroot/etc/openvdc/executor.toml @@ -1,5 +1,3 @@ -[hypervisor] -driver = "null" [zookeeper] endpoint = "zk://10.0.100.10:2181,10.0.100.11:2181,10.0.100.12:2181/openvdc" diff --git a/ci/citest/acceptance-test/multibox/10.0.100.14-vdc-executor-lxc/guestroot/etc/mesos-slave/attributes b/ci/citest/acceptance-test/multibox/10.0.100.14-vdc-executor-lxc/guestroot/etc/mesos-slave/attributes index e4ed3d13..4d502262 100644 --- a/ci/citest/acceptance-test/multibox/10.0.100.14-vdc-executor-lxc/guestroot/etc/mesos-slave/attributes +++ b/ci/citest/acceptance-test/multibox/10.0.100.14-vdc-executor-lxc/guestroot/etc/mesos-slave/attributes @@ -1 +1 @@ -hypervisor:lxc;node-groups:linuxbr +hypervisor:lxc;openvdc-node-id:lxc1;node-groups:linuxbr diff --git a/ci/citest/acceptance-test/multibox/10.0.100.14-vdc-executor-lxc/guestroot/etc/openvdc/executor.toml b/ci/citest/acceptance-test/multibox/10.0.100.14-vdc-executor-lxc/guestroot/etc/openvdc/executor.toml index 3528e757..2457e5e2 100644 --- a/ci/citest/acceptance-test/multibox/10.0.100.14-vdc-executor-lxc/guestroot/etc/openvdc/executor.toml +++ b/ci/citest/acceptance-test/multibox/10.0.100.14-vdc-executor-lxc/guestroot/etc/openvdc/executor.toml @@ -1,5 +1,4 @@ [hypervisor] -driver = "lxc" script-path = "/etc/openvdc/scripts/" image-server-uri = "http://10.0.100.12/images" cache-path = "/var/cache/lxc/" diff --git a/ci/citest/acceptance-test/multibox/10.0.100.15-vdc-executor-lxc-ovs/guestroot/etc/mesos-slave/attributes b/ci/citest/acceptance-test/multibox/10.0.100.15-vdc-executor-lxc-ovs/guestroot/etc/mesos-slave/attributes index 5c6bfa2e..63be12ff 100644 --- a/ci/citest/acceptance-test/multibox/10.0.100.15-vdc-executor-lxc-ovs/guestroot/etc/mesos-slave/attributes +++ b/ci/citest/acceptance-test/multibox/10.0.100.15-vdc-executor-lxc-ovs/guestroot/etc/mesos-slave/attributes @@ -1 +1 @@ -hypervisor:lxc;node-groups:ovs +hypervisor:lxc;openvdc-node-id:lxc2;node-groups:ovs diff --git a/ci/citest/acceptance-test/tests/crash_recovery_test.go b/ci/citest/acceptance-test/tests/crash_recovery_test.go new file mode 100644 index 00000000..63a7fc47 --- /dev/null +++ b/ci/citest/acceptance-test/tests/crash_recovery_test.go @@ -0,0 +1,35 @@ +// +build acceptance + +package tests + +import ( + "strings" + "testing" + "time" +) + +func TestCrashRecovery(t *testing.T) { + stdout, _ := RunCmdAndReportFail(t, "openvdc", "run", "centos/7/lxc") + instance_id := strings.TrimSpace(stdout.String()) + + WaitInstance(t, 5*time.Minute, instance_id, "RUNNING", []string{"QUEUED", "STARTING"}) + RunSshWithTimeoutAndReportFail(t, executor_lxc_ip, "sudo lxc-info -n "+instance_id, 10, 5) + + stdout, _ = RunCmdAndReportFail(t, "openvdc", "stop", instance_id) + WaitInstance(t, 5*time.Minute, instance_id, "STOPPED", []string{"RUNNING", "STOPPING"}) + + //Simulate crash + RunSshWithTimeoutAndReportFail(t, executor_lxc_ip, "sudo systemctl stop mesos-slave", 10, 5) + + //Give scheduler some time to register crash + time.Sleep(3 * time.Minute) + + RunSshWithTimeoutAndReportFail(t, executor_lxc_ip, "sudo systemctl start mesos-slave", 10, 5) + time.Sleep(1 * time.Minute) + + //stdout, _ = RunCmdAndReportFail(t, "openvdc", "stop", instance_id) + //WaitInstance(t, 5*time.Minute, instance_id, "STOPPED", []string{"RUNNING", "STOPPING"}) + + stdout, _ = RunCmdWithTimeoutAndReportFail(t, 10, 5, "openvdc", "destroy", instance_id) + WaitInstance(t, 5*time.Minute, instance_id, "TERMINATED", nil) +} diff --git a/ci/devbox/devbox-centos7.json b/ci/devbox/devbox-centos7.json index 72135431..32aa5532 100644 --- a/ci/devbox/devbox-centos7.json +++ b/ci/devbox/devbox-centos7.json @@ -89,7 +89,7 @@ "systemctl enable zookeeper", "systemctl enable mesos-master", "systemctl enable mesos-slave", - "echo 'hypervisor:lxc' > /etc/mesos-slave/attributes", + "echo 'hypervisor:lxc;openvdc-node-id:lxc1' > /etc/mesos-slave/attributes", "echo 'false' > /etc/mesos-slave/switch_user", "echo '{\"PATH\":\"/home/vagrant/go/src/github.com/axsh/openvdc:/usr/libexec/mesos:/usr/bin:/usr/sbin:/usr/local/bin\"}' > /etc/mesos-slave/executor_environment_variables", "#firewall-cmd --permanent --zone=public --add-port=5050/tcp", diff --git a/cmd/lxc-openvdc/init_linux.go b/cmd/lxc-openvdc/init_linux.go new file mode 100644 index 00000000..d41d746f --- /dev/null +++ b/cmd/lxc-openvdc/init_linux.go @@ -0,0 +1,15 @@ +package main + +import ( + "log/syslog" + "github.com/Sirupsen/logrus" + logrus_syslog "github.com/Sirupsen/logrus/hooks/syslog" +) + +func init() { + hook, err := logrus_syslog.NewSyslogHook("", "", syslog.LOG_DEBUG, "lxc-openvdc") + if err != nil { + logrus.Fatal("Failed to initialize syslog hook: ", err) + } + logrus.AddHook(hook) +} diff --git a/cmd/lxc-openvdc/main.go b/cmd/lxc-openvdc/main.go index 0f96c5a3..1667d814 100644 --- a/cmd/lxc-openvdc/main.go +++ b/cmd/lxc-openvdc/main.go @@ -5,7 +5,6 @@ import ( "fmt" "io" "io/ioutil" - "log/syslog" "net/http" "net/url" "os" @@ -14,7 +13,6 @@ import ( "strings" log "github.com/Sirupsen/logrus" - logrus_syslog "github.com/Sirupsen/logrus/hooks/syslog" "github.com/mholt/archiver" "github.com/pkg/errors" ) @@ -29,14 +27,6 @@ var dist string var release string var arch string -func init() { - hook, err := logrus_syslog.NewSyslogHook("", "", syslog.LOG_DEBUG, "lxc-openvdc") - if err != nil { - log.Fatal("Failed to initialize syslog hook: ", err) - } - log.AddHook(hook) -} - func main() { _dist := flag.String("dist", "centos", "Name of the distribution") diff --git a/cmd/openvdc-executor/main.go b/cmd/openvdc-executor/main.go index 7ff34ccd..9eb63f46 100644 --- a/cmd/openvdc-executor/main.go +++ b/cmd/openvdc-executor/main.go @@ -29,20 +29,53 @@ type VDCExecutor struct { nodeInfo *model.ExecutorNode } -func newVDCExecutor(ctx context.Context, provider hypervisor.HypervisorProvider, node *model.ExecutorNode) *VDCExecutor { +func newVDCExecutor(ctx context.Context, node *model.ExecutorNode) *VDCExecutor { return &VDCExecutor{ - hypervisorProvider: provider, - ctx: ctx, - nodeInfo: node, + ctx: ctx, + nodeInfo: node, } } +func (exec *VDCExecutor) GetHypervisorProvider() hypervisor.HypervisorProvider { + return exec.hypervisorProvider +} + func (exec *VDCExecutor) Registered(driver exec.ExecutorDriver, execInfo *mesos.ExecutorInfo, fwinfo *mesos.FrameworkInfo, slaveInfo *mesos.SlaveInfo) { - log.Infoln("Registered Executor on slave ", slaveInfo.GetHostname()) + log.Infoln("Registering Executor on slave ", slaveInfo.GetHostname()) + // Read and validate passed slave attributes: /etc/mesos-slave/attributes or --attributes + for _, attr := range slaveInfo.Attributes { + switch attr.GetName() { + case "hypervisor": + var ok bool + exec.hypervisorProvider, ok = hypervisor.FindProvider(attr.GetText().GetValue()) + if !ok { + log.Errorf("Unknown hypervisor driver: %s", attr.GetText().GetValue()) + } + case "openvdc-node-id": + exec.nodeInfo.NodeId = attr.GetText().GetValue() + } + } + if exec.nodeInfo.NodeId == "" { + _, err := driver.Abort() + log.WithError(err).Error("Failed to find 'openvdc-node-id' attribute") + return + } + if exec.hypervisorProvider == nil { + _, err := driver.Abort() + log.WithError(err).Error("Failed to find 'hypervisor' attribute") + return + } + if err := exec.hypervisorProvider.LoadConfig(viper.GetViper()); err != nil { + log.WithError(err).Error("Failed hypervisorProvider.LoadConfig") + driver.Abort() + return + } + exec.nodeInfo.Id = slaveInfo.GetId().GetValue() err := model.Cluster(exec.ctx).Register(exec.nodeInfo) if err != nil { - log.Error(err) + driver.Abort() + log.WithError(err).Error("Failed OpenVDC cluster registration") return } log.Infoln("Registered on OpenVDC cluster service: ", exec.nodeInfo) @@ -59,6 +92,17 @@ func (exec *VDCExecutor) Disconnected(driver exec.ExecutorDriver) { func (exec *VDCExecutor) LaunchTask(driver exec.ExecutorDriver, taskInfo *mesos.TaskInfo) { log.Infoln("Launching task", taskInfo.GetName(), "with command", taskInfo.Command.GetValue()) + var lastErr error + defer func() { + if lastErr == nil { + return + } + driver.SendStatusUpdate(&mesos.TaskStatus{ + TaskId: taskInfo.GetTaskId(), + State: mesos.TaskState_TASK_FAILED.Enum(), + Message: proto.String(lastErr.Error()), + }) + }() _, err := driver.SendStatusUpdate(&mesos.TaskStatus{ TaskId: taskInfo.GetTaskId(), State: mesos.TaskState_TASK_STARTING.Enum(), @@ -67,16 +111,40 @@ func (exec *VDCExecutor) LaunchTask(driver exec.ExecutorDriver, taskInfo *mesos. log.WithError(err).Errorln("Couldn't send status update") return } - if err := exec.bootInstance(driver, taskInfo); err != nil { + + instanceID := taskInfo.GetTaskId().GetValue() + + var ctx context.Context + ctx, lastErr = model.Connect(context.Background(), &zkAddr) + if lastErr != nil { + log.WithError(lastErr).Error("Failed model.Connect") + return + } + var instance *model.Instance + instance, lastErr = model.Instances(ctx).FindByID(instanceID) + if lastErr != nil { + log.WithError(lastErr).Error("Failed to fetch instance %s", instanceID) return } + instanceState := instance.GetLastState() + + if instanceState.State == model.InstanceState_QUEUED { + if err := exec.bootInstance(driver, taskInfo); err != nil { + // bootInstance() handles driver.SendStatusUpdate by itself. + return + } + } else { + if err := exec.recoverInstance(instance); err != nil { + log.WithError(err).Error("Failed recoverInstance") + } + } - _, err = driver.SendStatusUpdate(&mesos.TaskStatus{ + _, err1 := driver.SendStatusUpdate(&mesos.TaskStatus{ TaskId: taskInfo.GetTaskId(), State: mesos.TaskState_TASK_RUNNING.Enum(), }) - if err != nil { - log.WithError(err).Errorln("Couldn't send status update") + if err1 != nil { + log.WithError(err1).Errorln("Couldn't send status update") return } } @@ -108,6 +176,22 @@ func recordFailedState(ctx context.Context, driver exec.ExecutorDriver, instance return nil } +func (exec *VDCExecutor) recoverInstance(instance *model.Instance) error { + provider := exec.GetHypervisorProvider() + if provider != nil { + return errors.New("HypervisorProvider is not ready") + } + hv, err := provider.CreateDriver(instance, instance.ResourceTemplate()) + if err != nil { + return errors.Wrapf(err, "Hypervisorprovider failed to create driver. InstanceID: %s", instance.GetId()) + } + err = hv.Recover(*instance.LastState) + if err != nil { + return errors.Wrapf(err, "Hypervisor failed to recover instance. InstanceID: %s", instance.GetId()) + } + return nil +} + func (exec *VDCExecutor) bootInstance(driver exec.ExecutorDriver, taskInfo *mesos.TaskInfo) error { instanceID := taskInfo.GetTaskId().GetValue() log := log.WithFields(log.Fields{ @@ -165,6 +249,13 @@ func (exec *VDCExecutor) bootInstance(driver exec.ExecutorDriver, taskInfo *meso log.Infof("Instance launched successfully") // Here can bring the instance state to RUNNING finally. finState = model.InstanceState_RUNNING + + err = model.Instances(ctx).UpdateConnectionStatus(instanceID, model.ConnectionStatus_CONNECTED) + + if err != nil { + return errors.Wrapf(err, "Couldn't update instance connectionStatus. instanceID: %s connectionStatus: %s", instanceID, model.ConnectionStatus_CONNECTED) + } + return nil } @@ -217,6 +308,13 @@ func (exec *VDCExecutor) startInstance(driver exec.ExecutorDriver, instanceID st log.Infof("Instance started successfully") // Here can bring the instance state to RUNNING finally. finState = model.InstanceState_RUNNING + + err = model.Instances(ctx).UpdateConnectionStatus(instanceID, model.ConnectionStatus_CONNECTED) + + if err != nil { + return errors.Wrapf(err, "Couldn't update instance connectionStatus. instanceID: %s connectionStatus: %s", instanceID, model.ConnectionStatus_CONNECTED) + } + return nil } @@ -463,7 +561,6 @@ func startExecutorAPIServer(ctx context.Context, listener net.Listener) *executo } func init() { - viper.SetDefault("hypervisor.driver", "null") viper.SetDefault("zookeeper.endpoint", "zk://localhost/openvdc") viper.SetDefault("executor-api.listen", "0.0.0.0:"+defaultExecutorAPIPort) viper.SetDefault("executor-api.advertise-ip", "") @@ -473,8 +570,6 @@ func init() { cobra.OnInitialize(initConfig) pfs := rootCmd.PersistentFlags() pfs.String("config", DefaultConfPath, "Load config file from the path") - pfs.String("hypervisor", viper.GetString("hypervisor.driver"), "Hypervisor driver name") - viper.BindPFlag("hypervisor.driver", pfs.Lookup("hypervisor")) pfs.String("zk", viper.GetString("zookeeper.endpoint"), "Zookeeper address") viper.BindPFlag("zookeeper.endpoint", pfs.Lookup("zk")) } @@ -510,15 +605,6 @@ func execute(cmd *cobra.Command, args []string) { log.WithError(err).Fatal("Invalid zookeeper endpoint: ", viper.GetString("zookeeper.endpoint")) } - provider, ok := hypervisor.FindProvider(viper.GetString("hypervisor.driver")) - if ok == false { - log.Fatalln("Unknown hypervisor name:", viper.GetString("hypervisor.driver")) - } - if err := provider.LoadConfig(viper.GetViper()); err != nil { - log.WithError(err).Fatal("Failed to apply hypervisor configuration") - } - log.Infof("Initializing executor: hypervisor %s\n", provider.Name()) - ctx, err := model.ClusterConnect(context.Background(), &zkAddr) if err != nil { log.WithError(err).Fatalf("Failed to connect to cluster service %s", zkAddr.String()) @@ -543,9 +629,6 @@ func execute(cmd *cobra.Command, args []string) { if err != nil { log.WithError(err).Fatalln("Faild to bind address for Executor gRPC API: ", viper.GetString("executor-api.listen")) } - s := startExecutorAPIServer(ctx, executorAPIListener) - defer s.GracefulStop() - log.Infof("Listening Executor gRPC API on %s", executorAPIListener.Addr().String()) exposedExecutorAPIAddr := executorAPIListener.Addr().String() if viper.GetString("executor-api.advertise-ip") != "" { _, port, err := net.SplitHostPort(exposedExecutorAPIAddr) @@ -553,21 +636,18 @@ func execute(cmd *cobra.Command, args []string) { log.WithError(err).Fatal("Failed to parse host:port: ", exposedExecutorAPIAddr) } exposedExecutorAPIAddr = net.JoinHostPort(viper.GetString("executor-api.advertise-ip"), port) - log.Infof("Exposed Executor gRPC API on %s", exposedExecutorAPIAddr) } + s := startExecutorAPIServer(ctx, executorAPIListener) + defer s.GracefulStop() + log.Infof("Listening Executor gRPC API on %s", executorAPIListener.Addr().String()) + log.Infof("Exposed Executor gRPC API on %s", exposedExecutorAPIAddr) + sshListener, err := net.Listen("tcp", viper.GetString("console.ssh.listen")) if err != nil { log.WithError(err).Fatalf("Failed to listen SSH on %s", sshListener.Addr().String()) } defer sshListener.Close() - - sshd := NewSSHServer(provider, ctx) - if err := sshd.Setup(); err != nil { - log.WithError(err).Fatal("Failed to setup SSH Server") - } - go sshd.Run(sshListener) - log.Infof("Listening SSH on %s", sshListener.Addr().String()) exposedSSHAddr := sshListener.Addr().String() if viper.GetString("console.ssh.advertise-ip") != "" { _, port, err := net.SplitHostPort(exposedSSHAddr) @@ -575,18 +655,27 @@ func execute(cmd *cobra.Command, args []string) { log.WithError(err).Fatal("Failed to parse host:port: ", exposedSSHAddr) } exposedSSHAddr = net.JoinHostPort(viper.GetString("console.ssh.advertise-ip"), port) - log.Infof("Exposed SSH on %s", exposedSSHAddr) } - node := &model.ExecutorNode{ + clusterNode := &model.ExecutorNode{ GrpcAddr: exposedExecutorAPIAddr, Console: &model.Console{ Type: model.Console_SSH, BindAddr: exposedSSHAddr, }, } + vdcExecutor := newVDCExecutor(ctx, clusterNode) + + sshd := NewSSHServer(vdcExecutor, ctx) + if err := sshd.Setup(); err != nil { + log.WithError(err).Fatal("Failed to setup SSH Server") + } + go sshd.Run(sshListener) + log.Infof("Listening SSH on %s", sshListener.Addr().String()) + log.Infof("Exposed SSH on %s", exposedSSHAddr) + dconfig := exec.DriverConfig{ - Executor: newVDCExecutor(ctx, provider, node), + Executor: vdcExecutor, } driver, err := exec.NewMesosExecutorDriver(dconfig) if err != nil { diff --git a/cmd/openvdc-executor/sshd.go b/cmd/openvdc-executor/sshd.go index 082ae1a8..c344a946 100644 --- a/cmd/openvdc-executor/sshd.go +++ b/cmd/openvdc-executor/sshd.go @@ -14,22 +14,26 @@ import ( "golang.org/x/net/context" ) +type HypervisorProviderFinder interface { + GetHypervisorProvider() hypervisor.HypervisorProvider +} + type SSHServer struct { config *ssh.ServerConfig listener net.Listener - provider hypervisor.HypervisorProvider + finder HypervisorProviderFinder ctx context.Context } -func NewSSHServer(provider hypervisor.HypervisorProvider, ctx context.Context) *SSHServer { +func NewSSHServer(finder HypervisorProviderFinder, ctx context.Context) *SSHServer { config := &ssh.ServerConfig{ NoClientAuth: true, } return &SSHServer{ - config: config, - provider: provider, - ctx: ctx, + config: config, + finder: finder, + ctx: ctx, } } @@ -61,6 +65,7 @@ func (sshd *SSHServer) Setup() error { } sshd.config.AddHostKey(sshSigner) } + return nil } @@ -89,17 +94,23 @@ func (sshd *SSHServer) Run(listener net.Listener) { return } - hv, err := sshd.provider.CreateDriver(inst, inst.ResourceTemplate()) - if err != nil { - log.WithError(err).Errorf("Failed HypervisorProvider.CreateDriver: %T", sshd.provider) - sshConn.Close() - return - } for newChannel := range chans { if t := newChannel.ChannelType(); t != "session" { newChannel.Reject(ssh.UnknownChannelType, fmt.Sprintf("unknown channel type: %s", t)) continue } + provider := sshd.finder.GetHypervisorProvider() + if provider == nil { + log.Error("HypervisorProvider is not ready") + newChannel.Reject(ssh.Prohibited, "HypervisorProvider is not ready") + continue + } + hv, err := provider.CreateDriver(inst, inst.ResourceTemplate()) + if err != nil { + log.WithError(err).Errorf("Failed HypervisorProvider.CreateDriver: %T", provider) + sshConn.Close() + return + } session := sshSession{ instanceID: instanceID, sshd: sshd, diff --git a/cmd/openvdc-scheduler/main.go b/cmd/openvdc-scheduler/main.go index 000c97aa..7cbe68c4 100644 --- a/cmd/openvdc-scheduler/main.go +++ b/cmd/openvdc-scheduler/main.go @@ -72,7 +72,7 @@ func init() { pfs.Float64("failover-timeout", viper.GetFloat64("scheduler.failover-timeout"), "Failover timeout") viper.BindPFlag("scheduler.failover-timeout", pfs.Lookup("failover-timeout")) - pfs.Float64("executor-path", viper.GetFloat64("scheduler.executor-path"), "Executor path") + pfs.String("executor-path", viper.GetString("scheduler.executor-path"), "Executor path") viper.BindPFlag("scheduler.executor-path", pfs.Lookup("executor-path")) } diff --git a/cmd/openvdc/cmd/create.go b/cmd/openvdc/cmd/create.go index 83072be2..2cc92eb5 100644 --- a/cmd/openvdc/cmd/create.go +++ b/cmd/openvdc/cmd/create.go @@ -17,9 +17,10 @@ var hostName string func init() { createCmd.PersistentFlags().StringVarP(&hostName, "name", "n", "", "Existing host name") + createCmd.Flags().Bool("auto-recovery", true, "Set auto recovery flag") } -func prepareCreateAPICall(templateSlug string, args []string) *api.CreateRequest { +func prepareCreateAPICall(templateSlug string, args []string, flags *pflag.FlagSet) *api.CreateRequest { rt, err := util.FetchTemplate(templateSlug) if err != nil { switch err { @@ -32,8 +33,10 @@ func prepareCreateAPICall(templateSlug string, args []string) *api.CreateRequest if len(args) > 1 { rt.Template.Template = util.MergeTemplateParams(rt, args[1:]) } + autoRecovery, _ := flags.GetBool("auto-recovery") req := &api.CreateRequest{ - Template: rt.ToModel(), + Template: rt.ToModel(), + AutoRecovery: autoRecovery, } log.Printf("Found template: %s", templateSlug) return req @@ -56,7 +59,7 @@ var createCmd = &cobra.Command{ } templateSlug := args[0] - req := prepareCreateAPICall(templateSlug, args) + req := prepareCreateAPICall(templateSlug, args, cmd.Flags()) return util.RemoteCall(func(conn *grpc.ClientConn) error { c := api.NewInstanceClient(conn) res, err := c.Create(context.Background(), req) diff --git a/cmd/openvdc/cmd/run.go b/cmd/openvdc/cmd/run.go index e5121be3..c2142757 100644 --- a/cmd/openvdc/cmd/run.go +++ b/cmd/openvdc/cmd/run.go @@ -12,6 +12,10 @@ import ( "google.golang.org/grpc" ) +func init() { + runCmd.Flags().Bool("auto-recovery", true, "Set auto recovery flag") +} + var runCmd = &cobra.Command{ Use: "run [ResourceTemplate ID/URI]", Short: "Run an instance", @@ -45,7 +49,7 @@ var runCmd = &cobra.Command{ break } } - req := prepareCreateAPICall(templateSlug, left) + req := prepareCreateAPICall(templateSlug, left, cmd.Flags()) return util.RemoteCall(func(conn *grpc.ClientConn) error { c := api.NewInstanceClient(conn) res, err := c.Run(context.Background(), req) diff --git a/docs/diagrams/crash-recovery.png b/docs/diagrams/crash-recovery.png new file mode 100644 index 00000000..c7b10036 Binary files /dev/null and b/docs/diagrams/crash-recovery.png differ diff --git a/docs/diagrams/crash-recovery.svg b/docs/diagrams/crash-recovery.svg new file mode 100644 index 00000000..9469c69b --- /dev/null +++ b/docs/diagrams/crash-recovery.svg @@ -0,0 +1,3213 @@ + + + + + + image/svg+xml + + + + + + + + + + + + + + + Slave machine dies. + + + Slave machine dies. + + + + + + + + Add as crashed node. + + + Add as crashed node. + + + + + + + + ZK + + + ZK + + + + + + + + Instances + + + Instances + + + + + + + + Nodes + + + Nodes + + + + + + + + Scheduler + + + Scheduler + + + + + + + + + + 1. + + + 1. + + + + + + + 8. + + + 8. + + + + + + + Agent dies. + + + Agent dies. + + + + + + + + Crashed Nodes + + + Crashed Nodes + + + + + + + + + + + + + CONNECTED + + + CONNECTED + + + + + + + CONNECTED + + + CONNECTED + + + + + + + CONNECTED + + + CONNECTED + + + + + + + + Slave 1 + + + Slave 1 + + + + + + + + STOPPED + + + STOPPED + + + + + + + + STOPPED + + + STOPPED + + + + + + + + STOPPED + + + STOPPED + + + + + + + RUNNING + + + RUNNING + + + + + + + RUNNING + + + RUNNING + + + + + + + RUNNING + + + RUNNING + + + + + + + + + + 2. + + + 2. + + + + + + + + + 3. + + + 3. + + + + + + + + Slave sends resource offer + + + Slave sends resource offer + + + + + + + Update IDs, ConnectionStatuses etc. + + + Update IDs, ConnectionStatuses etc. + + + + + + + Use ID to check if agent exists as a crashed node. + + + Use ID to check if agent exists as a crashed node. + + + + + + + + ZK + + + ZK + + + + + + + + Instances + + + Instances + + + + + + + + Nodes + + + Nodes + + + + + + + + Scheduler + + + Scheduler + + + + + + + + + + 1. + + + 1. + + + + + + + 8. + + + 8. + + + + + + + 3. + + + 3. + + + + + + + Agent reconnects. + + + Agent reconnects. + + + + + + + + Crashed Nodes + + + Crashed Nodes + + + + + + + + + + + + + + + + Get instances assigned to that ID and set them to NOT_CONNECTED + + + Get instances assigned to that ID and set them to NOT_CONNECTED + + + + + + + 4. + + + 4. + + + + + + + + NOT_CONNECTED + + + NOT_CONNECTED + + + + + + + NOT_CONNECTED + + + NOT_CONNECTED + + + + + + + NOT_CONNECTED + + + NOT_CONNECTED + + + + + + + 5. + + + 5. + + + + + + + + + + + + + Relaunch queue + + + Relaunch queue + + + + + + + + + Slave 1 + + + Slave 1 + + + + + + + + + + + + + EXEC + + + EXEC + + + + + + + + + EXEC + + + EXEC + + + + + + + + + EXEC + + + EXEC + + + + + + + + + + + + + + + + + + + + + RUNNING + + + RUNNING + + + + + + + + RUNNING + + + RUNNING + + + + + + + + STOPPED + + + STOPPED + + + + + + + + + + + 7. + + + 7. + + + + + + + + + RUNNING + + + RUNNING + + + + + + + RUNNING + + + RUNNING + + + + + + + RUNNING + + + RUNNING + + + + + + + + + Glorious success. + + + Glorious success. + + + + + + + 8. + + + 8. + + + + + + + + 2. + + + 2. + + + + + + + Scheduler receives offer containing Agent IDs + + + Scheduler receives offer containing Agent IDs + + + + + + + + 3. + + + 3. + + + + + + + + 4. + + + 4. + + + + + + + + 5. + + + 5. + + + + + + + + 6. + + + 6. + + + + + + + + 7. + + + 7. + + + + + + + + 10. + + + 9. + + + + + + + + 9. + + + 8. + + + + + + + + 4. + + + 4. + + + + + + + + Check which unique ID matches with the recieved temporary one. + Scheduler recieves SlaveLost-call containing temporary Agent ID. + R.I.P.AgentID: cae491b6-2fed-45ab-b814-15d927fb9839-S0 + openvdc-node-id: hello-1AgentID: cae491b6-2fed-45ab-b814-15d927fb9839-S0 + temporary id:cae491b6-2fed-45ab-b814-15d927fb9839-S0 + openvdc-node-id:hello-1 + hello-1cae491b6-2fed-45ab-b814-15d927fb9839-S0 + hello-1cae491b6-2fed-45ab-b814-15d927fb9839-S0reconnected = falsecreated_at = current timestampreconnected_at = + + i-00000001openvdc-node-id: hello-1 + i-00000002openvdc-node-id: hello-1 + i-00000003openvdc-node-id: hello-1 + hello-1cae491b6-2fed-45ab-b814-15d927fb9839-S0 + hello-1cae491b6-2fed-45ab-b814-15d927fb9839-S0reconnected = falsecreated_at = current timestampreconnected_at = + i-00000001openvdc-node-id: hello-1 + i-00000002openvdc-node-id: hello-1 + i-00000003openvdc-node-id: hello-1 + TASKi-00000001 + TASKi-00000002 + TASKi-00000003 + LXCi-00000001 + LXCi-00000002 + LXCi-00000003 + Tasks are relaunched. + openvdc-node-id: hello-1AgentID: cae491b6-2fed-45ab-b814-15d927fb9839-S0 + openvdc-node-id: hello-1AgentID: cae491b6-2fed-45ab-b814-15d927fb9839-S0 + Hypervisor checks which state the instance had during crash, compares it to the current LXC-state and then starts or stops containers accordingly. + Check if there are any NOT_CONNECTEDinstances, and if so add them to relaunch queue. + LXCi-00000001 + LXCi-00000002 + LXCi-00000003 + temporary id:cae491b6-2fed-45ab-b814-15d927fb9839-S0 + openvdc-node-id:hello-1 + diff --git a/hypervisor/lxc/lxc.go b/hypervisor/lxc/lxc.go index 8a5082ed..1fc2e343 100644 --- a/hypervisor/lxc/lxc.go +++ b/hypervisor/lxc/lxc.go @@ -173,8 +173,6 @@ func (d *LXCHypervisorDriver) modifyConf() error { } defer lxcconf.Close() - // Start to append lxc.network entries to tmp file. - // Append comment header fmt.Fprintf(lxcconf, "\n# OpenVDC Network Configuration\n") @@ -264,6 +262,34 @@ var lxcArch = map[string]string{ // TODO: powerpc, arm, arm64 } +func (d *LXCHypervisorDriver) Recover(instanceState model.InstanceState) error { + + switch instanceState.State { + case model.InstanceState_RUNNING: + if d.container.State() == lxc.STOPPED { + err := d.StartInstance() + if err != nil { + return errors.Wrapf(err, "Failed to start instance: %s", d.container.Name()) + } + } + + case model.InstanceState_STOPPED: + if d.container.State() == lxc.STOPPED { + err := d.StartInstance() + if err != nil { + return errors.Wrapf(err, "Failed to start instance: %s", d.container.Name()) + } + err = d.StopInstance() + if err != nil { + return errors.Wrapf(err, "Failed to stop instance: %s", d.container.Name()) + } + } + default: + } + + return nil +} + func (d *LXCHypervisorDriver) CreateInstance() error { d.log().Infoln("Creating lxc-container...") lxcTmpl := d.template.GetLxcTemplate() diff --git a/hypervisor/null/null.go b/hypervisor/null/null.go index 4c4f8ce1..8ecba4d9 100644 --- a/hypervisor/null/null.go +++ b/hypervisor/null/null.go @@ -47,6 +47,11 @@ func (h *NullHypervisorDriver) log() *log.Entry { return h.Base.Log } +func (h *NullHypervisorDriver) Recover(instanceState model.InstanceState) error { + log.WithFields(log.Fields{"hypervisor": "null"}).Infoln("Recover") + return nil +} + func (h *NullHypervisorDriver) StartInstance() error { h.log().Infoln("StartInstance") if h.template.CrashStage == model.NullTemplate_START { diff --git a/hypervisor/types.go b/hypervisor/types.go index 85d724d2..31e2e59d 100644 --- a/hypervisor/types.go +++ b/hypervisor/types.go @@ -9,6 +9,8 @@ import ( "github.com/spf13/viper" ) +type ContainerState int + type HypervisorProvider interface { Name() string CreateDriver(instance *model.Instance, template model.ResourceTemplate) (HypervisorDriver, error) @@ -16,6 +18,7 @@ type HypervisorProvider interface { } type HypervisorDriver interface { + Recover(model.InstanceState) error CreateInstance() error DestroyInstance() error StartInstance() error diff --git a/hypervisor/types_test.go b/hypervisor/types_test.go index cdeb0da5..73eba132 100644 --- a/hypervisor/types_test.go +++ b/hypervisor/types_test.go @@ -47,6 +47,10 @@ func (d *testDriver) DestroyInstance() error { return nil } +func (d *testDriver) Recover(model.InstanceState) error { + return nil +} + type testConsole struct { } diff --git a/model/cluster.pb.go b/model/cluster.pb.go index 07693c70..ed87f18f 100644 --- a/model/cluster.pb.go +++ b/model/cluster.pb.go @@ -80,6 +80,7 @@ type ExecutorNode struct { Console *Console `protobuf:"bytes,3,opt,name=console" json:"console,omitempty"` GrpcAddr string `protobuf:"bytes,4,opt,name=grpc_addr" json:"grpc_addr,omitempty"` LastState *NodeState `protobuf:"bytes,5,opt,name=last_state" json:"last_state,omitempty"` + NodeId string `protobuf:"bytes,6,opt,name=node_id" json:"node_id,omitempty"` } func (m *ExecutorNode) Reset() { *m = ExecutorNode{} } @@ -122,6 +123,13 @@ func (m *ExecutorNode) GetLastState() *NodeState { return nil } +func (m *ExecutorNode) GetNodeId() string { + if m != nil { + return m.NodeId + } + return "" +} + type SchedulerNode struct { Id string `protobuf:"bytes,1,opt,name=id" json:"id,omitempty"` CreatedAt *google_protobuf.Timestamp `protobuf:"bytes,2,opt,name=created_at" json:"created_at,omitempty"` @@ -182,28 +190,29 @@ func init() { func init() { proto.RegisterFile("cluster.proto", fileDescriptor1) } var fileDescriptor1 = []byte{ - // 367 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x92, 0xcd, 0x8e, 0xd3, 0x30, - 0x14, 0x85, 0x9b, 0xb6, 0xa1, 0xca, 0x85, 0x46, 0x95, 0x85, 0x20, 0xaa, 0x40, 0x54, 0x59, 0x75, - 0x51, 0x39, 0xa8, 0xec, 0x60, 0x07, 0x44, 0xc0, 0x86, 0x85, 0xd3, 0x15, 0x2c, 0x2a, 0xc7, 0xf6, - 0xa4, 0xd1, 0x24, 0x71, 0x64, 0x3b, 0xa3, 0xce, 0x4b, 0xcc, 0xd3, 0xcd, 0x03, 0x8d, 0x62, 0xf7, - 0x6f, 0x66, 0x3b, 0xb3, 0x8b, 0xee, 0x3d, 0xf7, 0x9c, 0xef, 0x28, 0x86, 0x29, 0xab, 0x3a, 0x6d, - 0x84, 0xc2, 0xad, 0x92, 0x46, 0x22, 0xbf, 0x96, 0x5c, 0x54, 0xf3, 0x6f, 0x45, 0x69, 0x76, 0x5d, - 0x8e, 0x99, 0xac, 0x93, 0x42, 0x56, 0xb4, 0x29, 0x12, 0xbb, 0xcf, 0xbb, 0xab, 0xa4, 0x35, 0xb7, - 0xad, 0xd0, 0x89, 0x29, 0x6b, 0xa1, 0x0d, 0xad, 0xdb, 0xf3, 0x97, 0xf3, 0x88, 0xaf, 0x61, 0xf2, - 0x43, 0x36, 0x5a, 0x56, 0x02, 0xad, 0x60, 0xdc, 0xab, 0x23, 0x6f, 0xe1, 0x2d, 0xc3, 0x75, 0x84, - 0xad, 0x3b, 0x3e, 0x6c, 0xf1, 0x46, 0xd1, 0x46, 0xb7, 0x52, 0x19, 0x62, 0x55, 0xe8, 0x03, 0x04, - 0x79, 0xd9, 0xf0, 0x2d, 0xe5, 0x5c, 0x45, 0xc3, 0x85, 0xb7, 0x0c, 0xc8, 0x79, 0x10, 0xbf, 0x85, - 0xe0, 0x74, 0x80, 0x26, 0x30, 0xca, 0xb2, 0xdf, 0xb3, 0x41, 0x7c, 0xef, 0xc1, 0x9b, 0x74, 0x2f, - 0x58, 0x67, 0xa4, 0xfa, 0x2b, 0xb9, 0x40, 0x21, 0x0c, 0x4b, 0x6e, 0x03, 0x03, 0x32, 0x2c, 0x39, - 0xfa, 0x0a, 0xc0, 0x94, 0xa0, 0x46, 0xf0, 0x2d, 0x35, 0xd6, 0xf5, 0xf5, 0x7a, 0x8e, 0x0b, 0x29, - 0x8b, 0x4a, 0xe0, 0x63, 0x29, 0xbc, 0x39, 0x76, 0x20, 0x17, 0x6a, 0xb4, 0x84, 0x09, 0x73, 0xac, - 0xd1, 0xc8, 0x1e, 0x86, 0x8f, 0x1b, 0x90, 0xe3, 0xba, 0x47, 0x2f, 0x54, 0xcb, 0x1c, 0xfa, 0xd8, - 0xa1, 0x9f, 0x06, 0xe8, 0x33, 0x40, 0x45, 0xb5, 0xd9, 0x6a, 0x43, 0x8d, 0x88, 0x7c, 0x6b, 0x35, - 0x3b, 0x58, 0xf5, 0xd0, 0x59, 0x3f, 0x27, 0x17, 0x9a, 0xf8, 0x3f, 0x4c, 0x33, 0xb6, 0x13, 0xbc, - 0xab, 0xc4, 0x8b, 0xd7, 0x8a, 0xef, 0x3c, 0x08, 0x4e, 0xb1, 0x68, 0x05, 0xbe, 0xe3, 0x72, 0x3f, - 0xe9, 0xdd, 0x53, 0x2e, 0xec, 0xe8, 0x9c, 0xe8, 0x59, 0xb9, 0xef, 0xc1, 0x77, 0x91, 0x21, 0x00, - 0x49, 0x7f, 0xfd, 0xc9, 0x36, 0x29, 0x49, 0x7f, 0xce, 0x06, 0xdf, 0x3f, 0xfd, 0xfb, 0x78, 0xf1, - 0xe0, 0xe8, 0x5e, 0xef, 0x12, 0xd9, 0x8a, 0xe6, 0x86, 0xb3, 0xc4, 0xc2, 0xe4, 0xaf, 0xac, 0xf3, - 0x97, 0x87, 0x00, 0x00, 0x00, 0xff, 0xff, 0xc4, 0xc8, 0x99, 0x8c, 0xae, 0x02, 0x00, 0x00, + // 377 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x92, 0xb1, 0xce, 0xd3, 0x30, + 0x14, 0x85, 0xff, 0xf4, 0x6f, 0x1a, 0xe5, 0x42, 0xa3, 0xca, 0x42, 0x10, 0x55, 0x20, 0xaa, 0x4c, + 0x1d, 0x2a, 0x07, 0x95, 0x0d, 0x36, 0x20, 0x02, 0x16, 0x06, 0xa7, 0x13, 0x0c, 0x91, 0x63, 0x9b, + 0x34, 0x22, 0x89, 0x23, 0xdb, 0x41, 0xe5, 0x25, 0x78, 0x54, 0x9e, 0x01, 0xc5, 0x6e, 0xda, 0xc2, + 0x0a, 0x9b, 0x7d, 0xef, 0xb9, 0xe7, 0x7e, 0x47, 0x36, 0x2c, 0x59, 0x33, 0x68, 0x23, 0x14, 0xee, + 0x95, 0x34, 0x12, 0xf9, 0xad, 0xe4, 0xa2, 0x59, 0xbf, 0xae, 0x6a, 0x73, 0x1c, 0x4a, 0xcc, 0x64, + 0x9b, 0x56, 0xb2, 0xa1, 0x5d, 0x95, 0xda, 0x7e, 0x39, 0x7c, 0x4d, 0x7b, 0xf3, 0xa3, 0x17, 0x3a, + 0x35, 0x75, 0x2b, 0xb4, 0xa1, 0x6d, 0x7f, 0x3d, 0x39, 0x8f, 0xe4, 0x1b, 0x04, 0x6f, 0x65, 0xa7, + 0x65, 0x23, 0xd0, 0x0e, 0xe6, 0xa3, 0x3a, 0xf6, 0x36, 0xde, 0x36, 0xda, 0xc7, 0xd8, 0xba, 0xe3, + 0x73, 0x17, 0x1f, 0x14, 0xed, 0x74, 0x2f, 0x95, 0x21, 0x56, 0x85, 0x9e, 0x42, 0x58, 0xd6, 0x1d, + 0x2f, 0x28, 0xe7, 0x2a, 0x9e, 0x6d, 0xbc, 0x6d, 0x48, 0xae, 0x85, 0xe4, 0x11, 0x84, 0x97, 0x01, + 0x14, 0xc0, 0x7d, 0x9e, 0x7f, 0x58, 0xdd, 0x25, 0xbf, 0x3c, 0x78, 0x98, 0x9d, 0x04, 0x1b, 0x8c, + 0x54, 0x9f, 0x24, 0x17, 0x28, 0x82, 0x59, 0xcd, 0xed, 0xc2, 0x90, 0xcc, 0x6a, 0x8e, 0x5e, 0x01, + 0x30, 0x25, 0xa8, 0x11, 0xbc, 0xa0, 0xc6, 0xba, 0x3e, 0xd8, 0xaf, 0x71, 0x25, 0x65, 0xd5, 0x08, + 0x3c, 0x85, 0xc2, 0x87, 0x29, 0x03, 0xb9, 0x51, 0xa3, 0x2d, 0x04, 0xcc, 0xb1, 0xc6, 0xf7, 0x76, + 0x30, 0xfa, 0x33, 0x01, 0x99, 0xda, 0x23, 0x7a, 0xa5, 0x7a, 0xe6, 0xd0, 0xe7, 0x0e, 0xfd, 0x52, + 0x40, 0x2f, 0x00, 0x1a, 0xaa, 0x4d, 0xa1, 0x0d, 0x35, 0x22, 0xf6, 0xad, 0xd5, 0xea, 0x6c, 0x35, + 0x42, 0xe7, 0x63, 0x9d, 0xdc, 0x68, 0x50, 0x0c, 0x41, 0x27, 0xb9, 0x28, 0x6a, 0x1e, 0x2f, 0xac, + 0xdb, 0x74, 0x4d, 0xbe, 0xc0, 0x32, 0x67, 0x47, 0xc1, 0x87, 0x46, 0xfc, 0xf7, 0xc0, 0xc9, 0x4f, + 0x0f, 0xc2, 0x0b, 0x10, 0xda, 0x81, 0xef, 0x88, 0xdd, 0xf3, 0x3d, 0xfe, 0x9b, 0x18, 0x3b, 0x6e, + 0x27, 0xfa, 0xa7, 0xbd, 0x4f, 0xc0, 0x77, 0x2b, 0x23, 0x00, 0x92, 0xbd, 0xff, 0x98, 0x1f, 0x32, + 0x92, 0xbd, 0x5b, 0xdd, 0xbd, 0x79, 0xfe, 0xf9, 0xd9, 0xcd, 0x57, 0xa4, 0x27, 0x7d, 0x4c, 0x65, + 0x2f, 0xba, 0xef, 0x9c, 0xa5, 0x16, 0xa6, 0x5c, 0x58, 0xe7, 0x97, 0xbf, 0x03, 0x00, 0x00, 0xff, + 0xff, 0x3f, 0x1e, 0x75, 0x4e, 0xc8, 0x02, 0x00, 0x00, } diff --git a/model/crashed_nodes.go b/model/crashed_nodes.go new file mode 100644 index 00000000..fe5ccdbf --- /dev/null +++ b/model/crashed_nodes.go @@ -0,0 +1,159 @@ +package model + +import ( + "fmt" + "github.com/golang/protobuf/proto" + "github.com/golang/protobuf/ptypes" + "github.com/pborman/uuid" + "golang.org/x/net/context" + "time" +) + +const crashedNodesBaseKey = "crashed_nodes" + +func init() { + schemaKeys = append(schemaKeys, crashedNodesBaseKey) +} + +type CrashedAgentNode interface { + proto.Message + GetReconnected() bool + GetAgentID() string + GetUUID() string +} + +type CrashedNodesOps interface { + Add(node *CrashedNode) error + FindByAgentMesosID(agentMesosID string) (*CrashedNode, error) + FindByAgentID(agentID string) (*CrashedNode, error) + FindByUUID(nodeUUID string) (*CrashedNode, error) + Filter(limit int, cb func(*CrashedNode) int) error + SetReconnected(node *CrashedNode) error +} + +type crashedNodes struct { + base +} + +func CrashedNodes(ctx context.Context) CrashedNodesOps { + return &crashedNodes{base{ctx: ctx}} +} + +func (i *crashedNodes) Add(n *CrashedNode) error { + if n.GetAgentID() == "" { + return fmt.Errorf("ID is not set") + } + + createdAt, _ := ptypes.TimestampProto(time.Now()) + + n.Uuid = uuid.New() + n.CreatedAt = createdAt + + bk, err := i.connection() + if err != nil { + return err + } + buf, err := proto.Marshal(n) + if err != nil { + return err + } + + if err = bk.Backend().Create(fmt.Sprintf("%s/%v", crashedNodesBaseKey, n.Uuid), buf); err != nil { + return nil + } + return nil +} + +func (i *crashedNodes) Find(agentID string, in CrashedAgentNode) error { + return nil +} + +func (i *crashedNodes) SetReconnected(node *CrashedNode) error { + bk, err := i.connection() + if err != nil { + return err + } + + reconnectedAt, _ := ptypes.TimestampProto(time.Now()) + + updatedNode := &CrashedNode{ + Uuid: node.GetUUID(), + Agentid: node.GetAgentID(), + Agentmesosid: node.GetAgentMesosID(), + Reconnected: true, + CreatedAt: node.GetCreatedAt(), + ReconnectedAt: reconnectedAt, + } + return bk.Update(fmt.Sprintf("/%s/%s", crashedNodesBaseKey, node.Uuid), updatedNode) +} + +func (i *crashedNodes) FindByAgentMesosID(agentMesosID string) (*CrashedNode, error) { + res := []*CrashedNode{} + err := i.Filter(1, func(node *CrashedNode) int { + if node.GetAgentMesosID() == agentMesosID { + res = append(res, node) + } + return len(res) + }) + if err != nil { + return nil, err + } + if len(res) > 0 { + return res[0], nil + } else { + return nil, nil + } +} + +func (i *crashedNodes) FindByAgentID(agentID string) (*CrashedNode, error) { + res := []*CrashedNode{} + err := i.Filter(1, func(node *CrashedNode) int { + if node.GetAgentID() == agentID && node.GetReconnected() == false { + res = append(res, node) + } + return len(res) + }) + if err != nil { + return nil, err + } + if len(res) > 0 { + return res[0], nil + } else { + return nil, nil + } +} + +func (i *crashedNodes) Filter(limit int, cb func(*CrashedNode) int) error { + bk, err := i.connection() + if err != nil { + return err + } + keys, err := bk.Keys(fmt.Sprintf("/%s", crashedNodesBaseKey)) + if err != nil { + return err + } + for keys.Next() { + node, err := i.FindByUUID(keys.Value()) + if err != nil { + return err + } + if limit > 0 && cb(node) > limit { + break + } else { + cb(node) + } + } + return nil +} + +func (i *crashedNodes) FindByUUID(nodeUUID string) (*CrashedNode, error) { + bk, err := i.connection() + if err != nil { + return nil, err + } + n := &CrashedNode{} + if err := bk.Find(fmt.Sprintf("/%s/%s", crashedNodesBaseKey, nodeUUID), n); err != nil { + return nil, err + } + return n, nil +} diff --git a/model/crashed_nodes.pb.go b/model/crashed_nodes.pb.go new file mode 100644 index 00000000..9b71e32d --- /dev/null +++ b/model/crashed_nodes.pb.go @@ -0,0 +1,112 @@ +// Code generated by protoc-gen-go. +// source: crashed_nodes.proto +// DO NOT EDIT! + +/* +Package model is a generated protocol buffer package. + +It is generated from these files: + crashed_nodes.proto + +It has these top-level messages: + CrashedNode +*/ +package model + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" +import google_protobuf "github.com/golang/protobuf/ptypes/timestamp" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package + +type CrashedNode struct { + Uuid string `protobuf:"bytes,1,opt,name=uuid" json:"uuid,omitempty"` + Agentid string `protobuf:"bytes,2,opt,name=agentid" json:"agentid,omitempty"` + Agentmesosid string `protobuf:"bytes,3,opt,name=agentmesosid" json:"agentmesosid,omitempty"` + Reconnected bool `protobuf:"varint,4,opt,name=reconnected" json:"reconnected,omitempty"` + CreatedAt *google_protobuf.Timestamp `protobuf:"bytes,5,opt,name=created_at,json=createdAt" json:"created_at,omitempty"` + ReconnectedAt *google_protobuf.Timestamp `protobuf:"bytes,6,opt,name=reconnected_at,json=reconnectedAt" json:"reconnected_at,omitempty"` +} + +func (m *CrashedNode) Reset() { *m = CrashedNode{} } +func (m *CrashedNode) String() string { return proto.CompactTextString(m) } +func (*CrashedNode) ProtoMessage() {} +func (*CrashedNode) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } + +func (m *CrashedNode) GetUUID() string { + if m != nil { + return m.Uuid + } + return "" +} + +func (m *CrashedNode) GetAgentID() string { + if m != nil { + return m.Agentid + } + return "" +} + +func (m *CrashedNode) GetAgentMesosID() string { + if m != nil { + return m.Agentmesosid + } + return "" +} + +func (m *CrashedNode) GetReconnected() bool { + if m != nil { + return m.Reconnected + } + return false +} + +func (m *CrashedNode) GetCreatedAt() *google_protobuf.Timestamp { + if m != nil { + return m.CreatedAt + } + return nil +} + +func (m *CrashedNode) GetReconnectedAt() *google_protobuf.Timestamp { + if m != nil { + return m.ReconnectedAt + } + return nil +} + +func init() { + proto.RegisterType((*CrashedNode)(nil), "model.CrashedNode") +} + +func init() { proto.RegisterFile("crashed_nodes.proto", fileDescriptor0) } + +var fileDescriptor3 = []byte{ + // 254 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x84, 0x90, 0xb1, 0x4f, 0xf3, 0x30, + 0x10, 0xc5, 0x95, 0xef, 0x6b, 0x0b, 0x75, 0x80, 0xc1, 0x2c, 0x51, 0x25, 0x44, 0xd5, 0x89, 0xc9, + 0x96, 0x60, 0x42, 0x4c, 0x81, 0x9d, 0x21, 0x62, 0x62, 0xa9, 0x1c, 0xfb, 0x70, 0x22, 0x25, 0x3e, + 0x2b, 0xbe, 0x20, 0xf8, 0xeb, 0xc1, 0xb8, 0x44, 0x84, 0x89, 0xed, 0xdd, 0xbb, 0xf7, 0x7e, 0xb2, + 0x8f, 0x9d, 0xeb, 0x41, 0x85, 0x06, 0xcc, 0xde, 0xa1, 0x81, 0x20, 0xfc, 0x80, 0x84, 0x7c, 0xd9, + 0xc7, 0xa1, 0xdb, 0xdc, 0xd9, 0x96, 0x9a, 0xb1, 0x16, 0x1a, 0x7b, 0x69, 0xb1, 0x53, 0xce, 0xca, + 0xb4, 0xaf, 0xc7, 0x17, 0xe9, 0xe9, 0xdd, 0x43, 0x90, 0xd4, 0xf6, 0x10, 0x48, 0xf5, 0xfe, 0x47, + 0x1d, 0x18, 0xbb, 0x8f, 0x8c, 0xe5, 0x0f, 0x07, 0xf6, 0x63, 0xa4, 0x71, 0xce, 0x16, 0xe3, 0xd8, + 0x9a, 0x22, 0xdb, 0x66, 0x57, 0xeb, 0x2a, 0x69, 0x5e, 0xb0, 0x23, 0x65, 0xc1, 0x51, 0xb4, 0xff, + 0x25, 0x7b, 0x1a, 0xf9, 0x8e, 0x9d, 0x24, 0x19, 0x99, 0x18, 0xe2, 0xfa, 0x7f, 0x5a, 0xff, 0xf2, + 0xf8, 0x96, 0xe5, 0x03, 0x68, 0x74, 0x0e, 0x34, 0x81, 0x29, 0x16, 0x31, 0x72, 0x5c, 0xcd, 0x2d, + 0x7e, 0xcb, 0x98, 0x1e, 0x40, 0x45, 0xb9, 0x57, 0x54, 0x2c, 0x63, 0x20, 0xbf, 0xde, 0x08, 0x8b, + 0x68, 0x3b, 0x10, 0xd3, 0x57, 0xc4, 0xd3, 0xf4, 0xf2, 0x6a, 0xfd, 0x9d, 0x2e, 0x89, 0x97, 0xec, + 0x6c, 0x46, 0xfa, 0xaa, 0xaf, 0xfe, 0xac, 0x9f, 0xce, 0x1a, 0x25, 0xdd, 0x5f, 0x3e, 0x5f, 0xcc, + 0x0e, 0xa8, 0xde, 0x42, 0x23, 0xd1, 0x83, 0x7b, 0x35, 0x5a, 0xa6, 0xfb, 0xd6, 0xab, 0xc4, 0xb8, + 0xf9, 0x0c, 0x00, 0x00, 0xff, 0xff, 0x72, 0x3c, 0xfb, 0x18, 0x84, 0x01, 0x00, 0x00, +} diff --git a/model/instances.go b/model/instances.go index cebc1377..cae34626 100644 --- a/model/instances.go +++ b/model/instances.go @@ -19,7 +19,9 @@ type InstanceOps interface { AddFailureMessage(id string, failureMessage FailureMessage_ErrorType) error //GetLatestFailureMessage(id string) (*FailureMessage, error) UpdateState(id string, next InstanceState_State) error + UpdateConnectionStatus(id string, connStatus ConnectionStatus_Status) error FilterByState(state InstanceState_State) ([]*Instance, error) + FilterByAgentMesosID(agentID string) ([]*Instance, error) Update(*Instance) error Filter(limit int, cb func(*Instance) int) error WaitStateUpdate(id string) (*InstanceState, error) @@ -93,6 +95,11 @@ func (i *instances) Create(n *Instance) (*Instance, error) { } n.LastState = initState + initConnectionStatus := &ConnectionStatus{ + Status: ConnectionStatus_NOT_CONNECTED, + } + n.ConnectionStatus = initConnectionStatus + bk, err := i.connection() if err != nil { return nil, err @@ -112,7 +119,6 @@ func (i *instances) Create(n *Instance) (*Instance, error) { if err != nil { return nil, err } - if err = bk.Backend().Create(fmt.Sprintf("%s/failure-messages", nkey), []byte{}); err != nil { return nil, err } @@ -187,6 +193,28 @@ func (i *instances) findLastState(id string) (*InstanceState, error) { return n, nil } +func (i *instances) UpdateConnectionStatus(id string, connStatus ConnectionStatus_Status) error { + instance, err := i.FindByID(id) + if err != nil { + return err + } + + cStatus := &ConnectionStatus{ + Status: connStatus, + } + + instance.ConnectionStatus = cStatus + bk, err := i.connection() + if err != nil { + return err + } + err = bk.Update(fmt.Sprintf("%s/%s", instancesBaseKey, id), instance) + if err != nil { + return err + } + return nil +} + func (i *instances) UpdateState(id string, next InstanceState_State) error { instance, err := i.FindByID(id) if err != nil { @@ -225,6 +253,20 @@ func (i *instances) FilterByState(state InstanceState_State) ([]*Instance, error return res, nil } +func (i *instances) FilterByAgentMesosID(agentMesosID string) ([]*Instance, error) { + res := []*Instance{} + err := i.Filter(0, func(inst *Instance) int { + if inst.GetSlaveId() == agentMesosID { + res = append(res, inst) + } + return len(res) + }) + if err != nil { + return nil, err + } + return res, nil +} + func (i *instances) Filter(limit int, cb func(*Instance) int) error { bk, err := i.connection() if err != nil { diff --git a/model/model.pb.go b/model/model.pb.go index c3d4a795..d65bcbbf 100644 --- a/model/model.pb.go +++ b/model/model.pb.go @@ -11,6 +11,7 @@ It is generated from these files: It has these top-level messages: Instance + ConnectionStatus InstanceState FailureMessage Template @@ -40,6 +41,27 @@ var _ = math.Inf // proto package needs to be updated. const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package +type ConnectionStatus_Status int32 + +const ( + ConnectionStatus_CONNECTED ConnectionStatus_Status = 0 + ConnectionStatus_NOT_CONNECTED ConnectionStatus_Status = 1 +) + +var ConnectionStatus_Status_name = map[int32]string{ + 0: "CONNECTED", + 1: "NOT_CONNECTED", +} +var ConnectionStatus_Status_value = map[string]int32{ + "CONNECTED": 0, + "NOT_CONNECTED": 1, +} + +func (x ConnectionStatus_Status) String() string { + return proto.EnumName(ConnectionStatus_Status_name, int32(x)) +} +func (ConnectionStatus_Status) EnumDescriptor() ([]byte, []int) { return fileDescriptor0, []int{1, 0} } + type InstanceState_State int32 const ( @@ -83,7 +105,7 @@ var InstanceState_State_value = map[string]int32{ func (x InstanceState_State) String() string { return proto.EnumName(InstanceState_State_name, int32(x)) } -func (InstanceState_State) EnumDescriptor() ([]byte, []int) { return fileDescriptor0, []int{1, 0} } +func (InstanceState_State) EnumDescriptor() ([]byte, []int) { return fileDescriptor0, []int{2, 0} } type FailureMessage_ErrorType int32 @@ -113,7 +135,7 @@ var FailureMessage_ErrorType_value = map[string]int32{ func (x FailureMessage_ErrorType) String() string { return proto.EnumName(FailureMessage_ErrorType_name, int32(x)) } -func (FailureMessage_ErrorType) EnumDescriptor() ([]byte, []int) { return fileDescriptor0, []int{2, 0} } +func (FailureMessage_ErrorType) EnumDescriptor() ([]byte, []int) { return fileDescriptor0, []int{3, 0} } type NullTemplate_CrashStage int32 @@ -146,16 +168,18 @@ var NullTemplate_CrashStage_value = map[string]int32{ func (x NullTemplate_CrashStage) String() string { return proto.EnumName(NullTemplate_CrashStage_name, int32(x)) } -func (NullTemplate_CrashStage) EnumDescriptor() ([]byte, []int) { return fileDescriptor0, []int{6, 0} } +func (NullTemplate_CrashStage) EnumDescriptor() ([]byte, []int) { return fileDescriptor0, []int{7, 0} } type Instance struct { Id string `protobuf:"bytes,1,opt,name=id" json:"id,omitempty"` SlaveId string `protobuf:"bytes,2,opt,name=slave_id" json:"slave_id,omitempty"` // string resource_id = 3; // Obsolete - LastState *InstanceState `protobuf:"bytes,4,opt,name=last_state" json:"last_state,omitempty"` - CreatedAt *google_protobuf.Timestamp `protobuf:"bytes,5,opt,name=created_at" json:"created_at,omitempty"` - Template *Template `protobuf:"bytes,6,opt,name=template" json:"template,omitempty"` - LatestFailure *FailureMessage `protobuf:"bytes,7,opt,name=latest_failure,json=latestFailure" json:"latest_failure,omitempty"` + LastState *InstanceState `protobuf:"bytes,4,opt,name=last_state" json:"last_state,omitempty"` + CreatedAt *google_protobuf.Timestamp `protobuf:"bytes,5,opt,name=created_at" json:"created_at,omitempty"` + Template *Template `protobuf:"bytes,6,opt,name=template" json:"template,omitempty"` + LatestFailure *FailureMessage `protobuf:"bytes,7,opt,name=latest_failure" json:"latest_failure,omitempty"` + AutoRecovery bool `protobuf:"varint,8,opt,name=auto_recovery" json:"auto_recovery,omitempty"` + ConnectionStatus *ConnectionStatus `protobuf:"bytes,9,opt,name=connection_status" json:"connection_status,omitempty"` } func (m *Instance) Reset() { *m = Instance{} } @@ -205,6 +229,36 @@ func (m *Instance) GetLatestFailure() *FailureMessage { return nil } +func (m *Instance) GetAutoRecovery() bool { + if m != nil { + return m.AutoRecovery + } + return false +} + +func (m *Instance) GetConnectionStatus() *ConnectionStatus { + if m != nil { + return m.ConnectionStatus + } + return nil +} + +type ConnectionStatus struct { + Status ConnectionStatus_Status `protobuf:"varint,1,opt,name=status,enum=model.ConnectionStatus_Status" json:"status,omitempty"` +} + +func (m *ConnectionStatus) Reset() { *m = ConnectionStatus{} } +func (m *ConnectionStatus) String() string { return proto.CompactTextString(m) } +func (*ConnectionStatus) ProtoMessage() {} +func (*ConnectionStatus) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} } + +func (m *ConnectionStatus) GetStatus() ConnectionStatus_Status { + if m != nil { + return m.Status + } + return ConnectionStatus_CONNECTED +} + type InstanceState struct { State InstanceState_State `protobuf:"varint,1,opt,name=state,enum=model.InstanceState_State" json:"state,omitempty"` CreatedAt *google_protobuf.Timestamp `protobuf:"bytes,2,opt,name=created_at" json:"created_at,omitempty"` @@ -213,7 +267,7 @@ type InstanceState struct { func (m *InstanceState) Reset() { *m = InstanceState{} } func (m *InstanceState) String() string { return proto.CompactTextString(m) } func (*InstanceState) ProtoMessage() {} -func (*InstanceState) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} } +func (*InstanceState) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} } func (m *InstanceState) GetState() InstanceState_State { if m != nil { @@ -230,14 +284,14 @@ func (m *InstanceState) GetCreatedAt() *google_protobuf.Timestamp { } type FailureMessage struct { - ErrorType FailureMessage_ErrorType `protobuf:"varint,1,opt,name=error_type,json=errorType,enum=model.FailureMessage_ErrorType" json:"error_type,omitempty"` - FailedAt *google_protobuf.Timestamp `protobuf:"bytes,2,opt,name=failed_at,json=failedAt" json:"failed_at,omitempty"` + ErrorType FailureMessage_ErrorType `protobuf:"varint,1,opt,name=error_type,enum=model.FailureMessage_ErrorType" json:"error_type,omitempty"` + FailedAt *google_protobuf.Timestamp `protobuf:"bytes,2,opt,name=failed_at" json:"failed_at,omitempty"` } func (m *FailureMessage) Reset() { *m = FailureMessage{} } func (m *FailureMessage) String() string { return proto.CompactTextString(m) } func (*FailureMessage) ProtoMessage() {} -func (*FailureMessage) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} } +func (*FailureMessage) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3} } func (m *FailureMessage) GetErrorType() FailureMessage_ErrorType { if m != nil { @@ -266,7 +320,7 @@ type Template struct { func (m *Template) Reset() { *m = Template{} } func (m *Template) String() string { return proto.CompactTextString(m) } func (*Template) ProtoMessage() {} -func (*Template) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3} } +func (*Template) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{4} } type isTemplate_Item interface { isTemplate_Item() @@ -427,7 +481,7 @@ type NoneTemplate struct { func (m *NoneTemplate) Reset() { *m = NoneTemplate{} } func (m *NoneTemplate) String() string { return proto.CompactTextString(m) } func (*NoneTemplate) ProtoMessage() {} -func (*NoneTemplate) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{4} } +func (*NoneTemplate) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} } type LxcTemplate struct { Vcpu int32 `protobuf:"varint,1,opt,name=vcpu" json:"vcpu,omitempty"` @@ -443,7 +497,7 @@ type LxcTemplate struct { func (m *LxcTemplate) Reset() { *m = LxcTemplate{} } func (m *LxcTemplate) String() string { return proto.CompactTextString(m) } func (*LxcTemplate) ProtoMessage() {} -func (*LxcTemplate) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} } +func (*LxcTemplate) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} } func (m *LxcTemplate) GetVcpu() int32 { if m != nil { @@ -510,7 +564,7 @@ type LxcTemplate_Image struct { func (m *LxcTemplate_Image) Reset() { *m = LxcTemplate_Image{} } func (m *LxcTemplate_Image) String() string { return proto.CompactTextString(m) } func (*LxcTemplate_Image) ProtoMessage() {} -func (*LxcTemplate_Image) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5, 0} } +func (*LxcTemplate_Image) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6, 0} } func (m *LxcTemplate_Image) GetDownloadUrl() string { if m != nil { @@ -542,7 +596,7 @@ type LxcTemplate_Interface struct { func (m *LxcTemplate_Interface) Reset() { *m = LxcTemplate_Interface{} } func (m *LxcTemplate_Interface) String() string { return proto.CompactTextString(m) } func (*LxcTemplate_Interface) ProtoMessage() {} -func (*LxcTemplate_Interface) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5, 1} } +func (*LxcTemplate_Interface) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6, 1} } func (m *LxcTemplate_Interface) GetType() string { if m != nil { @@ -597,7 +651,7 @@ type LxcTemplate_Template struct { func (m *LxcTemplate_Template) Reset() { *m = LxcTemplate_Template{} } func (m *LxcTemplate_Template) String() string { return proto.CompactTextString(m) } func (*LxcTemplate_Template) ProtoMessage() {} -func (*LxcTemplate_Template) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5, 2} } +func (*LxcTemplate_Template) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6, 2} } func (m *LxcTemplate_Template) GetTemplate() string { if m != nil { @@ -700,7 +754,7 @@ type NullTemplate struct { func (m *NullTemplate) Reset() { *m = NullTemplate{} } func (m *NullTemplate) String() string { return proto.CompactTextString(m) } func (*NullTemplate) ProtoMessage() {} -func (*NullTemplate) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} } +func (*NullTemplate) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} } func (m *NullTemplate) GetVcpu() int32 { if m != nil { @@ -732,6 +786,7 @@ func (m *NullTemplate) GetNodeGroups() []string { func init() { proto.RegisterType((*Instance)(nil), "model.Instance") + proto.RegisterType((*ConnectionStatus)(nil), "model.ConnectionStatus") proto.RegisterType((*InstanceState)(nil), "model.InstanceState") proto.RegisterType((*FailureMessage)(nil), "model.FailureMessage") proto.RegisterType((*Template)(nil), "model.Template") @@ -741,6 +796,7 @@ func init() { proto.RegisterType((*LxcTemplate_Interface)(nil), "model.LxcTemplate.Interface") proto.RegisterType((*LxcTemplate_Template)(nil), "model.LxcTemplate.Template") proto.RegisterType((*NullTemplate)(nil), "model.NullTemplate") + proto.RegisterEnum("model.ConnectionStatus_Status", ConnectionStatus_Status_name, ConnectionStatus_Status_value) proto.RegisterEnum("model.InstanceState_State", InstanceState_State_name, InstanceState_State_value) proto.RegisterEnum("model.FailureMessage_ErrorType", FailureMessage_ErrorType_name, FailureMessage_ErrorType_value) proto.RegisterEnum("model.NullTemplate_CrashStage", NullTemplate_CrashStage_name, NullTemplate_CrashStage_value) @@ -749,73 +805,78 @@ func init() { func init() { proto.RegisterFile("model.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ - // 1084 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x56, 0x6d, 0x8f, 0xdb, 0x44, - 0x10, 0xbe, 0x24, 0x76, 0x62, 0x4f, 0x5e, 0x6a, 0x96, 0x52, 0x59, 0xa1, 0xb4, 0x51, 0x84, 0xc4, - 0x09, 0xa4, 0x04, 0x1d, 0x55, 0x91, 0xa0, 0x02, 0xee, 0x1a, 0xb7, 0x8d, 0xd4, 0x26, 0xc7, 0x26, - 0x27, 0x04, 0x5f, 0xac, 0x8d, 0xbd, 0xe7, 0x58, 0xe7, 0x37, 0xad, 0xed, 0x90, 0xfb, 0x0f, 0x7c, - 0x43, 0xea, 0x5f, 0xe0, 0x07, 0x02, 0xfd, 0x8c, 0x76, 0xd7, 0x4e, 0x9c, 0x36, 0x48, 0xbc, 0x7c, - 0xca, 0x3e, 0xcf, 0x3e, 0x33, 0x3b, 0x33, 0x3b, 0x9e, 0x0d, 0xb4, 0xc3, 0xd8, 0xa5, 0xc1, 0x28, - 0x61, 0x71, 0x16, 0x23, 0x55, 0x80, 0xfe, 0xd7, 0x9e, 0x9f, 0xad, 0xf3, 0xd5, 0xc8, 0x89, 0xc3, - 0xb1, 0x17, 0x07, 0x24, 0xf2, 0xc6, 0x62, 0x7f, 0x95, 0x5f, 0x8f, 0x93, 0xec, 0x36, 0xa1, 0xe9, - 0x38, 0xf3, 0x43, 0x9a, 0x66, 0x24, 0x4c, 0xf6, 0x2b, 0xe9, 0x63, 0xf8, 0x6b, 0x1d, 0xb4, 0x69, - 0x94, 0x66, 0x24, 0x72, 0x28, 0xea, 0x41, 0xdd, 0x77, 0xcd, 0xda, 0xa0, 0x76, 0xaa, 0xe3, 0xba, - 0xef, 0xa2, 0x3e, 0x68, 0x69, 0x40, 0x36, 0xd4, 0xf6, 0x5d, 0xb3, 0x2e, 0xd8, 0x1d, 0x46, 0x8f, - 0x00, 0x02, 0x92, 0x66, 0x76, 0x9a, 0x91, 0x8c, 0x9a, 0xca, 0xa0, 0x76, 0xda, 0x3e, 0xbb, 0x3b, - 0x92, 0xe1, 0x95, 0x0e, 0x17, 0x7c, 0x0f, 0x57, 0x74, 0xe8, 0x2b, 0x00, 0x87, 0x51, 0x92, 0x51, - 0xd7, 0x26, 0x99, 0xa9, 0x0a, 0xab, 0xfe, 0xc8, 0x8b, 0x63, 0x2f, 0xa0, 0xa3, 0x32, 0xea, 0xd1, - 0xb2, 0x0c, 0x12, 0x57, 0xd4, 0xe8, 0x33, 0xd0, 0x32, 0x1a, 0x26, 0x01, 0x3f, 0xaf, 0x29, 0x2c, - 0xef, 0x14, 0xe7, 0x2d, 0x0b, 0x1a, 0xef, 0x04, 0xe8, 0x09, 0xf4, 0xf8, 0x6f, 0x9a, 0xd9, 0xd7, - 0xc4, 0x0f, 0x72, 0x46, 0xcd, 0x96, 0x30, 0xf9, 0xa0, 0x30, 0x79, 0x26, 0xd9, 0x57, 0x34, 0x4d, - 0x89, 0x47, 0x71, 0x57, 0x8a, 0x0b, 0x76, 0xf8, 0xba, 0x0e, 0xdd, 0x83, 0x24, 0xd0, 0xe7, 0xa0, - 0xca, 0x4c, 0x79, 0x75, 0x7a, 0x67, 0xfd, 0x63, 0x99, 0x8e, 0x64, 0xbe, 0xea, 0xb1, 0x54, 0xeb, - 0xff, 0x26, 0xd5, 0xe1, 0xeb, 0x1a, 0xa8, 0xf2, 0xdc, 0x1e, 0x00, 0xb6, 0x9e, 0x4f, 0x17, 0x4b, - 0x0b, 0x5b, 0x13, 0xe3, 0x04, 0x01, 0x34, 0xbf, 0xbf, 0xb2, 0xae, 0xac, 0x89, 0x51, 0x43, 0x1d, - 0xd0, 0x16, 0xcb, 0x73, 0xbc, 0x9c, 0xce, 0x9e, 0x1b, 0x75, 0xd4, 0x86, 0x16, 0xbe, 0x9a, 0xcd, - 0x38, 0x68, 0xc8, 0xad, 0xf9, 0xe5, 0x25, 0x47, 0x0a, 0xdf, 0x12, 0xc8, 0x9a, 0x18, 0x2a, 0xea, - 0x82, 0x8e, 0xad, 0x8b, 0xf9, 0x5c, 0x98, 0x35, 0x91, 0x01, 0x9d, 0xc5, 0x8b, 0xab, 0x25, 0x47, - 0x93, 0xf9, 0x0f, 0x33, 0xa3, 0xc5, 0x8f, 0x5c, 0x5a, 0xf8, 0xd5, 0x74, 0x76, 0xbe, 0xb4, 0x26, - 0x86, 0xc6, 0x8f, 0x7c, 0x76, 0x3e, 0x7d, 0x69, 0x4d, 0x0c, 0x7d, 0xf8, 0x7b, 0x0d, 0x7a, 0x87, - 0xa5, 0x43, 0xdf, 0x00, 0x50, 0xc6, 0x62, 0x66, 0xf3, 0x56, 0x2b, 0xca, 0xf3, 0xf0, 0x68, 0x95, - 0x47, 0x16, 0xd7, 0x2d, 0x6f, 0x13, 0x8a, 0x75, 0x5a, 0x2e, 0xd1, 0x97, 0xa0, 0xf3, 0x2b, 0xfa, - 0xa7, 0x65, 0xd2, 0xa4, 0xf8, 0x3c, 0x1b, 0xae, 0x41, 0xdf, 0x39, 0x44, 0x77, 0xa0, 0x2d, 0x83, - 0xb4, 0x79, 0x6a, 0xc6, 0x09, 0xcf, 0xab, 0x20, 0x44, 0x8d, 0x8c, 0x5a, 0x45, 0xc2, 0x8b, 0x61, - 0xd4, 0xd1, 0x7b, 0xd0, 0x2d, 0x08, 0x59, 0x10, 0xa3, 0x81, 0xee, 0x82, 0x51, 0x50, 0xbb, 0x12, - 0x18, 0x0a, 0xcf, 0x5a, 0x2b, 0x7b, 0x0c, 0x0d, 0xa1, 0x53, 0x76, 0x99, 0x9d, 0x33, 0xbf, 0xf8, - 0x5c, 0x0e, 0x38, 0xf4, 0x29, 0x28, 0x51, 0x1c, 0x51, 0xf3, 0x8f, 0x86, 0xc8, 0xe7, 0xfd, 0xa2, - 0x1c, 0xb3, 0x38, 0xa2, 0xa5, 0x9f, 0x17, 0x27, 0x58, 0x68, 0xd0, 0x27, 0xd0, 0x08, 0xb6, 0x8e, - 0xf9, 0xa7, 0x94, 0xa2, 0x42, 0xfa, 0x72, 0xeb, 0x54, 0x94, 0x5c, 0x21, 0x9c, 0xe6, 0x41, 0x60, - 0xbe, 0x79, 0xcb, 0x69, 0x1e, 0x04, 0x07, 0x4e, 0xf3, 0x20, 0xf8, 0x3f, 0xcd, 0x77, 0xd1, 0x04, - 0x65, 0x9a, 0xd1, 0x70, 0xd8, 0x83, 0x4e, 0x35, 0xe0, 0xe1, 0x6f, 0x2d, 0x68, 0x57, 0xc2, 0x42, - 0x08, 0x94, 0x8d, 0x93, 0xe4, 0xa2, 0x00, 0x2a, 0x16, 0x6b, 0x74, 0x1f, 0xf4, 0x90, 0x86, 0x31, - 0xbb, 0xb5, 0xbd, 0x95, 0x38, 0x56, 0xc5, 0x7b, 0x82, 0xcf, 0x93, 0xd0, 0x8f, 0x6c, 0x61, 0xd5, - 0x10, 0x9b, 0x3b, 0x8c, 0x3e, 0x86, 0x2e, 0x5f, 0xef, 0xad, 0x15, 0x21, 0x38, 0x24, 0xd1, 0x63, - 0xd0, 0x83, 0xad, 0x63, 0xfb, 0x21, 0xf1, 0x68, 0x31, 0x3e, 0xcc, 0x77, 0x2b, 0x36, 0x9a, 0xf2, - 0x7d, 0xbc, 0x97, 0xa2, 0x27, 0x00, 0x7e, 0x94, 0x51, 0x76, 0x4d, 0x1c, 0x9a, 0x9a, 0xcd, 0x41, - 0xe3, 0xb4, 0x7d, 0x76, 0xff, 0x98, 0x61, 0x29, 0xc2, 0x15, 0x3d, 0xfa, 0x16, 0x3a, 0xdc, 0xd5, - 0x6e, 0xfa, 0xc8, 0x51, 0xf2, 0xe1, 0x11, 0xfb, 0xdd, 0x24, 0x3a, 0x30, 0x40, 0x03, 0x68, 0x47, - 0xb1, 0x4b, 0x6d, 0x8f, 0xc5, 0x79, 0x92, 0x9a, 0xda, 0xa0, 0x71, 0xaa, 0xe3, 0x2a, 0xd5, 0xa7, - 0xa0, 0x8a, 0xa0, 0x79, 0x7b, 0xb9, 0xf1, 0xcf, 0x51, 0x10, 0x13, 0xd7, 0xce, 0x59, 0x50, 0xb6, - 0x57, 0x95, 0xe3, 0xee, 0x9c, 0xf5, 0x4d, 0x9a, 0x87, 0xf2, 0x9b, 0x93, 0xa3, 0xb9, 0x4a, 0xa1, - 0x7b, 0xd0, 0x94, 0x50, 0xd4, 0x59, 0xc7, 0x05, 0xea, 0x5f, 0x81, 0xbe, 0x4b, 0x91, 0x5f, 0xe0, - 0xee, 0x9b, 0xd5, 0xb1, 0x58, 0x23, 0x13, 0x5a, 0x21, 0x71, 0x88, 0xeb, 0xb2, 0xc2, 0x6d, 0x09, - 0xf9, 0xe5, 0xf9, 0xc9, 0xe6, 0x91, 0xd8, 0x92, 0x4e, 0x77, 0xb8, 0xff, 0x4b, 0xa3, 0xf2, 0x81, - 0xf4, 0x2b, 0x73, 0x5a, 0xba, 0xde, 0x8f, 0x65, 0x13, 0x5a, 0x2b, 0xe2, 0xdc, 0xd0, 0xc8, 0x2d, - 0xba, 0xa3, 0x84, 0x3c, 0x62, 0xd7, 0x4f, 0x33, 0x16, 0x97, 0x11, 0x4b, 0xc4, 0x2d, 0x18, 0x0d, - 0x28, 0x49, 0xe5, 0x23, 0xa3, 0xe3, 0x12, 0xf2, 0xf0, 0x09, 0x73, 0xd6, 0xa2, 0x0d, 0x74, 0x2c, - 0xd6, 0x5c, 0xbd, 0x21, 0xcc, 0x27, 0x51, 0x26, 0x9e, 0x08, 0x1d, 0x97, 0x90, 0xfb, 0x4f, 0x29, - 0xdb, 0x50, 0x26, 0x6e, 0x4f, 0xc7, 0x05, 0xe2, 0xfc, 0x0d, 0xbd, 0xe5, 0x2f, 0x9c, 0x26, 0x79, - 0x89, 0xd0, 0x03, 0x00, 0xbe, 0x2a, 0x6c, 0x74, 0xb1, 0x57, 0x61, 0xd0, 0x63, 0xb8, 0xe7, 0xfa, - 0x29, 0x59, 0x05, 0xd4, 0xf6, 0x12, 0xcf, 0xde, 0x90, 0xc0, 0x77, 0x49, 0xe6, 0xc7, 0x91, 0x09, - 0x83, 0xda, 0xa9, 0x86, 0xff, 0x66, 0x97, 0xdf, 0xdd, 0x75, 0x90, 0xa7, 0x6b, 0xdb, 0x21, 0xce, - 0x9a, 0x9a, 0x6d, 0x21, 0xae, 0x52, 0x42, 0x11, 0x33, 0x87, 0x16, 0x8a, 0x4e, 0xa1, 0xd8, 0x53, - 0x3c, 0x36, 0xba, 0xcd, 0x18, 0xb1, 0x09, 0xf3, 0x52, 0xb3, 0x2b, 0xba, 0xa9, 0xc2, 0x0c, 0xdf, - 0xd4, 0xa0, 0x53, 0x1d, 0x0b, 0xff, 0xe1, 0x53, 0xfd, 0x0e, 0xda, 0x0e, 0x23, 0xe9, 0x9a, 0xbf, - 0xdb, 0x1e, 0x15, 0x77, 0xd2, 0x3b, 0x7b, 0x70, 0x64, 0xe4, 0x8c, 0x9e, 0x72, 0xd9, 0x82, 0xab, - 0x70, 0xd5, 0xe4, 0xed, 0x9e, 0x57, 0xde, 0xe9, 0xf9, 0xe1, 0x25, 0xc0, 0xde, 0x18, 0x69, 0xa0, - 0xcc, 0xe6, 0x33, 0xcb, 0x38, 0x41, 0x3a, 0xa8, 0xe5, 0xcc, 0xd6, 0x40, 0x29, 0x86, 0x35, 0x40, - 0xf3, 0x29, 0xb6, 0xf8, 0x3c, 0x6e, 0xf0, 0xf7, 0x6c, 0x62, 0x2d, 0x96, 0x78, 0xfe, 0xa3, 0xa1, - 0xf0, 0x8d, 0x62, 0x7c, 0xab, 0x17, 0x0f, 0x7f, 0xfa, 0xa8, 0xf2, 0x67, 0x88, 0x6c, 0xd3, 0xf5, - 0x38, 0x4e, 0x68, 0xb4, 0x71, 0x9d, 0xb1, 0x88, 0x7c, 0xd5, 0x14, 0xb3, 0xef, 0x8b, 0xbf, 0x02, - 0x00, 0x00, 0xff, 0xff, 0x37, 0x32, 0x06, 0xfc, 0x48, 0x09, 0x00, 0x00, + // 1165 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x56, 0x4b, 0x6f, 0xdb, 0xc6, + 0x13, 0xb7, 0xde, 0xe4, 0x48, 0x56, 0x98, 0xfd, 0xe7, 0x9f, 0x12, 0x6a, 0x9a, 0x08, 0x44, 0x81, + 0x1a, 0x29, 0x20, 0x15, 0x69, 0x10, 0x14, 0x7d, 0x20, 0x4d, 0x6c, 0x26, 0x11, 0x90, 0x48, 0xe9, + 0x4a, 0x46, 0xd1, 0x5e, 0x88, 0x15, 0xb9, 0xa6, 0x88, 0xf0, 0x21, 0x2c, 0x49, 0xd5, 0x3e, 0xf6, + 0xde, 0x73, 0xbe, 0x42, 0x3f, 0x60, 0x9b, 0x9c, 0x8b, 0x59, 0x2e, 0x25, 0xca, 0x76, 0x80, 0x3e, + 0x4e, 0xe2, 0xfc, 0xe6, 0x37, 0xb3, 0xb3, 0x33, 0xb3, 0x33, 0x82, 0x6e, 0x94, 0x78, 0x3c, 0x1c, + 0xad, 0x45, 0x92, 0x25, 0xa4, 0x25, 0x85, 0xc1, 0x37, 0x7e, 0x90, 0xad, 0xf2, 0xe5, 0xc8, 0x4d, + 0xa2, 0xb1, 0x9f, 0x84, 0x2c, 0xf6, 0xc7, 0x52, 0xbf, 0xcc, 0xcf, 0xc6, 0xeb, 0xec, 0x62, 0xcd, + 0xd3, 0x71, 0x16, 0x44, 0x3c, 0xcd, 0x58, 0xb4, 0xde, 0x7d, 0x15, 0x3e, 0xac, 0x5f, 0x1b, 0xa0, + 0x4d, 0xe2, 0x34, 0x63, 0xb1, 0xcb, 0x49, 0x1f, 0xea, 0x81, 0x67, 0xd6, 0x86, 0xb5, 0x23, 0x9d, + 0xd6, 0x03, 0x8f, 0x0c, 0x40, 0x4b, 0x43, 0xb6, 0xe1, 0x4e, 0xe0, 0x99, 0x75, 0x89, 0x6e, 0x65, + 0xf2, 0x10, 0x20, 0x64, 0x69, 0xe6, 0xa4, 0x19, 0xcb, 0xb8, 0xd9, 0x1c, 0xd6, 0x8e, 0xba, 0x0f, + 0x6e, 0x8d, 0x8a, 0xf0, 0x4a, 0x87, 0x73, 0xd4, 0xd1, 0x0a, 0x8f, 0x7c, 0x0d, 0xe0, 0x0a, 0xce, + 0x32, 0xee, 0x39, 0x2c, 0x33, 0x5b, 0xd2, 0x6a, 0x30, 0xf2, 0x93, 0xc4, 0x0f, 0xf9, 0xa8, 0x8c, + 0x7a, 0xb4, 0x28, 0x83, 0xa4, 0x15, 0x36, 0xf9, 0x1c, 0xb4, 0x8c, 0x47, 0xeb, 0x10, 0xcf, 0x6b, + 0x4b, 0xcb, 0x1b, 0xea, 0xbc, 0x85, 0x82, 0xe9, 0x96, 0x40, 0xbe, 0x83, 0x3e, 0xfe, 0xa6, 0x99, + 0x73, 0xc6, 0x82, 0x30, 0x17, 0xdc, 0xec, 0x48, 0x93, 0xff, 0x2b, 0x93, 0x67, 0x05, 0xfa, 0x8a, + 0xa7, 0x29, 0xf3, 0x39, 0xbd, 0x44, 0x26, 0x9f, 0xc2, 0x21, 0xcb, 0xb3, 0xc4, 0x11, 0xdc, 0x4d, + 0x36, 0x5c, 0x5c, 0x98, 0xda, 0xb0, 0x76, 0xa4, 0xd1, 0x7d, 0x90, 0xd8, 0x70, 0xd3, 0x4d, 0xe2, + 0x98, 0xbb, 0x59, 0x90, 0xc4, 0xf2, 0x86, 0x79, 0x6a, 0xea, 0xf2, 0x9c, 0x8f, 0xd4, 0x39, 0xc7, + 0x5b, 0xfd, 0x5c, 0xaa, 0xe9, 0x55, 0x0b, 0x6b, 0x03, 0xc6, 0x65, 0x1a, 0x79, 0x04, 0x6d, 0xe5, + 0x0f, 0xcb, 0xd1, 0x7f, 0x70, 0xf7, 0x03, 0xfe, 0x46, 0xca, 0xad, 0x62, 0x5b, 0xf7, 0xa1, 0xad, + 0x3c, 0x1c, 0x82, 0x7e, 0x3c, 0x9b, 0x4e, 0xed, 0xe3, 0x85, 0x7d, 0x62, 0x1c, 0x90, 0x9b, 0x70, + 0x38, 0x9d, 0x2d, 0x9c, 0x1d, 0x54, 0xb3, 0xde, 0xd6, 0xe1, 0x70, 0xaf, 0x54, 0xe4, 0x0b, 0x68, + 0x15, 0xf5, 0x2c, 0x0e, 0x1d, 0x5c, 0x57, 0xcf, 0x51, 0x51, 0xd5, 0xd6, 0x75, 0x05, 0xad, 0xff, + 0x93, 0x82, 0x5a, 0x6f, 0x6b, 0xd0, 0x2a, 0xce, 0xed, 0x03, 0x50, 0xfb, 0xf9, 0x64, 0xbe, 0xb0, + 0xa9, 0x0c, 0x16, 0xa0, 0xfd, 0xc3, 0xa9, 0x7d, 0x8a, 0x51, 0x92, 0x1e, 0x68, 0xf3, 0xc5, 0x13, + 0xba, 0x98, 0x4c, 0x9f, 0x1b, 0x75, 0xd2, 0x85, 0x0e, 0x3d, 0x9d, 0x4e, 0x51, 0x68, 0x14, 0xaa, + 0xd9, 0xeb, 0xd7, 0x28, 0x35, 0x51, 0x25, 0x25, 0xfb, 0xc4, 0x68, 0xe1, 0xed, 0xa9, 0xfd, 0x74, + 0x36, 0x93, 0x66, 0x6d, 0x62, 0x40, 0x6f, 0xfe, 0xe2, 0x74, 0x81, 0xd2, 0xc9, 0xec, 0xc7, 0xa9, + 0xd1, 0xc1, 0x23, 0x17, 0x36, 0x7d, 0x35, 0x99, 0x3e, 0xc1, 0x64, 0x68, 0x78, 0xe4, 0xb3, 0x27, + 0x93, 0x97, 0xf6, 0x89, 0xa1, 0x5b, 0xef, 0x6a, 0xd0, 0xdf, 0x6f, 0x10, 0xf2, 0x18, 0x80, 0x0b, + 0x91, 0x08, 0x07, 0x1f, 0x94, 0x4a, 0xcf, 0xbd, 0x6b, 0x7b, 0x69, 0x64, 0x23, 0x6f, 0x71, 0xb1, + 0xe6, 0xb4, 0x62, 0x42, 0xbe, 0x02, 0x1d, 0x9b, 0xeb, 0xef, 0xe6, 0x69, 0x47, 0xb6, 0x56, 0xa0, + 0x6f, 0x5d, 0x92, 0x1b, 0xd0, 0x2d, 0xc2, 0x74, 0xf0, 0x72, 0xc6, 0x01, 0xde, 0x4c, 0x01, 0x32, + 0x4b, 0x46, 0xad, 0x42, 0xc1, 0x74, 0x18, 0x75, 0x2c, 0xbd, 0x02, 0x8a, 0x94, 0x18, 0x0d, 0x72, + 0x0b, 0x0c, 0x05, 0x6d, 0x93, 0x60, 0x34, 0xad, 0x3f, 0x6a, 0xa0, 0x95, 0x6f, 0x89, 0x58, 0xd0, + 0x2b, 0x5f, 0x93, 0x93, 0x8b, 0x40, 0x8d, 0x85, 0x3d, 0x8c, 0xdc, 0x87, 0x66, 0x9c, 0xc4, 0xdc, + 0xfc, 0xb3, 0x21, 0x2f, 0xf4, 0x3f, 0x95, 0x90, 0x69, 0x12, 0xf3, 0xd2, 0xcf, 0x8b, 0x03, 0x2a, + 0x39, 0xe4, 0x33, 0x68, 0x84, 0xe7, 0xae, 0xf9, 0xae, 0xa0, 0x12, 0x45, 0x7d, 0x79, 0xee, 0x56, + 0x98, 0xc8, 0x90, 0x4e, 0xf3, 0x30, 0x34, 0xdf, 0x5f, 0x72, 0x9a, 0x87, 0xe1, 0x9e, 0xd3, 0x3c, + 0x0c, 0xff, 0x4b, 0xfb, 0x3d, 0x6d, 0x43, 0x73, 0x92, 0xf1, 0xc8, 0xea, 0x43, 0xaf, 0x1a, 0xb0, + 0xf5, 0x7b, 0x07, 0xba, 0x95, 0xb0, 0x08, 0x81, 0xe6, 0xc6, 0x5d, 0xe7, 0x32, 0x01, 0x2d, 0x2a, + 0xbf, 0xc9, 0x1d, 0xd0, 0x23, 0x1e, 0x25, 0xe2, 0xc2, 0xf1, 0x97, 0xf2, 0xd8, 0x16, 0xdd, 0x01, + 0x38, 0x37, 0xa3, 0x20, 0x76, 0xa4, 0x55, 0x43, 0x2a, 0xb7, 0x32, 0x4e, 0x16, 0xfc, 0xde, 0x59, + 0x37, 0x25, 0x61, 0x1f, 0x24, 0x8f, 0x40, 0x0f, 0xcf, 0x5d, 0x27, 0x88, 0x98, 0xcf, 0xd5, 0x98, + 0x34, 0xaf, 0x66, 0x6c, 0x34, 0x41, 0x3d, 0xdd, 0x51, 0xc9, 0xb7, 0x00, 0x41, 0x9c, 0x71, 0x71, + 0xc6, 0x5c, 0x9e, 0x9a, 0xed, 0x61, 0xe3, 0xa8, 0xfb, 0xe0, 0xce, 0x75, 0x86, 0x25, 0x89, 0x56, + 0xf8, 0xe4, 0x31, 0xf4, 0xd0, 0xd5, 0x76, 0xca, 0x16, 0x23, 0xf3, 0xe3, 0x6b, 0xec, 0xb7, 0x13, + 0x77, 0xcf, 0x80, 0x0c, 0xa1, 0x1b, 0x27, 0x1e, 0x77, 0x7c, 0x91, 0xe4, 0xeb, 0xd4, 0xd4, 0x86, + 0x8d, 0x23, 0x9d, 0x56, 0xa1, 0x01, 0x87, 0x96, 0x0c, 0x1a, 0xdb, 0xcb, 0x4b, 0x7e, 0x89, 0xc3, + 0x84, 0x79, 0x4e, 0x2e, 0xc2, 0xb2, 0xbd, 0xaa, 0x18, 0xba, 0x73, 0x57, 0x6f, 0xd2, 0x3c, 0x2a, + 0x5e, 0x5d, 0xb1, 0x82, 0xaa, 0x10, 0xb9, 0x0d, 0xed, 0x42, 0x94, 0x79, 0xd6, 0xa9, 0x92, 0x06, + 0xa7, 0xa0, 0x6f, 0xaf, 0x88, 0x05, 0xdc, 0xbe, 0x5a, 0x9d, 0xca, 0x6f, 0x62, 0x42, 0x27, 0x62, + 0x2e, 0xf3, 0x3c, 0xa1, 0xdc, 0x96, 0x22, 0x16, 0x2f, 0x58, 0x6f, 0x1e, 0x4a, 0x55, 0xe1, 0x74, + 0x2b, 0x0f, 0x7e, 0x6b, 0x54, 0x1e, 0xc8, 0xa0, 0xb2, 0x8f, 0x0a, 0xd7, 0xbb, 0xf5, 0x63, 0x42, + 0x67, 0xc9, 0xdc, 0x37, 0x3c, 0xf6, 0x54, 0x77, 0x94, 0x22, 0x46, 0xec, 0x05, 0x69, 0x26, 0x92, + 0x32, 0xe2, 0x42, 0x42, 0x0b, 0xc1, 0x43, 0xce, 0xd2, 0x62, 0x99, 0xea, 0xb4, 0x14, 0x31, 0x7c, + 0x26, 0xdc, 0x95, 0x6c, 0x03, 0x9d, 0xca, 0x6f, 0x64, 0x6f, 0x98, 0x08, 0x58, 0x9c, 0xc9, 0x55, + 0xa8, 0xd3, 0x52, 0x44, 0xff, 0x29, 0x17, 0x1b, 0x2e, 0x64, 0xf5, 0x74, 0xaa, 0x24, 0xc4, 0xdf, + 0xf0, 0x0b, 0xdc, 0xe4, 0x5a, 0x81, 0x17, 0x12, 0xb9, 0x0b, 0x80, 0x5f, 0xca, 0x46, 0x97, 0xba, + 0x0a, 0x42, 0x1e, 0xc1, 0x6d, 0x2f, 0x48, 0xd9, 0x32, 0xe4, 0x8e, 0xbf, 0xf6, 0x9d, 0x0d, 0x0b, + 0x03, 0x8f, 0xe1, 0xfe, 0x31, 0x41, 0xae, 0xc4, 0x0f, 0x68, 0xb1, 0x76, 0x67, 0x61, 0x9e, 0xae, + 0x1c, 0x97, 0xb9, 0x2b, 0x6e, 0x76, 0x25, 0xb9, 0x0a, 0x49, 0x46, 0x22, 0x5c, 0xae, 0x18, 0x3d, + 0xc5, 0xd8, 0x41, 0x18, 0x1b, 0x3f, 0xcf, 0x04, 0x73, 0x98, 0xf0, 0x53, 0xf3, 0x50, 0x76, 0x53, + 0x05, 0xb1, 0xde, 0xd7, 0xa0, 0x57, 0x1d, 0x0b, 0xff, 0xe2, 0xa9, 0x7e, 0x0f, 0x5d, 0x57, 0xb0, + 0x74, 0x85, 0xbb, 0xd8, 0xe7, 0xb2, 0x26, 0xbb, 0x65, 0x5b, 0xf5, 0x3d, 0x3a, 0x46, 0xda, 0x1c, + 0x59, 0xb4, 0x6a, 0x72, 0xb9, 0xe7, 0x9b, 0x57, 0x7a, 0xde, 0x7a, 0x0d, 0xb0, 0x33, 0x26, 0x1a, + 0x34, 0xa7, 0xb3, 0xa9, 0x6d, 0x1c, 0x10, 0x1d, 0x5a, 0xe5, 0xcc, 0xd6, 0xa0, 0xa9, 0x86, 0x35, + 0x40, 0xfb, 0x98, 0xda, 0x38, 0x8f, 0x1b, 0xb8, 0xd1, 0x4e, 0xec, 0xf9, 0x82, 0xce, 0x7e, 0x32, + 0x9a, 0xa8, 0x50, 0xe3, 0xbb, 0xf5, 0xf4, 0xde, 0xcf, 0x9f, 0x54, 0xfe, 0xf4, 0xb1, 0xf3, 0x74, + 0x35, 0x4e, 0xd6, 0x3c, 0xde, 0x78, 0xee, 0x58, 0x46, 0xbe, 0x6c, 0xcb, 0xd9, 0xf7, 0xe5, 0x5f, + 0x01, 0x00, 0x00, 0xff, 0xff, 0x3b, 0xf0, 0x6b, 0x1c, 0x30, 0x0a, 0x00, 0x00, } diff --git a/model/nodes.go b/model/nodes.go new file mode 100644 index 00000000..ef974dfa --- /dev/null +++ b/model/nodes.go @@ -0,0 +1,139 @@ +package model + +import ( + "fmt" + + "github.com/golang/protobuf/proto" + "golang.org/x/net/context" +) + +const nodesBaseKey = "nodes" + +func init() { + schemaKeys = append(schemaKeys, nodesBaseKey) +} + +type Node interface { + proto.Message + GetAgentID() string + GetAgentMesosID() string +} + +type NodeOps interface { + Add(node Node) error + FindByAgentMesosID(agentMesosID string) (*AgentNode, error) + FindByAgentID(agentID string) (*AgentNode, error) + Filter(limit int, cb func(*AgentNode) int) error + UpdateAgentMesosID(agentID string, agentMesosID string) error +} + +type nodes struct { + base +} + +func Nodes(ctx context.Context) NodeOps { + return &nodes{base{ctx: ctx}} +} + +func (i *nodes) Add(n Node) error { + if n.GetAgentID() == "" { + return fmt.Errorf("ID is not set") + } + + agentNode, _ := i.FindByAgentID(n.GetAgentID()) + + if agentNode != nil { + return nil + } + + bk, err := i.connection() + if err != nil { + return err + } + + buf, err := proto.Marshal(n) + if err != nil { + return err + } + + if err = bk.Backend().Create(fmt.Sprintf("%s/%s", nodesBaseKey, n.GetAgentID()), buf); err != nil { + return nil + } + + return nil +} + +func (i *nodes) FindByAgentID(agentID string) (*AgentNode, error) { + bk, err := i.connection() + if err != nil { + return nil, err + } + n := &AgentNode{} + if err := bk.Find(fmt.Sprintf("/%s/%s", nodesBaseKey, agentID), n); err != nil { + return nil, err + } + + return n, nil +} + +func (i *nodes) FindByAgentMesosID(agentMesosID string) (*AgentNode, error) { + res := []*AgentNode{} + err := i.Filter(1, func(node *AgentNode) int { + if node.GetAgentMesosID() == agentMesosID { + res = append(res, node) + } + return len(res) + }) + if err != nil { + return nil, err + } + + if len(res) > 0 { + return res[0], nil + } else { + return nil, nil + } +} + +func (i *nodes) Filter(limit int, cb func(*AgentNode) int) error { + bk, err := i.connection() + if err != nil { + return err + } + keys, err := bk.Keys(fmt.Sprintf("/%s", nodesBaseKey)) + if err != nil { + return err + } + for keys.Next() { + node, err := i.FindByAgentID(keys.Value()) + if err != nil { + return err + } + if limit > 0 && cb(node) > limit { + break + } else { + cb(node) + } + } + return nil +} + +func (i *nodes) UpdateAgentMesosID(agentID string, agentMesosID string) error { + node, err := i.FindByAgentID(agentID) + if err != nil { + return err + } + + node.Agentmesosid = agentMesosID + + bk, err := i.connection() + if err != nil { + return err + } + + err = bk.Update(fmt.Sprintf("/%s/%s/", nodesBaseKey, node.Agentid), node) + if err != nil { + return err + } + return nil +} diff --git a/model/nodes.pb.go b/model/nodes.pb.go new file mode 100644 index 00000000..299cd7d8 --- /dev/null +++ b/model/nodes.pb.go @@ -0,0 +1,85 @@ +// Code generated by protoc-gen-go. +// source: nodes.proto +// DO NOT EDIT! + +/* +Package model is a generated protocol buffer package. + +It is generated from these files: + nodes.proto + +It has these top-level messages: + AgentNode +*/ +package model + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" +import google_protobuf "github.com/golang/protobuf/ptypes/timestamp" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package + +type AgentNode struct { + Agentmesosid string `protobuf:"bytes,1,opt,name=agentmesosid" json:"agentmesosid,omitempty"` + Agentid string `protobuf:"bytes,2,opt,name=agentid" json:"agentid,omitempty"` + CreatedAt *google_protobuf.Timestamp `protobuf:"bytes,3,opt,name=created_at,json=createdAt" json:"created_at,omitempty"` +} + +func (m *AgentNode) Reset() { *m = AgentNode{} } +func (m *AgentNode) String() string { return proto.CompactTextString(m) } +func (*AgentNode) ProtoMessage() {} +func (*AgentNode) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } + +func (m *AgentNode) GetAgentID() string { + if m != nil { + return m.Agentid + } + return "" +} + +func (m *AgentNode) GetAgentMesosID() string { + if m != nil { + return m.Agentmesosid + } + return "" +} + +func (m *AgentNode) GetCreatedAt() *google_protobuf.Timestamp { + if m != nil { + return m.CreatedAt + } + return nil +} + +func init() { + proto.RegisterType((*AgentNode)(nil), "model.AgentNode") +} + +func init() { proto.RegisterFile("nodes.proto", fileDescriptor2) } + +var fileDescriptor2 = []byte{ + // 197 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x4c, 0x4f, 0x3d, 0x6b, 0xc3, 0x30, + 0x10, 0xc5, 0xfd, 0xc4, 0xf2, 0xa6, 0xc9, 0x18, 0x4a, 0x4b, 0xa7, 0x4e, 0x12, 0xb4, 0x53, 0xe9, + 0xe4, 0xfe, 0x80, 0x0e, 0xa6, 0x53, 0x97, 0x22, 0x5b, 0x17, 0xd9, 0x60, 0xe9, 0x84, 0x75, 0x0a, + 0xc9, 0xbf, 0x8f, 0xa2, 0xc4, 0x24, 0xdb, 0xfb, 0xd2, 0xd3, 0x3b, 0x56, 0x39, 0xd4, 0x10, 0x84, + 0x5f, 0x90, 0x90, 0xdf, 0xdb, 0x44, 0xe6, 0xe6, 0xcb, 0x4c, 0x34, 0xc6, 0x5e, 0x0c, 0x68, 0xa5, + 0xc1, 0x59, 0x39, 0x23, 0xb3, 0xdf, 0xc7, 0x8d, 0xf4, 0xb4, 0xf7, 0x10, 0x24, 0x4d, 0x16, 0x02, + 0x29, 0xeb, 0x2f, 0xe8, 0xd4, 0xf1, 0x4a, 0xac, 0x6c, 0x0d, 0x38, 0xfa, 0x49, 0x55, 0x9c, 0xb3, + 0xbb, 0x18, 0x27, 0x5d, 0x17, 0x2f, 0xc5, 0x5b, 0xd9, 0x65, 0xcc, 0x6b, 0xf6, 0xa8, 0x8e, 0x81, + 0x24, 0xdf, 0x64, 0x79, 0xa5, 0xfc, 0x93, 0xb1, 0x61, 0x01, 0x45, 0xa0, 0xff, 0x15, 0xd5, 0xb7, + 0xc9, 0xac, 0xde, 0x1b, 0x61, 0x10, 0xcd, 0x0c, 0x62, 0x5d, 0x20, 0x7e, 0xd7, 0x0f, 0xbb, 0xf2, + 0x9c, 0x6e, 0xe9, 0xfb, 0xf9, 0xef, 0xe9, 0x6a, 0xb4, 0xda, 0x85, 0x51, 0xa2, 0x07, 0xb7, 0xd5, + 0x83, 0xcc, 0x37, 0xf5, 0x0f, 0xf9, 0xfd, 0xc7, 0x21, 0x00, 0x00, 0xff, 0xff, 0x72, 0xc4, 0x43, + 0x94, 0xf0, 0x00, 0x00, 0x00, +} diff --git a/pkg/conf/executor.toml b/pkg/conf/executor.toml index d6316d7f..00f4a619 100644 --- a/pkg/conf/executor.toml +++ b/pkg/conf/executor.toml @@ -1,5 +1,4 @@ [hypervisor] -# driver = "null" # script-path = "/etc/openvdc/scripts/" # image-server-uri = "http://127.0.0.1/images" # cache-path = "/var/cache/lxc/" diff --git a/proto/cluster.proto b/proto/cluster.proto index 56049de4..0af1e6b9 100644 --- a/proto/cluster.proto +++ b/proto/cluster.proto @@ -21,6 +21,7 @@ message ExecutorNode { Console console = 3; string grpc_addr = 4 [json_name="grpc_addr"]; NodeState last_state = 5 [json_name="last_state"]; + string node_id = 6 [json_name="node_id"]; } message SchedulerNode { diff --git a/proto/crashed_nodes.proto b/proto/crashed_nodes.proto new file mode 100644 index 00000000..4c90b5bb --- /dev/null +++ b/proto/crashed_nodes.proto @@ -0,0 +1,16 @@ +syntax = "proto3"; + +import "github.com/golang/protobuf/ptypes/timestamp/timestamp.proto"; + +package model; + +option go_package = "github.com/axsh/openvdc/model"; + +message CrashedNode { + string uuid = 1; + string agent_id = 2; + string agent_mesos_id = 3; + bool reconnected = 4; + google.protobuf.Timestamp created_at = 5; + google.protobuf.Timestamp reconnected_at = 6; +} diff --git a/proto/model.proto b/proto/model.proto index c72333e8..949e8482 100644 --- a/proto/model.proto +++ b/proto/model.proto @@ -13,7 +13,17 @@ message Instance { InstanceState last_state = 4 [json_name="last_state"]; google.protobuf.Timestamp created_at = 5 [json_name="created_at"]; Template template = 6; - FailureMessage latest_failure = 7; + FailureMessage latest_failure = 7 [json_name="latest_failure"]; + bool auto_recovery = 8 [json_name="auto_recovery"]; + ConnectionStatus connection_status = 9 [json_name="connection_status"]; +} + +message ConnectionStatus { + enum Status { + CONNECTED = 0; + NOT_CONNECTED = 1; + } + Status status = 1; } message InstanceState { @@ -41,8 +51,8 @@ message FailureMessage { FAILED_REBOOT = 3; FAILED_TERMINATE = 4; } - ErrorType error_type = 1; - google.protobuf.Timestamp failed_at = 2; + ErrorType error_type = 1 [json_name="error_type"]; + google.protobuf.Timestamp failed_at = 2 [json_name="failed_at"]; } message Template { diff --git a/proto/nodes.proto b/proto/nodes.proto new file mode 100644 index 00000000..6abda6ce --- /dev/null +++ b/proto/nodes.proto @@ -0,0 +1,13 @@ +syntax = "proto3"; + +import "github.com/golang/protobuf/ptypes/timestamp/timestamp.proto"; + +package model; + +option go_package = "github.com/axsh/openvdc/model"; + +message AgentNode { + string agent_mesos_id = 1; + string agent_id = 2; + google.protobuf.Timestamp created_at = 3; +} diff --git a/proto/v1.proto b/proto/v1.proto index f6ddc180..61a3bff6 100644 --- a/proto/v1.proto +++ b/proto/v1.proto @@ -72,6 +72,7 @@ message ConsoleReply { message CreateRequest{ //string resource_id = 1; // Obsolete model.Template template = 2; + bool auto_recovery = 3; } message CreateReply{ string instance_id = 1 [json_name="instance_id"]; diff --git a/registry/bindata_assetfs.go b/registry/bindata_assetfs.go index 1ea7dc09..0d21b0d8 100644 --- a/registry/bindata_assetfs.go +++ b/registry/bindata_assetfs.go @@ -87,7 +87,7 @@ func schemaNoneJson() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "schema/none.json", size: 404, mode: os.FileMode(420), modTime: time.Unix(1495216653, 0)} + info := bindataFileInfo{name: "schema/none.json", size: 404, mode: os.FileMode(420), modTime: time.Unix(1492412458, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -107,7 +107,7 @@ func schemaV1Json() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "schema/v1.json", size: 668, mode: os.FileMode(420), modTime: time.Unix(1495216653, 0)} + info := bindataFileInfo{name: "schema/v1.json", size: 668, mode: os.FileMode(420), modTime: time.Unix(1492412458, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -127,7 +127,7 @@ func schemaVmLxcJson() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "schema/vm/lxc.json", size: 3522, mode: os.FileMode(420), modTime: time.Unix(1495534488, 0)} + info := bindataFileInfo{name: "schema/vm/lxc.json", size: 3522, mode: os.FileMode(420), modTime: time.Unix(1497004394, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -147,7 +147,7 @@ func schemaVmNullJson() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "schema/vm/null.json", size: 930, mode: os.FileMode(420), modTime: time.Unix(1495534663, 0)} + info := bindataFileInfo{name: "schema/vm/null.json", size: 930, mode: os.FileMode(420), modTime: time.Unix(1497004394, 0)} a := &asset{bytes: bytes, info: info} return a, nil } diff --git a/scheduler/mesos.go b/scheduler/mesos.go index ada445f0..d20d47eb 100644 --- a/scheduler/mesos.go +++ b/scheduler/mesos.go @@ -3,12 +3,9 @@ package scheduler import ( "fmt" "net" - - log "github.com/Sirupsen/logrus" - "github.com/pkg/errors" - "strings" + log "github.com/Sirupsen/logrus" "github.com/axsh/openvdc/model" "github.com/axsh/openvdc/model/backend" "github.com/gogo/protobuf/proto" @@ -18,6 +15,7 @@ import ( mesos "github.com/mesos/mesos-go/mesosproto" util "github.com/mesos/mesos-go/mesosutil" sched "github.com/mesos/mesos-go/scheduler" + "github.com/pkg/errors" "golang.org/x/net/context" ) @@ -91,77 +89,203 @@ func (sched *VDCScheduler) ResourceOffers(driver sched.SchedulerDriver, offers [ } func (sched *VDCScheduler) processOffers(driver sched.SchedulerDriver, offers []*mesos.Offer, ctx context.Context) error { - queued, err := model.Instances(ctx).FilterByState(model.InstanceState_QUEUED) - if err != nil { - return err + + checkAgents(offers, ctx) + + if sched.tasksLaunched == 0 { + sched.CheckForCrashedNodes(offers, ctx) } - if len(queued) == 0 { - log.Infoln("Skip offers since no allocation requests.") - for _, offer := range offers { - _, err := driver.DeclineOffer(offer.Id, &mesos.Filters{RefuseSeconds: proto.Float64(5)}) - if err != nil { - log.WithError(err).Error("Failed to response DeclineOffer.") + disconnected := getDisconnectedInstances(offers, ctx, driver) + + if len(disconnected) > 0 { + sched.InstancesRelaunching(driver, offers, ctx, disconnected) + } else { + sched.InstancesQueued(driver, offers, ctx) + } + + return nil +} + +func (sched *VDCScheduler) CheckForCrashedNodes(offers []*mesos.Offer, ctx context.Context) error { + for _, offer := range offers { + node, err := model.Nodes(ctx).FindByAgentID(getAgentID(offer)) + if err != nil { + log.WithError(err).Error("Failed to get node") + } + + if node == nil { + continue + } + + instances, err := model.Instances(ctx).FilterByAgentMesosID(node.GetAgentMesosID()) + + if err != nil { + return errors.Wrapf(err, "Failed to retrieve instances.") + } + + foundCrashedNode := false + + CheckInstances: + for _, instance := range instances { + if instance.GetLastState().State != model.InstanceState_REGISTERED && + instance.GetLastState().State != model.InstanceState_QUEUED && + instance.GetLastState().State != model.InstanceState_TERMINATED && + instance.GetAutoRecovery() == true { + + disconnectedAgent, err := model.CrashedNodes(ctx).FindByAgentMesosID(*offer.SlaveId.Value) + + if err != nil { + return errors.Wrapf(err, "Failed to check if crashed node existed.") + } + + if disconnectedAgent == nil || disconnectedAgent.GetReconnected() == true { + foundCrashedNode = true + break CheckInstances + } } } - return nil - } - findMatching := func(i *model.Instance) *mesos.Offer { - log := log.WithField("instance_id", i.GetId()) - for _, offer := range offers { - log := log.WithField("agent", offer.SlaveId.String()) - var agentAttrs struct { - Hypervisor string // Required - NodeGroups []string // Optional + if foundCrashedNode == true { + + agentID := getAgentID(offer) + + err := model.Nodes(ctx).UpdateAgentMesosID(agentID, *offer.SlaveId.Value) + if err != nil { + log.WithError(err).Error("Failed to update node agentMesosID. node: '%s'", agentID) } - // Read and validate attribute entries from agent offer. - for _, attr := range offer.Attributes { - switch attr.GetName() { - case "hypervisor": - if attr.GetType() != mesos.Value_TEXT { - log.Error("Invalid value type for 'hypervisor' attribute") - break + + err = model.CrashedNodes(ctx).Add(&model.CrashedNode{ + Agentid: agentID, + Agentmesosid: *offer.SlaveId.Value, + Reconnected: false, + }) + + if err != nil { + return errors.Wrapf(err, "Failed to add '%s' to lost of crashed agents.", agentID) + } + + log.Infoln("Added '%s' to list of crashed agents", agentID) + + instances, err := model.Instances(ctx).FilterByAgentMesosID(*offer.SlaveId.Value) + + if err != nil { + return errors.Wrapf(err, "Failed to retrieve instances.") + } + + if len(instances) > 0 { + for _, instance := range instances { + err := model.Instances(ctx).UpdateConnectionStatus(instance.GetId(), model.ConnectionStatus_NOT_CONNECTED) + if err != nil { + return errors.Wrapf(err, "Failed to update instance ConnectionStatus. instance: '%s' ConnectionStatus: '%s'", instance.GetId(), model.ConnectionStatus_NOT_CONNECTED) } - agentAttrs.Hypervisor = attr.GetText().GetValue() + } + } + } + } + + return nil +} + +func getDisconnectedInstances(offers []*mesos.Offer, ctx context.Context, driver sched.SchedulerDriver) []*model.Instance { + + disconnectedInstances := []*model.Instance{} + + for _, offer := range offers { + + agentID := getAgentID(offer) + disconnectedAgent, err := model.CrashedNodes(ctx).FindByAgentID(agentID) + + if err != nil { + log.WithError(err).Error("Failed to retrieve crashed node.") + } + + if disconnectedAgent != nil { + if disconnectedAgent.GetReconnected() == false { + instances, err := model.Instances(ctx).FilterByAgentMesosID(disconnectedAgent.GetAgentMesosID()) + + if err != nil { + log.WithError(err).Error("Failed to retrieve disconnected instances.") + } - case "node-groups": - if attr.GetType() == mesos.Value_TEXT { - if attr.GetText().GetValue() == "" { - log.Error("'node-groups' attribute must be non-empty string") - break + if len(instances) > 0 { + for _, instance := range instances { + connStatus := instance.GetConnectionStatus() + autoRecovery := instance.GetAutoRecovery() + + if connStatus.Status == model.ConnectionStatus_NOT_CONNECTED && autoRecovery == true { + instance.SlaveId = offer.SlaveId.GetValue() + model.Instances(ctx).Update(instance) + + log.Infoln(fmt.Sprintf("Added instance %s to relaunch-queue.", instance.GetId())) + disconnectedInstances = append(disconnectedInstances, instance) } - agentAttrs.NodeGroups = strings.Split(attr.GetText().GetValue(), ",") - } else { - log.Errorf("Invalid value type for 'bridge' attribute: %s", attr.GetText()) - break } - default: - log.Warnf("Found unsupported attribute: %s", attr.GetName()) + } + + if len(disconnectedInstances) == 0 { + model.CrashedNodes(ctx).SetReconnected(disconnectedAgent) + agentID := disconnectedAgent.GetAgentID() + log.Infoln(fmt.Sprintf("Node '%s' reconnected.", agentID)) + + err := model.Nodes(ctx).UpdateAgentMesosID(agentID, *offer.SlaveId.Value) + if err != nil { + log.WithError(err).Error("Failed to update node agentMesosID. node: '%s'", agentID) + } } } + } + } + return disconnectedInstances +} - if agentAttrs.Hypervisor == "" { - log.Error("Required attributes are not advertised from agent") +func (sched *VDCScheduler) InstancesRelaunching(driver sched.SchedulerDriver, offers []*mesos.Offer, ctx context.Context, relaunchQueued []*model.Instance) error { + + tasks := []*mesos.TaskInfo{} + acceptIDs := []*mesos.OfferID{} + + for _, instance := range relaunchQueued { + for _, offer := range offers { + if instance.SlaveId != offer.SlaveId.GetValue() { continue } - // TODO: Avoid type switch to find template types. - switch t := i.GetTemplate().GetItem(); t.(type) { - case *model.Template_Lxc: - if agentAttrs.Hypervisor == "lxc" { - lxc := i.GetTemplate().GetLxc() - if !model.IsMatchingNodeGroups(lxc, agentAttrs.NodeGroups) { - return nil - } - return offer - } - case *model.Template_Null: - if agentAttrs.Hypervisor == "null" { - return offer + alreadyAdded := false + + for i, _ := range acceptIDs { + if acceptIDs[i] == offer.Id { + alreadyAdded = true } - default: - log.Warnf("Unknown template type: %T", t) + } + + if alreadyAdded != true { + hypervisorName := getHypervisorName(offer) + model.Instances(ctx).UpdateConnectionStatus(instance.GetId(), model.ConnectionStatus_CONNECTED) + task := sched.NewTask(instance, util.NewSlaveID(instance.SlaveId), ctx, sched.NewExecutor(hypervisorName)) + tasks = append(tasks, task) + acceptIDs = append(acceptIDs, offer.Id) + } + } + } + + sched.LaunchTasks(driver, tasks, acceptIDs, offers) + sched.DeclineUnusedOffers(driver, offers, acceptIDs) + + return nil +} + +func (sched *VDCScheduler) InstancesQueued(driver sched.SchedulerDriver, offers []*mesos.Offer, ctx context.Context) error { + queued, err := model.Instances(ctx).FilterByState(model.InstanceState_QUEUED) + if err != nil { + return errors.WithStack(err) + } + + if len(queued) == 0 { + log.Infoln("Skip offers since no allocation requests.") + for _, offer := range offers { + _, err := driver.DeclineOffer(offer.Id, &mesos.Filters{RefuseSeconds: proto.Float64(5)}) + if err != nil { + log.WithError(err).Error("Failed to response DeclineOffer.") } } return nil @@ -169,8 +293,9 @@ func (sched *VDCScheduler) processOffers(driver sched.SchedulerDriver, offers [] tasks := []*mesos.TaskInfo{} acceptIDs := []*mesos.OfferID{} + for _, i := range queued { - found := findMatching(i) + found := findMatching(i, offers, ctx) for i, _ := range acceptIDs { if acceptIDs[i] == found.Id { found = nil @@ -181,46 +306,123 @@ func (sched *VDCScheduler) processOffers(driver sched.SchedulerDriver, offers [] continue } - hypervisorName := strings.TrimPrefix(i.GetTemplate().ResourceTemplate().ResourceName(), "vm/") + hypervisorName := getHypervisorName(found) log.WithFields(log.Fields{ "instance_id": i.GetId(), "hypervisor": hypervisorName, }).Info("Found matching offer") - executor := &mesos.ExecutorInfo{ - ExecutorId: util.NewExecutorID(fmt.Sprintf("vdc-hypervisor-%s", hypervisorName)), - Name: proto.String("VDC Executor"), - Command: &mesos.CommandInfo{ - Value: proto.String(fmt.Sprintf("%s --hypervisor=%s --zk=%s", - ExecutorPath, hypervisorName, sched.zkAddr.String())), - }, - } - - taskId := util.NewTaskID(i.GetId()) - task := &mesos.TaskInfo{ - Name: proto.String("VDC" + "_" + taskId.GetValue()), - TaskId: taskId, - SlaveId: found.SlaveId, - Data: []byte("instance_id=" + i.GetId()), - Executor: executor, - Resources: []*mesos.Resource{ - util.NewScalarResource("cpus", float64(i.GetTemplate().GetLxc().GetVcpu())), - util.NewScalarResource("mem", float64(i.GetTemplate().GetLxc().GetMemoryGb()*1024)), - }, - } - + task := sched.NewTask(i, found.SlaveId, ctx, sched.NewExecutor(hypervisorName)) tasks = append(tasks, task) acceptIDs = append(acceptIDs, found.Id) - // Associate mesos Slave ID to the instance. i.SlaveId = found.SlaveId.GetValue() model.Instances(ctx).Update(i) } - _, err = driver.LaunchTasks(acceptIDs, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(5)}) + + sched.LaunchTasks(driver, tasks, acceptIDs, offers) + sched.DeclineUnusedOffers(driver, offers, acceptIDs) + + return nil +} + +func (sched *VDCScheduler) NewTask(i *model.Instance, slaveID *mesos.SlaveID, ctx context.Context, executor *mesos.ExecutorInfo) *mesos.TaskInfo { + taskId := util.NewTaskID(i.GetId()) + task := &mesos.TaskInfo{ + Name: proto.String("VDC" + "_" + taskId.GetValue()), + TaskId: taskId, + SlaveId: slaveID, + Data: []byte("instance_id=" + i.GetId()), + Executor: executor, + Resources: []*mesos.Resource{ + util.NewScalarResource("cpus", float64(i.GetTemplate().GetLxc().GetVcpu())), + util.NewScalarResource("mem", float64(i.GetTemplate().GetLxc().GetMemoryGb()*1024)), + }, + } + return task +} + +func (sched *VDCScheduler) NewExecutor(hypervisorName string) *mesos.ExecutorInfo { + executor := &mesos.ExecutorInfo{ + ExecutorId: util.NewExecutorID(fmt.Sprintf("vdc-hypervisor-%s", hypervisorName)), + Name: proto.String("VDC Executor"), + Command: &mesos.CommandInfo{ + Value: proto.String(fmt.Sprintf("%s --zk=%s", + ExecutorPath, sched.zkAddr.String())), + }, + } + return executor +} + +func (sched *VDCScheduler) LaunchTasks(driver sched.SchedulerDriver, tasks []*mesos.TaskInfo, acceptIDs []*mesos.OfferID, offers []*mesos.Offer) error { + _, err := driver.LaunchTasks(acceptIDs, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(5)}) if err != nil { - log.WithError(err).Error("Faild to response LaunchTasks.") + return errors.Wrapf(err, "Failed to launch tasks. Tasks: %s", tasks) + } + sched.tasksLaunched++ + + return nil +} + +func findMatching(i *model.Instance, offers []*mesos.Offer, ctx context.Context) *mesos.Offer { + for _, offer := range offers { + log := log.WithField("agent", offer.SlaveId.String()) + var agentAttrs struct { + Hypervisor string // Required + NodeGroups []string // Optional + } // Read and validate attribute entries from agent offer. + for _, attr := range offer.Attributes { + switch attr.GetName() { + case "hypervisor": + if attr.GetType() != mesos.Value_TEXT { + log.Error("Invalid value type for 'hypervisor' attribute") + break + } + agentAttrs.Hypervisor = attr.GetText().GetValue() + + case "node-groups": + if attr.GetType() == mesos.Value_TEXT { + if attr.GetText().GetValue() == "" { + log.Error("'node-groups' attribute must be non-empty string") + break + } + agentAttrs.NodeGroups = strings.Split(attr.GetText().GetValue(), ",") + } else { + log.Errorf("Invalid value type for 'bridge' attribute: %s", attr.GetText()) + break + } + default: + log.Warnf("Found unsupported attribute: %s", attr.GetName()) + } + } + + if agentAttrs.Hypervisor == "" { + log.Error("Required attributes are not advertised from agent") + continue + } + + // TODO: Avoid type switch to find template types. + switch t := i.GetTemplate().GetItem(); t.(type) { + case *model.Template_Lxc: + if agentAttrs.Hypervisor == "lxc" { + lxc := i.GetTemplate().GetLxc() + if !model.IsMatchingNodeGroups(lxc, agentAttrs.NodeGroups) { + return nil + } + return offer + } + case *model.Template_Null: + if agentAttrs.Hypervisor == "null" { + return offer + } + default: + log.Warnf("Unknown template type: %T", t) + } } + return nil +} +func (sched *VDCScheduler) DeclineUnusedOffers(driver sched.SchedulerDriver, offers []*mesos.Offer, acceptIDs []*mesos.OfferID) { exists := func(s []*mesos.OfferID, i *mesos.OfferID) bool { for _, o := range s { if o.GetValue() == i.GetValue() { @@ -229,17 +431,53 @@ func (sched *VDCScheduler) processOffers(driver sched.SchedulerDriver, offers [] } return false } + for _, offer := range offers { - if !exists(acceptIDs, offer.GetId()) { + if !exists(acceptIDs, offer.Id) { _, err := driver.DeclineOffer(offer.Id, &mesos.Filters{RefuseSeconds: proto.Float64(5)}) if err != nil { - log.WithError(err).Error("Failed to response DeclineOffer.") + log.WithError(err).Error("Failed to decline offer.") } } } +} + +func checkAgents(offers []*mesos.Offer, ctx context.Context) error { + for _, offer := range offers { + slaveID := offer.SlaveId.GetValue() + agentID := getAgentID(offer) + + err := model.Nodes(ctx).Add(&model.AgentNode{ + Agentmesosid: slaveID, + Agentid: agentID, + }) + if err != nil { + return err + } + } return nil } +func getAgentID(offer *mesos.Offer) string { + for _, attr := range offer.Attributes { + if attr.GetName() == "openvdc-node-id" && + attr.GetType() == mesos.Value_TEXT { + return attr.GetText().GetValue() + } + } + return "" +} + +func getHypervisorName(offer *mesos.Offer) string { + for _, attr := range offer.Attributes { + if attr.GetName() == "hypervisor" && + attr.GetType() == mesos.Value_TEXT { + return attr.GetText().GetValue() + } + } + return "" +} + func (sched *VDCScheduler) StatusUpdate(driver sched.SchedulerDriver, status *mesos.TaskStatus) { log.Println("Framework Resource Offers from master", status) @@ -253,6 +491,7 @@ func (sched *VDCScheduler) StatusUpdate(driver sched.SchedulerDriver, status *me status.GetState() == mesos.TaskState_TASK_FAILED || status.GetState() == mesos.TaskState_TASK_KILLED { sched.tasksErrored++ + driver.ReviveOffers() } } @@ -266,10 +505,55 @@ func (sched *VDCScheduler) FrameworkMessage(_ sched.SchedulerDriver, eid *mesos. func (sched *VDCScheduler) SlaveLost(_ sched.SchedulerDriver, sid *mesos.SlaveID) { log.Errorln("slave lost: %v", sid) + + agentMesosID := *sid.Value + + ctx, err := model.Connect(context.Background(), sched.zkAddr) + if err != nil { + log.WithError(err).Error("Failed model.Connect") + } + + instances, err := model.Instances(ctx).FilterByAgentMesosID(agentMesosID) + + if err != nil { + log.WithError(err).Error("Failed to retrieve instances.") + } + + if len(instances) > 0 { + for _, instance := range instances { + err := model.Instances(ctx).UpdateConnectionStatus(instance.GetId(), model.ConnectionStatus_NOT_CONNECTED) + if err != nil { + log.WithError(err).Error("Failed to update instance ConnectionStatus. instance: '%s' ConnectionStatus: '%s'", instance.GetId(), model.ConnectionStatus_NOT_CONNECTED) + } + } + } + + res, err := model.Nodes(ctx).FindByAgentMesosID(agentMesosID) + if err != nil { + log.WithError(err).Error("Failed to fetch agent nodes") + } + + if res != nil { + + agentID := res.Agentid + + err = model.CrashedNodes(ctx).Add(&model.CrashedNode{ + Agentid: agentID, + Agentmesosid: agentMesosID, + Reconnected: false, + }) + + if err != nil { + log.WithError(err).Error("Failed to add '%s' to list of crashed agents", agentID) + } + + log.Infoln("Added '%s' to list of crashed agents", agentID) + } } func (sched *VDCScheduler) ExecutorLost(_ sched.SchedulerDriver, eid *mesos.ExecutorID, sid *mesos.SlaveID, code int) { log.Errorln("executor %q lost on slave %q code %d", eid, sid, code) + } func (sched *VDCScheduler) Error(_ sched.SchedulerDriver, err string) { diff --git a/vendor/vendor.json b/vendor/vendor.json index 1965e9ea..99fb4feb 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -671,5 +671,5 @@ "revisionTime": "2016-09-28T15:37:09Z" } ], - "rootPath": "/github.com/axsh/openvdc" + "rootPath": "github.com/axsh/openvdc" }