From 45e405b9807356d9567550fd2756805030d524c8 Mon Sep 17 00:00:00 2001 From: Fletcher Haynes Date: Thu, 17 Nov 2022 08:58:22 -0800 Subject: [PATCH] Fixed some import issues resulting from syncing the private repo --- api.go | 1 - batch/batch.go | 10 +- batch/batch_test.go | 6 +- batch/convert.go | 4 +- batch/egpool/egpool_test.go | 2 +- batch/importer.go | 8 +- batch/metrics.go | 2 +- client/api.go | 4 +- client/importer.go | 4 +- cluster.go | 1 - cmd/badloader/badloader.go | 12 +- encoding/proto/proto.go | 1 - encoding/proto/proto_test.go | 1 - fragment.go | 3 - handler.go | 5 +- handler_test.go | 2 +- http_handler.go | 1 - internal_client.go | 1 - iterator.go | 1 - querycontext/query_context.go | 2 +- querycontext/query_context_test.go | 2 +- querycontext/rbf.go | 6 +- querycontext/rbf_test.go | 2 +- querycontext/reduce_err_test.go | 2 +- rbf.go | 1 - server/server.go | 178 +++++++++++++++++-- sql/show.go | 4 +- sql3/planner/executionplanner.go | 2 + sql3/planner/executionplannersystemtables.go | 6 +- sql3/planner/expressionanalyzer.go | 2 + sql3/planner/opbulkinsert.go | 4 +- sql3/planner/opinsert.go | 4 +- sql3/planner/opsystemtable.go | 8 +- sql3/test/defs/defs_aggregate.go | 21 ++- sql3/test/defs/defs_binops.go | 2 +- sql3/test/defs/defs_unkeyed.go | 2 +- sql3/test/defs/types.go | 4 +- translate.go | 1 - 38 files changed, 238 insertions(+), 84 deletions(-) diff --git a/api.go b/api.go index a2dc92b4a..396b5fa5a 100644 --- a/api.go +++ b/api.go @@ -23,7 +23,6 @@ import ( "time" "github.com/featurebasedb/featurebase/v3/disco" - "github.com/featurebasedb/featurebase/v3/ingest" "github.com/featurebasedb/featurebase/v3/rbf" //"github.com/featurebasedb/featurebase/v3/pg" diff --git a/batch/batch.go b/batch/batch.go index 4ffc7036a..4f01a1857 100644 --- a/batch/batch.go +++ b/batch/batch.go @@ -1,6 +1,5 @@ -// Copyright 2022 Molecula Corp. (DBA FeatureBase). -// SPDX-License-Identifier: Apache-2.0 -package client +// Package batch provides tooling to prepare batches of records for ingest. +package batch import ( "bytes" @@ -11,8 +10,9 @@ import ( "time" featurebase "github.com/featurebasedb/featurebase/v3" - "github.com/featurebasedb/featurebase/v3/client/egpool" + "github.com/featurebasedb/featurebase/v3/batch/egpool" "github.com/featurebasedb/featurebase/v3/logger" + "github.com/featurebasedb/featurebase/v3/pql" "github.com/featurebasedb/featurebase/v3/roaring" "github.com/pkg/errors" ) @@ -716,7 +716,7 @@ func (b *Batch) Add(rec Row) error { return nil } -// ErrBatchNowFull — similar to io.EOF — is a marker error to notify the user of +// ErrBatchNowFull — similar to io.EOF — is a marker error to notify the user of // a batch that it is time to call Import. var ErrBatchNowFull = errors.New("batch is now full - you cannot add any more records (though the one you just added was accepted)") diff --git a/batch/batch_test.go b/batch/batch_test.go index 77db39ab4..442f8aa38 100644 --- a/batch/batch_test.go +++ b/batch/batch_test.go @@ -10,9 +10,9 @@ import ( "testing" "time" - featurebase "github.com/molecula/featurebase/v3" - "github.com/molecula/featurebase/v3/client" - "github.com/molecula/featurebase/v3/pql" + featurebase "github.com/featurebasedb/featurebase/v3" + "github.com/featurebasedb/featurebase/v3/client" + "github.com/featurebasedb/featurebase/v3/pql" "github.com/stretchr/testify/assert" "github.com/pkg/errors" diff --git a/batch/convert.go b/batch/convert.go index 85b530c8b..03e0ebcc0 100644 --- a/batch/convert.go +++ b/batch/convert.go @@ -3,8 +3,8 @@ package batch import ( "time" - featurebase "github.com/molecula/featurebase/v3" - "github.com/molecula/featurebase/v3/errors" + featurebase "github.com/featurebasedb/featurebase/v3" + "github.com/featurebasedb/featurebase/v3/errors" ) var ( diff --git a/batch/egpool/egpool_test.go b/batch/egpool/egpool_test.go index 2cdd6eacb..01c1167c4 100644 --- a/batch/egpool/egpool_test.go +++ b/batch/egpool/egpool_test.go @@ -6,7 +6,7 @@ import ( "errors" "testing" - "github.com/featurebasedb/featurebase/v3/client/egpool" + "github.com/featurebasedb/featurebase/v3/batch/egpool" ) func TestEGPool(t *testing.T) { diff --git a/batch/importer.go b/batch/importer.go index 7f1e54478..24284d27b 100644 --- a/batch/importer.go +++ b/batch/importer.go @@ -5,10 +5,10 @@ import ( "time" "github.com/golang/protobuf/proto" //nolint:staticcheck - featurebase "github.com/molecula/featurebase/v3" - featurebaseproto "github.com/molecula/featurebase/v3/encoding/proto" - "github.com/molecula/featurebase/v3/pb" - "github.com/molecula/featurebase/v3/roaring" + featurebase "github.com/featurebasedb/featurebase/v3" + featurebaseproto "github.com/featurebasedb/featurebase/v3/encoding/proto" + "github.com/featurebasedb/featurebase/v3/pb" + "github.com/featurebasedb/featurebase/v3/roaring" "github.com/pkg/errors" ) diff --git a/batch/metrics.go b/batch/metrics.go index d17917da2..591c7765c 100644 --- a/batch/metrics.go +++ b/batch/metrics.go @@ -1,6 +1,6 @@ // Copyright 2022 Molecula Corp. (DBA FeatureBase). // SPDX-License-Identifier: Apache-2.0 -package client +package batch const ( // MetricBatchImportDurationSeconds records the full time of the diff --git a/client/api.go b/client/api.go index d01ffb97e..47b05da4a 100644 --- a/client/api.go +++ b/client/api.go @@ -3,8 +3,8 @@ package client import ( "context" - featurebase "github.com/molecula/featurebase/v3" - "github.com/molecula/featurebase/v3/errors" + featurebase "github.com/featurebasedb/featurebase/v3" + "github.com/featurebasedb/featurebase/v3/errors" ) var _ featurebase.SchemaAPI = &schemaAPI{} diff --git a/client/importer.go b/client/importer.go index c55cc04bc..ac4dccc15 100644 --- a/client/importer.go +++ b/client/importer.go @@ -4,8 +4,8 @@ import ( "context" "time" - featurebase "github.com/molecula/featurebase/v3" - "github.com/molecula/featurebase/v3/roaring" + featurebase "github.com/featurebasedb/featurebase/v3" + "github.com/featurebasedb/featurebase/v3/roaring" "github.com/pkg/errors" ) diff --git a/cluster.go b/cluster.go index c3e8b6ecc..3cee3ac47 100644 --- a/cluster.go +++ b/cluster.go @@ -9,7 +9,6 @@ import ( "time" "github.com/featurebasedb/featurebase/v3/disco" - "github.com/featurebasedb/featurebase/v3/ingest" "github.com/featurebasedb/featurebase/v3/logger" "github.com/featurebasedb/featurebase/v3/roaring" "github.com/pkg/errors" diff --git a/cmd/badloader/badloader.go b/cmd/badloader/badloader.go index 46227401d..c868565c1 100644 --- a/cmd/badloader/badloader.go +++ b/cmd/badloader/badloader.go @@ -1,5 +1,4 @@ -// Copyright 2022 Molecula Corp. (DBA FeatureBase). -// SPDX-License-Identifier: Apache-2.0 +// Copyright 2021 Molecula Corp. All rights reserved. package main import ( @@ -8,16 +7,17 @@ import ( "context" "fmt" "io" + "os" + "strconv" + "strings" + "time" + gohttp "net/http" pilosa "github.com/featurebasedb/featurebase/v3" "github.com/featurebasedb/featurebase/v3/encoding/proto" pnet "github.com/featurebasedb/featurebase/v3/net" "github.com/featurebasedb/featurebase/v3/vprint" - - "os" - "strconv" - "strings" ) func UploadTar(srcFile string, client *pilosa.InternalClient) error { diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index cc6edadb9..88cf0ce32 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -10,7 +10,6 @@ import ( "github.com/gogo/protobuf/proto" pilosa "github.com/featurebasedb/featurebase/v3" "github.com/featurebasedb/featurebase/v3/disco" - "github.com/featurebasedb/featurebase/v3/ingest" pnet "github.com/featurebasedb/featurebase/v3/net" "github.com/featurebasedb/featurebase/v3/pb" "github.com/featurebasedb/featurebase/v3/pql" diff --git a/encoding/proto/proto_test.go b/encoding/proto/proto_test.go index 8888ebdc4..b4794754b 100644 --- a/encoding/proto/proto_test.go +++ b/encoding/proto/proto_test.go @@ -7,7 +7,6 @@ import ( "testing" pilosa "github.com/featurebasedb/featurebase/v3" - "github.com/featurebasedb/featurebase/v3/ingest" "github.com/featurebasedb/featurebase/v3/pb" ) diff --git a/fragment.go b/fragment.go index d4420d7d6..8eb2bcf6e 100644 --- a/fragment.go +++ b/fragment.go @@ -19,10 +19,7 @@ import ( "sync" "time" - "github.com/cespare/xxhash" - "github.com/featurebasedb/featurebase/v3/disco" "github.com/featurebasedb/featurebase/v3/logger" - pnet "github.com/featurebasedb/featurebase/v3/net" "github.com/featurebasedb/featurebase/v3/pb" "github.com/featurebasedb/featurebase/v3/pql" "github.com/featurebasedb/featurebase/v3/roaring" diff --git a/handler.go b/handler.go index ff1143ed1..3cdaffea2 100644 --- a/handler.go +++ b/handler.go @@ -1,5 +1,4 @@ -// Copyright 2022 Molecula Corp. (DBA FeatureBase). -// SPDX-License-Identifier: Apache-2.0 +// Copyright 2021 Molecula Corp. All rights reserved. package pilosa import ( @@ -7,7 +6,7 @@ import ( "math/bits" "time" - "github.com/featurebasedb/featurebase/v3/ingest" + "github.com/featurebasedb/featurebase/v3/shardwidth" "github.com/featurebasedb/featurebase/v3/tracing" "github.com/pkg/errors" ) diff --git a/handler_test.go b/handler_test.go index 7e102d303..9c1c7df21 100644 --- a/handler_test.go +++ b/handler_test.go @@ -5,7 +5,7 @@ import ( "math/rand" "testing" - pilosa "github.com/molecula/featurebase/v3" + pilosa "github.com/featurebasedb/featurebase/v3" ) func TestSortToShards(t *testing.T) { diff --git a/http_handler.go b/http_handler.go index 65005a8ca..94a1e293e 100644 --- a/http_handler.go +++ b/http_handler.go @@ -31,7 +31,6 @@ import ( "github.com/featurebasedb/featurebase/v3/authn" "github.com/featurebasedb/featurebase/v3/authz" "github.com/featurebasedb/featurebase/v3/disco" - "github.com/featurebasedb/featurebase/v3/ingest" "github.com/featurebasedb/featurebase/v3/logger" "github.com/featurebasedb/featurebase/v3/monitor" "github.com/featurebasedb/featurebase/v3/pql" diff --git a/internal_client.go b/internal_client.go index c4d40131e..fce0d2dc8 100644 --- a/internal_client.go +++ b/internal_client.go @@ -21,7 +21,6 @@ import ( "github.com/featurebasedb/featurebase/v3/authn" "github.com/featurebasedb/featurebase/v3/disco" - "github.com/featurebasedb/featurebase/v3/ingest" "github.com/featurebasedb/featurebase/v3/logger" pnet "github.com/featurebasedb/featurebase/v3/net" "github.com/featurebasedb/featurebase/v3/tracing" diff --git a/iterator.go b/iterator.go index dd490bb26..1b02d74e9 100644 --- a/iterator.go +++ b/iterator.go @@ -5,7 +5,6 @@ package pilosa import ( "fmt" - "github.com/featurebasedb/featurebase/v3/roaring" ) // iterator is an interface for looping over row/column pairs. diff --git a/querycontext/query_context.go b/querycontext/query_context.go index 6331d24e0..7486c1ae9 100644 --- a/querycontext/query_context.go +++ b/querycontext/query_context.go @@ -5,7 +5,7 @@ import ( "sort" "strings" - "github.com/molecula/featurebase/v3/roaring" + "github.com/featurebasedb/featurebase/v3/roaring" ) // QueryContext represents the lifespan of a query or similar thing which diff --git a/querycontext/query_context_test.go b/querycontext/query_context_test.go index 1001b2d1b..fe4ea409e 100644 --- a/querycontext/query_context_test.go +++ b/querycontext/query_context_test.go @@ -11,7 +11,7 @@ import ( "testing" "time" - rbfcfg "github.com/molecula/featurebase/v3/rbf/cfg" + rbfcfg "github.com/featurebasedb/featurebase/v3/rbf/cfg" "golang.org/x/sync/errgroup" ) diff --git a/querycontext/rbf.go b/querycontext/rbf.go index da44596a2..795315dd6 100644 --- a/querycontext/rbf.go +++ b/querycontext/rbf.go @@ -10,9 +10,9 @@ import ( "runtime" "sync" - "github.com/molecula/featurebase/v3/rbf" - rbfcfg "github.com/molecula/featurebase/v3/rbf/cfg" - "github.com/molecula/featurebase/v3/roaring" + "github.com/featurebasedb/featurebase/v3/rbf" + rbfcfg "github.com/featurebasedb/featurebase/v3/rbf/cfg" + "github.com/featurebasedb/featurebase/v3/roaring" ) // rbfDBQueryContexts represents an actual backend DB, and a map of the diff --git a/querycontext/rbf_test.go b/querycontext/rbf_test.go index ba39ba4a7..6664dec5d 100644 --- a/querycontext/rbf_test.go +++ b/querycontext/rbf_test.go @@ -4,7 +4,7 @@ import ( "context" "testing" - "github.com/molecula/featurebase/v3/roaring" + "github.com/featurebasedb/featurebase/v3/roaring" ) func TestRbfWrite(t *testing.T) { diff --git a/querycontext/reduce_err_test.go b/querycontext/reduce_err_test.go index 6b71800ac..e7c42b8d8 100644 --- a/querycontext/reduce_err_test.go +++ b/querycontext/reduce_err_test.go @@ -5,7 +5,7 @@ import ( "sync/atomic" "testing" - "github.com/molecula/featurebase/v3/roaring" + "github.com/featurebasedb/featurebase/v3/roaring" ) type oopsieWrapper struct { diff --git a/rbf.go b/rbf.go index d5c460e01..b82f94363 100644 --- a/rbf.go +++ b/rbf.go @@ -16,7 +16,6 @@ import ( txkey "github.com/featurebasedb/featurebase/v3/short_txkey" "github.com/featurebasedb/featurebase/v3/storage" - "github.com/featurebasedb/featurebase/v3/vprint" "github.com/pkg/errors" ) diff --git a/server/server.go b/server/server.go index 613706e96..4187a69e7 100644 --- a/server/server.go +++ b/server/server.go @@ -1,5 +1,4 @@ -// Copyright 2022 Molecula Corp. (DBA FeatureBase). -// SPDX-License-Identifier: Apache-2.0 +// Copyright 2021 Molecula Corp. All rights reserved. // // Package server contains the `pilosa server` subcommand which runs Pilosa // itself. The purpose of this package is to define an easily tested Command @@ -16,6 +15,7 @@ import ( "log" "math/rand" "net" + "net/http" "os" "os/signal" "path/filepath" @@ -26,12 +26,16 @@ import ( "syscall" "time" + "github.com/featurebasedb/featurebase/v3/dax" "golang.org/x/sync/errgroup" pilosa "github.com/featurebasedb/featurebase/v3" "github.com/featurebasedb/featurebase/v3/authn" "github.com/featurebasedb/featurebase/v3/authz" + "github.com/featurebasedb/featurebase/v3/batch" "github.com/featurebasedb/featurebase/v3/boltdb" + "github.com/featurebasedb/featurebase/v3/dax/computer" + "github.com/featurebasedb/featurebase/v3/dax/computer/alpha" "github.com/featurebasedb/featurebase/v3/encoding/proto" petcd "github.com/featurebasedb/featurebase/v3/etcd" "github.com/featurebasedb/featurebase/v3/gcnotify" @@ -75,7 +79,12 @@ type Command struct { logger loggerLogger queryLogger loggerLogger + mds pilosa.MDS + writeLogger pilosa.WriteLogger + snapshotter pilosa.Snapshotter + Handler pilosa.HandlerI + httpHandler http.Handler grpcServer *grpcServer grpcLn net.Listener API *pilosa.API @@ -86,6 +95,10 @@ type Command struct { serverOptions []pilosa.ServerOption auth *authn.Auth + + // isComputeNode is set to true if this node is running as a DAX compute + // node. + isComputeNode bool } type CommandOption func(c *Command) error @@ -111,6 +124,8 @@ func OptCommandConfig(config *Config) CommandOption { c.Config.Etcd = config.Etcd c.Config.Auth = config.Auth c.Config.TLS = config.TLS + c.Config.MDSAddress = config.MDSAddress + c.Config.WriteLogger = config.WriteLogger return nil } c.Config = config @@ -118,6 +133,41 @@ func OptCommandConfig(config *Config) CommandOption { } } +// OptCommandSetConfig was added because OptCommandConfig only sets a small +// sub-set of the config options (it doesn't seem to be used for anything but +// tests). We need a functional option which sets the full Config. +func OptCommandSetConfig(config *Config) CommandOption { + return func(c *Command) error { + defer c.Config.MustValidate() + c.Config = config + return nil + } +} + +// OptCommandInjections injects the interface implementations. +func OptCommandInjections(inj Injections) CommandOption { + return func(c *Command) error { + if inj.MDS != nil { + c.mds = inj.MDS + } + if inj.WriteLogger != nil { + c.writeLogger = inj.WriteLogger + } + if inj.Snapshotter != nil { + c.snapshotter = inj.Snapshotter + } + c.isComputeNode = inj.IsComputeNode + return nil + } +} + +type Injections struct { + MDS pilosa.MDS + WriteLogger pilosa.WriteLogger + Snapshotter pilosa.Snapshotter + IsComputeNode bool +} + // NewCommand returns a new instance of Main. func NewCommand(stdin io.Reader, stdout, stderr io.Writer, opts ...CommandOption) *Command { c := &Command{ @@ -183,7 +233,7 @@ func (m *Command) doSetupResourceLimits() error { return fmt.Errorf("checking open file limit: %w", err) } else { if oldLimit.Cur != targetFileLimit { - m.logger.Warnf("tried to set open file limit to %d, but it is %d; see https://docs.featurebase.com/reference/hostsystem#operating-system-configuration", targetFileLimit, oldLimit.Cur) + m.logger.Warnf("tried to set open file limit to %d, but it is %d; see https://docs.featurebasedb.cloud/reference/hostsystem#operating-system-configuration", targetFileLimit, oldLimit.Cur) } } } @@ -215,12 +265,70 @@ func (m *Command) setupResourceLimits() error { return setupResourceLimitsErr } +// StartNoServe starts the pilosa server, but doesn't serve on the http handler. +func (m *Command) StartNoServe() (err error) { + // Seed random number generator + rand.Seed(time.Now().UTC().UnixNano()) + + // setupServer + err = m.setupServer() + if err != nil { + return errors.Wrap(err, "setting up server") + } + err = m.setupResourceLimits() + if err != nil { + return errors.Wrap(err, "setting resource limits") + } + + // Initialize server. + if err = m.Server.Open(); err != nil { + return errors.Wrap(err, "opening server") + } + + return nil +} + +// Register registers the node with the MDS service using whatever MDS +// implementation was injected during setup. +func (m *Command) Register() (err error) { + if m.mds == nil { + return errors.New("no MDS implementation with which to register") + } + + node := &dax.Node{ + Address: dax.Address(m.Config.Advertise), + RoleTypes: []dax.RoleType{ + dax.RoleTypeCompute, + dax.RoleTypeTranslate, + }, + } + return m.mds.RegisterNode(context.Background(), node) +} + +// CheckIn is called periodically to check in with the MDS service using +// whatever MDS implementation was injected during setup. +func (m *Command) CheckIn() (err error) { + if m.mds == nil { + return errors.New("no MDS implementation with which to check-in") + } + + node := &dax.Node{ + Address: dax.Address(m.Config.Advertise), + RoleTypes: []dax.RoleType{ + dax.RoleTypeCompute, + dax.RoleTypeTranslate, + }, + } + return m.mds.CheckInNode(context.Background(), node) +} + // Start starts the pilosa server - it returns once the server is running. func (m *Command) Start() (err error) { // Seed random number generator rand.Seed(time.Now().UTC().UnixNano()) - // SetupServer - err = m.SetupServer() + + // setupServer + err = m.setupServer() if err != nil { return errors.Wrap(err, "setting up server") } @@ -258,8 +366,8 @@ func (m *Command) UpAndDown() (err error) { // Seed random number generator rand.Seed(time.Now().UTC().UnixNano()) - // SetupServer - err = m.SetupServer() + // setupServer + err = m.setupServer() if err != nil { return errors.Wrap(err, "setting up server") } @@ -300,8 +408,8 @@ func (m *Command) Wait() error { } } -// SetupServer uses the cluster configuration to set up this server. -func (m *Command) SetupServer() error { +// setupServer uses the cluster configuration to set up this server. +func (m *Command) setupServer() error { runtime.SetBlockProfileRate(m.Config.Profile.BlockRate) runtime.SetMutexProfileFraction(m.Config.Profile.MutexFraction) @@ -368,9 +476,13 @@ func (m *Command) SetupServer() error { return errors.Wrap(err, "new stats client") } - m.ln, err = getListener(*uri, m.tlsConfig) - if err != nil { - return errors.Wrap(err, "getting listener") + if m.Config.Listener == nil { + m.ln, err = getListener(*uri, m.tlsConfig) + if err != nil { + return errors.Wrap(err, "getting listener") + } + } else { + m.ln = m.Config.Listener } // If port is 0, get auto-allocated port from listener @@ -439,6 +551,21 @@ func (m *Command) SetupServer() error { m.Config.Etcd.Dir = filepath.Join(path, pilosa.DiscoDir) } + // WriteLogger setup. + var wlw computer.WriteLogWriter = computer.NewNopWriteLogWriter() + var wlr computer.WriteLogReader = computer.NewNopWriteLogReader() + if m.writeLogger != nil { + alphaWriteLog := alpha.NewAlphaWriteLog(m.writeLogger) + wlr = alphaWriteLog + wlw = alphaWriteLog + } + + // Snapshotter setup. + var snap computer.SnapshotReadWriter = computer.NewNopSnapshotReadWriter() + if m.snapshotter != nil { + snap = alpha.NewAlphaSnapshot(m.snapshotter) + } + m.Config.Etcd.Id = m.Config.Name // TODO(twg) rethink this e := petcd.NewEtcd(m.Config.Etcd, m.logger, m.Config.Cluster.ReplicaN, version) @@ -477,6 +604,9 @@ func (m *Command) SetupServer() error { pilosa.OptServerPartitionAssigner(m.Config.Cluster.PartitionToNodeAssignment), pilosa.OptServerDisCo(e, e, e, e), pilosa.OptServerExecutionPlannerFn(executionPlannerFn), + pilosa.OptServerWriteLogReader(wlr), + pilosa.OptServerWriteLogWriter(wlw), + pilosa.OptServerSnapshotReadWriter(snap), } if m.Config.LookupDBDSN != "" { @@ -492,7 +622,6 @@ func (m *Command) SetupServer() error { } m.Server, err = pilosa.NewServer(serverOptions...) - if err != nil { return errors.Wrap(err, "new server") } @@ -500,6 +629,11 @@ func (m *Command) SetupServer() error { m.API, err = pilosa.NewAPI( pilosa.OptAPIServer(m.Server), pilosa.OptAPIImportWorkerPoolSize(m.Config.ImportWorkerPoolSize), + pilosa.OptAPIWriteLogReader(wlr), + pilosa.OptAPIWriteLogWriter(wlw), + pilosa.OptAPISnapshotter(snap), + pilosa.OptAPIDirectiveWorkerPoolSize(m.Config.DirectiveWorkerPoolSize), + pilosa.OptAPIIsComputeNode(m.isComputeNode), ) if err != nil { return errors.Wrap(err, "new api") @@ -559,7 +693,7 @@ func (m *Command) SetupServer() error { return errors.Wrap(err, "getting grpcServer") } - m.Handler, err = pilosa.NewHandler( + hndlr, err := pilosa.NewHandler( pilosa.OptHandlerAllowedOrigins(m.Config.Handler.AllowedOrigins), pilosa.OptHandlerAPI(m.API), pilosa.OptHandlerLogger(m.logger), @@ -574,7 +708,21 @@ func (m *Command) SetupServer() error { pilosa.OptHandlerRoaringSerializer(proto.RoaringSerializer), pilosa.OptHandlerSQLEnabled(m.Config.SQL.EndpointEnabled), ) - return errors.Wrap(err, "new handler") + if err != nil { + return errors.Wrap(err, "new handler") + } + + m.httpHandler = hndlr + m.Handler = hndlr + + return nil +} + +// HTTPHandler was added for the case where we want to get the full +// http.Handler, and not just those methods which satisfy the pilosa.HandlerI +// interface. +func (m *Command) HTTPHandler() http.Handler { + return m.httpHandler } // setupLogger sets up the logger based on the configuration. diff --git a/sql/show.go b/sql/show.go index 9a3ada249..4e714629f 100644 --- a/sql/show.go +++ b/sql/show.go @@ -1,5 +1,4 @@ -// Copyright 2022 Molecula Corp. (DBA FeatureBase). -// SPDX-License-Identifier: Apache-2.0 +// Copyright 2021 Molecula Corp. All rights reserved. package sql import ( @@ -7,6 +6,7 @@ import ( "fmt" pilosa "github.com/featurebasedb/featurebase/v3" + "github.com/featurebasedb/featurebase/v3/authn" pproto "github.com/featurebasedb/featurebase/v3/proto" "github.com/pkg/errors" "google.golang.org/grpc/codes" diff --git a/sql3/planner/executionplanner.go b/sql3/planner/executionplanner.go index b9e966d15..d1ebdd15c 100644 --- a/sql3/planner/executionplanner.go +++ b/sql3/planner/executionplanner.go @@ -7,6 +7,8 @@ import ( "encoding/json" pilosa "github.com/featurebasedb/featurebase/v3" + "github.com/featurebasedb/featurebase/v3/batch" + "github.com/featurebasedb/featurebase/v3/logger" "github.com/featurebasedb/featurebase/v3/sql3" "github.com/featurebasedb/featurebase/v3/sql3/parser" "github.com/featurebasedb/featurebase/v3/sql3/planner/types" diff --git a/sql3/planner/executionplannersystemtables.go b/sql3/planner/executionplannersystemtables.go index f079dd402..b0fe25770 100644 --- a/sql3/planner/executionplannersystemtables.go +++ b/sql3/planner/executionplannersystemtables.go @@ -5,9 +5,9 @@ package planner import ( "context" - pilosa "github.com/molecula/featurebase/v3" - "github.com/molecula/featurebase/v3/sql3" - "github.com/molecula/featurebase/v3/sql3/parser" + pilosa "github.com/featurebasedb/featurebase/v3" + "github.com/featurebasedb/featurebase/v3/sql3" + "github.com/featurebasedb/featurebase/v3/sql3/parser" "github.com/pkg/errors" ) diff --git a/sql3/planner/expressionanalyzer.go b/sql3/planner/expressionanalyzer.go index b01263a9b..ca86d78bb 100644 --- a/sql3/planner/expressionanalyzer.go +++ b/sql3/planner/expressionanalyzer.go @@ -3,6 +3,8 @@ package planner import ( + "strings" + "github.com/featurebasedb/featurebase/v3/sql3" "github.com/featurebasedb/featurebase/v3/sql3/parser" ) diff --git a/sql3/planner/opbulkinsert.go b/sql3/planner/opbulkinsert.go index 4fb6278a8..aa5f00d67 100644 --- a/sql3/planner/opbulkinsert.go +++ b/sql3/planner/opbulkinsert.go @@ -15,7 +15,9 @@ import ( "strings" "time" - pilosa "github.com/featurebasedb/featurebase/v3" + "github.com/PaesslerAG/gval" + "github.com/PaesslerAG/jsonpath" + "github.com/featurebasedb/featurebase/v3/pql" "github.com/featurebasedb/featurebase/v3/sql3" "github.com/featurebasedb/featurebase/v3/sql3/parser" "github.com/featurebasedb/featurebase/v3/sql3/planner/types" diff --git a/sql3/planner/opinsert.go b/sql3/planner/opinsert.go index bfd482a0f..8912f49f5 100644 --- a/sql3/planner/opinsert.go +++ b/sql3/planner/opinsert.go @@ -9,10 +9,10 @@ import ( "time" pilosa "github.com/featurebasedb/featurebase/v3" - "github.com/featurebasedb/featurebase/v3/pql" + fbbatch "github.com/featurebasedb/featurebase/v3/batch" "github.com/featurebasedb/featurebase/v3/sql3" - "github.com/featurebasedb/featurebase/v3/sql3/parser" "github.com/featurebasedb/featurebase/v3/sql3/planner/types" + "github.com/pkg/errors" ) // PlanOpInsert plan operator to handle INSERT. diff --git a/sql3/planner/opsystemtable.go b/sql3/planner/opsystemtable.go index a8f348b56..78daa66a8 100644 --- a/sql3/planner/opsystemtable.go +++ b/sql3/planner/opsystemtable.go @@ -6,10 +6,10 @@ import ( "context" "fmt" - pilosa "github.com/molecula/featurebase/v3" - "github.com/molecula/featurebase/v3/sql3" - "github.com/molecula/featurebase/v3/sql3/parser" - "github.com/molecula/featurebase/v3/sql3/planner/types" + pilosa "github.com/featurebasedb/featurebase/v3" + "github.com/featurebasedb/featurebase/v3/sql3" + "github.com/featurebasedb/featurebase/v3/sql3/parser" + "github.com/featurebasedb/featurebase/v3/sql3/planner/types" ) //fb_exec_requests diff --git a/sql3/test/defs/defs_aggregate.go b/sql3/test/defs/defs_aggregate.go index 8ce6eb52b..15ee3d275 100644 --- a/sql3/test/defs/defs_aggregate.go +++ b/sql3/test/defs/defs_aggregate.go @@ -5,9 +5,9 @@ import ( "github.com/featurebasedb/featurebase/v3/sql3/parser" ) -//aggregate function tests -var countTests = tableTest{ - table: tbl( +// aggregate function tests +var countTests = TableTest{ + Table: tbl( "count_test", srcHdrs( srcHdr("_id", fldTypeID), @@ -51,7 +51,20 @@ var countTests = tableTest{ Compare: CompareExactUnordered, }, { - sqls: sqls( + SQLs: sqls( + "SELECT COUNT(i1) as a, COUNT(i2) as b FROM count_test", + ), + ExpHdrs: hdrs( + hdr("a", fldTypeInt), + hdr("b", fldTypeInt), + ), + ExpRows: rows( + row(int64(6), int64(2)), + ), + Compare: CompareExactUnordered, + }, + { + SQLs: sqls( "SELECT COUNT(*) + 10 - 11 * 2 AS count_rows FROM count_test", ), ExpHdrs: hdrs( diff --git a/sql3/test/defs/defs_binops.go b/sql3/test/defs/defs_binops.go index 91eb1f84c..15d04d44f 100644 --- a/sql3/test/defs/defs_binops.go +++ b/sql3/test/defs/defs_binops.go @@ -3,7 +3,7 @@ package defs import ( "time" - "github.com/molecula/featurebase/v3/pql" + "github.com/featurebasedb/featurebase/v3/pql" ) // INT bin op tests diff --git a/sql3/test/defs/defs_unkeyed.go b/sql3/test/defs/defs_unkeyed.go index b148f4931..daf7254d2 100644 --- a/sql3/test/defs/defs_unkeyed.go +++ b/sql3/test/defs/defs_unkeyed.go @@ -1,6 +1,6 @@ package defs -import "github.com/molecula/featurebase/v3/pql" +import "github.com/featurebasedb/featurebase/v3/pql" var unkeyed = TableTest{ name: "unkeyed", diff --git a/sql3/test/defs/types.go b/sql3/test/defs/types.go index 9ddd28dc5..fbb93962e 100644 --- a/sql3/test/defs/types.go +++ b/sql3/test/defs/types.go @@ -8,8 +8,8 @@ import ( "testing" "time" - "github.com/molecula/featurebase/v3/sql3/parser" - planner_types "github.com/molecula/featurebase/v3/sql3/planner/types" + "github.com/featurebasedb/featurebase/v3/sql3/parser" + planner_types "github.com/featurebasedb/featurebase/v3/sql3/planner/types" ) type fldType parser.ExprDataType diff --git a/translate.go b/translate.go index b1fc232ad..5d00d3eeb 100644 --- a/translate.go +++ b/translate.go @@ -11,7 +11,6 @@ import ( "sync" "github.com/featurebasedb/featurebase/v3/disco" - "github.com/featurebasedb/featurebase/v3/ingest" "github.com/featurebasedb/featurebase/v3/roaring" "github.com/pkg/errors" )