Skip to content

Commit 9cb7be2

Browse files
author
Sriram Melkote
committed
MB-24898: Allow plasma to be close source, take #2
Change-Id: Ieaa95c5a0e4a9cf4ab6d9b1f440cde589a114491
1 parent 801ffee commit 9cb7be2

17 files changed

+221
-57
lines changed

‎CMakeLists.txt

Lines changed: 30 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -4,27 +4,27 @@ PROJECT (indexing)
44
INCLUDE (FindCouchbaseGo)
55
INCLUDE (FindCouchbaseJemalloc)
66

7-
get_filename_component(JEMALLOC_LIB_DIR ${JEMALLOC_LIBRARIES} DIRECTORY)
8-
9-
SET (_ldflags)
7+
IF (BUILD_ENTERPRISE)
8+
MESSAGE(STATUS "Building EE edition of GSI")
9+
SET (TAGS "jemalloc")
10+
ELSE (BUILD_ENTERPRISE)
11+
MESSAGE(STATUS "Building CE edition of GSI")
12+
SET (TAGS "jemalloc community")
13+
ENDIF (BUILD_ENTERPRISE)
14+
15+
SET (LDFLAGS)
1016
IF (APPLE)
11-
# On OS X 10.11 (El Capitan) upwards we can no longer use DYLD_LIBRARY_PATH to locate
12-
# runtime dependancies. Instead add the location of forestdb etc to the actual
13-
# binary itself.
14-
15-
# TODO: Spock GA
16-
# Temporarily adding -s flag to allow 1.7.3 to work on OSX 10.12.4
17-
# Remove -s flag (which strips debug info) when we move to 1.8.1
18-
# See https://github.com/golang/go/issues/19734
19-
20-
SET (_ldflags "-s -extldflags '-Wl,-rpath,@executable_path/../lib'")
17+
# On macOS, we must use -rpath to locate libraries
18+
# TODO: Remove debug info stripping (-s) added to allow 1.7.3 to work on macOS
19+
SET (LDFLAGS "-s -extldflags '-Wl,-rpath,@executable_path/../lib'")
2120
ENDIF ()
2221

2322
IF (WIN32)
24-
SET (_ldflags "${_ldflags} -linkmode internal")
23+
SET (LDFLAGS "${LDFLAGS} -linkmode internal")
2524
ENDIF ()
2625

2726
GET_FILENAME_COMPONENT (CURL_LIBRARY_DIR "${CURL_LIBRARIES}" DIRECTORY)
27+
GET_FILENAME_COMPONENT (JEMALLOC_LIB_DIR ${JEMALLOC_LIBRARIES} DIRECTORY)
2828

2929
SET(CGO_INCLUDE_DIRS "${FORESTDB_INCLUDE_DIR};${sigar_SOURCE_DIR}/include;${Platform_SOURCE_DIR}/include;${CURL_INCLUDE_DIR}")
3030
SET(CGO_LIBRARY_DIRS "${FORESTDB_LIBRARY_DIR};${sigar_BINARY_DIR}/src;${Platform_BINARY_DIR};${CURL_LIBRARY_DIR}")
@@ -36,15 +36,14 @@ ENDIF(NOT JEMALLOC_FOUND)
3636
SET (ENV{CGO_CFLAGS} "$ENV{CGO_CFLAGS} -DJEMALLOC=1")
3737
SET (CGO_INCLUDE_DIRS "${CGO_INCLUDE_DIRS};${JEMALLOC_INCLUDE_DIR}")
3838
SET (CGO_LIBRARY_DIRS "${CGO_LIBRARY_DIRS};${JEMALLOC_LIB_DIR}")
39-
SET (TAGS "jemalloc")
4039

4140
GoInstall (TARGET indexer PACKAGE github.com/couchbase/indexing/secondary/cmd/indexer
4241
GOPATH "${PROJECT_SOURCE_DIR}/../../../.." "${GODEPSDIR}"
4342
DEPENDS forestdb
4443
CGO_INCLUDE_DIRS "${CGO_INCLUDE_DIRS}"
4544
CGO_LIBRARY_DIRS "${CGO_LIBRARY_DIRS}"
4645
GOTAGS "${TAGS}"
47-
LDFLAGS "${_ldflags}"
46+
LDFLAGS "${LDFLAGS}"
4847
INSTALL_PATH bin OUTPUT indexer
4948
GOVERSION 1.7.3)
5049

@@ -53,21 +52,21 @@ GoInstall (TARGET projector PACKAGE github.com/couchbase/indexing/secondary/cmd/
5352
CGO_INCLUDE_DIRS "${CGO_INCLUDE_DIRS}"
5453
CGO_LIBRARY_DIRS "${CGO_LIBRARY_DIRS}"
5554
GOTAGS "${TAGS}"
56-
LDFLAGS "${_ldflags}"
55+
LDFLAGS "${LDFLAGS}"
5756
INSTALL_PATH bin
5857
GOVERSION 1.7.3)
5958

6059
GoInstall (TARGET cbindex PACKAGE github.com/couchbase/indexing/secondary/cmd/cbindex
6160
GOPATH "${PROJECT_SOURCE_DIR}/../../../.." "${GODEPSDIR}"
6261
GOTAGS "${TAGS}"
63-
LDFLAGS "${_ldflags}"
62+
LDFLAGS "${LDFLAGS}"
6463
INSTALL_PATH bin
6564
GOVERSION 1.7.3)
6665

6766
GoInstall (TARGET cbindexperf PACKAGE github.com/couchbase/indexing/secondary/cmd/cbindexperf
6867
GOPATH "${PROJECT_SOURCE_DIR}/../../../.." "${GODEPSDIR}"
6968
GOTAGS "${TAGS}"
70-
LDFLAGS "${_ldflags}"
69+
LDFLAGS "${LDFLAGS}"
7170
INSTALL_PATH bin
7271
GOVERSION 1.7.3)
7372

@@ -77,15 +76,18 @@ GoInstall (TARGET cbindexplan PACKAGE github.com/couchbase/indexing/secondary/cm
7776
CGO_INCLUDE_DIRS "${CGO_INCLUDE_DIRS}"
7877
CGO_LIBRARY_DIRS "${CGO_LIBRARY_DIRS}"
7978
GOTAGS "${TAGS}"
80-
LDFLAGS "${_ldflags}"
79+
LDFLAGS "${LDFLAGS}"
8180
INSTALL_PATH bin OUTPUT cbindexplan
8281
GOVERSION 1.7.3)
8382

84-
GoInstall(TARGET plasma_dump PACKAGE github.com/couchbase/nitro/cmd/plasma_dump
85-
GOPATH "${CMAKE_SOURCE_DIR}/goproj" "${GODEPSDIR}"
86-
CGO_INCLUDE_DIRS "${CGO_INCLUDE_DIRS}"
87-
CGO_LIBRARY_DIRS "${CGO_LIBRARY_DIRS}"
88-
GOTAGS "${TAGS}"
89-
LDFLAGS "${_ldflags}"
90-
INSTALL_PATH bin OUTPUT plasma_dump
91-
GOVERSION 1.7.3)
83+
IF (BUILD_ENTERPRISE)
84+
GoInstall(TARGET plasma_dump PACKAGE github.com/couchbase/nitro/cmd/plasma_dump
85+
GOPATH "${CMAKE_SOURCE_DIR}/goproj" "${GODEPSDIR}"
86+
CGO_INCLUDE_DIRS "${CGO_INCLUDE_DIRS}"
87+
CGO_LIBRARY_DIRS "${CGO_LIBRARY_DIRS}"
88+
GOTAGS "${TAGS}"
89+
LDFLAGS "${LDFLAGS}"
90+
INSTALL_PATH bin OUTPUT plasma_dump
91+
GOVERSION 1.7.3)
92+
ENDIF (BUILD_ENTERPRISE)
93+

‎secondary/build.sh

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
if [ "$C_INCLUDE_PATH" == "" ]; then
44
top="`pwd`/../../../../../.."
55
export GOPATH="$top/goproj:$top/godeps"
6-
export C_INCLUDE_PATH="$top/install/platform/include:$top/install/include:$top/forestdb/include:$top/install/build/tlm/deps/curl.exploded/include"
6+
export C_INCLUDE_PATH="$top/install/platform/include:$top/install/include:$top/forestdb/include:$top/install/build/tlm/deps/curl.exploded/include:$top/sigar/include"
77
export CGO_LDFLAGS="-L $top/install/lib"
88
export LD_LIBRARY_PATH="$top/install/lib"
99
fi
@@ -50,11 +50,13 @@ clean_projector(){
5050

5151
build_protobuf(){
5252

53-
echo "Building Protobuf..."
54-
cd protobuf
55-
make
56-
cd ..
57-
echo "Done"
53+
if which protoc; then
54+
echo "Building Protobuf..."
55+
cd protobuf
56+
make
57+
cd ..
58+
echo "Done"
59+
fi
5860
}
5961

6062
if [ -z "$1" ]

‎secondary/common/index.go

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -435,8 +435,7 @@ func (s IndexSnapType) String() string {
435435

436436
}
437437

438-
//NOTE: This type needs to be in sync with
439-
//smStrMap in index/global.go
438+
//NOTE: This type needs to be in sync with smStrMap
440439
type IndexType string
441440

442441
const (

‎secondary/common/storage_mode.go

Lines changed: 18 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,9 @@ package common
1212
import (
1313
"strings"
1414
"sync"
15+
16+
"github.com/couchbase/indexing/secondary/logging"
17+
"github.com/couchbase/indexing/secondary/stubs/nitro/plasma"
1518
)
1619

1720
type StorageMode byte
@@ -28,11 +31,11 @@ func (s StorageMode) String() string {
2831
case NOT_SET:
2932
return "not_set"
3033
case MOI:
31-
return "memory_optimized"
34+
return MemoryOptimized
3235
case FORESTDB:
33-
return "forestdb"
36+
return ForestDB
3437
case PLASMA:
35-
return "plasma"
38+
return PlasmaDB
3639
default:
3740
return "invalid"
3841
}
@@ -41,10 +44,10 @@ func (s StorageMode) String() string {
4144
//NOTE: This map needs to be in sync with IndexType in
4245
//common/index.go
4346
var smStrMap = map[string]StorageMode{
44-
"memdb": MOI,
45-
"memory_optimized": MOI,
46-
"forestdb": FORESTDB,
47-
"plasma": PLASMA,
47+
MemDB: MOI,
48+
MemoryOptimized: MOI,
49+
ForestDB: FORESTDB,
50+
PlasmaDB: PLASMA,
4851
}
4952

5053
//Global Storage Mode
@@ -64,7 +67,10 @@ func SetStorageMode(mode StorageMode) {
6467
smLock.Lock()
6568
defer smLock.Unlock()
6669
gStorageMode = mode
67-
70+
if gStorageMode == PLASMA && !plasma.UsePlasma() {
71+
logging.Warnf("Plasma is available only in EE but this is CE. Using ForestDB")
72+
gStorageMode = FORESTDB
73+
}
6874
}
6975

7076
func SetStorageModeStr(mode string) bool {
@@ -73,6 +79,10 @@ func SetStorageModeStr(mode string) bool {
7379
defer smLock.Unlock()
7480
if s, ok := smStrMap[strings.ToLower(mode)]; ok {
7581
gStorageMode = s
82+
if gStorageMode == PLASMA && !plasma.UsePlasma() {
83+
logging.Warnf("Plasma is available only in EE but this is CE. Using ForestDB")
84+
gStorageMode = FORESTDB
85+
}
7686
return true
7787
} else {
7888
gStorageMode = NOT_SET

‎secondary/indexer/indexer.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,8 @@ import (
3434
"github.com/couchbase/indexing/secondary/memdb"
3535
"github.com/couchbase/indexing/secondary/memdb/nodetable"
3636
projClient "github.com/couchbase/indexing/secondary/projector/client"
37-
"github.com/couchbase/nitro/mm"
38-
"github.com/couchbase/nitro/plasma"
37+
"github.com/couchbase/indexing/secondary/stubs/nitro/mm"
38+
"github.com/couchbase/indexing/secondary/stubs/nitro/plasma"
3939
)
4040

4141
type Indexer interface {

‎secondary/indexer/memdb_slice_impl.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ import (
3333
"github.com/couchbase/indexing/secondary/memdb"
3434
"github.com/couchbase/indexing/secondary/memdb/nodetable"
3535
"github.com/couchbase/indexing/secondary/memdb/skiplist"
36-
"github.com/couchbase/nitro/mm"
36+
"github.com/couchbase/indexing/secondary/stubs/nitro/mm"
3737
)
3838

3939
const (

‎secondary/indexer/plasma_community.go

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
// +build community
2+
3+
package indexer
4+
5+
// Copyright (c) 2014 Couchbase, Inc.
6+
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
7+
// except in compliance with the License. You may obtain a copy of the License at
8+
// http://www.apache.org/licenses/LICENSE-2.0
9+
// Unless required by applicable law or agreed to in writing, software distributed under the
10+
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
11+
// either express or implied. See the License for the specific language governing permissions
12+
// and limitations under the License.
13+
14+
import (
15+
"github.com/couchbase/indexing/secondary/common"
16+
)
17+
18+
func NewPlasmaSlice(path string, sliceId SliceId, idxDefn common.IndexDefn,
19+
idxInstId common.IndexInstId, isPrimary bool,
20+
sysconf common.Config, idxStats *IndexStats) (Slice, error) {
21+
panic("Plasma is only supported in Enterprise Edition")
22+
}

‎secondary/indexer/plasma_enterprise.go

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
// +build !community
2+
3+
package indexer
4+
5+
// Copyright (c) 2014 Couchbase, Inc.
6+
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
7+
// except in compliance with the License. You may obtain a copy of the License at
8+
// http://www.apache.org/licenses/LICENSE-2.0
9+
// Unless required by applicable law or agreed to in writing, software distributed under the
10+
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
11+
// either express or implied. See the License for the specific language governing permissions
12+
// and limitations under the License.
13+
14+
import (
15+
"github.com/couchbase/indexing/secondary/common"
16+
)
17+
18+
func NewPlasmaSlice(path string, sliceId SliceId, idxDefn common.IndexDefn,
19+
idxInstId common.IndexInstId, isPrimary bool,
20+
sysconf common.Config, idxStats *IndexStats) (*plasmaSlice, error) {
21+
return newPlasmaSlice(path, sliceId,
22+
idxDefn, idxInstId, isPrimary,
23+
sysconf, idxStats)
24+
}

‎secondary/indexer/plasma_slice.go

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,7 @@
1+
// +build !community
2+
3+
package indexer
4+
15
// Copyright (c) 2014 Couchbase, Inc.
26
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
37
// except in compliance with the License. You may obtain a copy of the License at
@@ -7,8 +11,6 @@
711
// either express or implied. See the License for the specific language governing permissions
812
// and limitations under the License.
913

10-
package indexer
11-
1214
import (
1315
"bytes"
1416
"encoding/binary"
@@ -91,7 +93,7 @@ type plasmaSlice struct {
9193
hasPersistence bool
9294
}
9395

94-
func NewPlasmaSlice(path string, sliceId SliceId, idxDefn common.IndexDefn,
96+
func newPlasmaSlice(path string, sliceId SliceId, idxDefn common.IndexDefn,
9597
idxInstId common.IndexInstId, isPrimary bool,
9698
sysconf common.Config, idxStats *IndexStats) (*plasmaSlice, error) {
9799

‎secondary/indexer/settings.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,13 +13,15 @@ import (
1313
"bytes"
1414
"errors"
1515
"fmt"
16+
1617
"github.com/couchbase/cbauth"
1718
"github.com/couchbase/cbauth/metakv"
1819
"github.com/couchbase/indexing/secondary/common"
1920
"github.com/couchbase/indexing/secondary/logging"
2021
"github.com/couchbase/indexing/secondary/pipeline"
21-
"github.com/couchbase/nitro/mm"
22-
"github.com/couchbase/nitro/plasma"
22+
"github.com/couchbase/indexing/secondary/stubs/nitro/mm"
23+
"github.com/couchbase/indexing/secondary/stubs/nitro/plasma"
24+
2325
"io/ioutil"
2426
"net/http"
2527
"os"

‎secondary/indexer/stats_manager.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ import (
2222
"github.com/couchbase/indexing/secondary/common"
2323
"github.com/couchbase/indexing/secondary/logging"
2424
"github.com/couchbase/indexing/secondary/stats"
25-
"github.com/couchbase/nitro/mm"
25+
"github.com/couchbase/indexing/secondary/stubs/nitro/mm"
2626
)
2727

2828
var uptime time.Time

‎secondary/memdb/memdb.go

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,6 @@ import (
66
"encoding/json"
77
"errors"
88
"fmt"
9-
"github.com/couchbase/indexing/secondary/memdb/skiplist"
10-
"github.com/couchbase/nitro/mm"
119
"io"
1210
"io/ioutil"
1311
"math"
@@ -19,6 +17,9 @@ import (
1917
"sync/atomic"
2018
"time"
2119
"unsafe"
20+
21+
"github.com/couchbase/indexing/secondary/memdb/skiplist"
22+
"github.com/couchbase/indexing/secondary/stubs/nitro/mm"
2223
)
2324

2425
var version = 1
@@ -1174,5 +1175,5 @@ func MemoryInUse() (sz int64) {
11741175

11751176
func Debug(flag bool) {
11761177
skiplist.Debug = flag
1177-
mm.Debug = flag
1178+
*mm.Debug = flag
11781179
}

‎secondary/memdb/memdb_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ import "math/rand"
99
import "sync"
1010
import "runtime"
1111
import "encoding/binary"
12-
import "github.com/couchbase/nitro/mm"
12+
import "github.com/couchbase/indexing/secondary/stubs/nitro/mm"
1313

1414
var testConf Config
1515

‎secondary/stubs/nitro/mm/mm_community.go

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
// +build community
2+
3+
package mm
4+
5+
import "github.com/couchbase/indexing/secondary/memdb/skiplist"
6+
7+
var Malloc skiplist.MallocFn
8+
var Free skiplist.FreeFn
9+
10+
var Debug *bool = &[]bool{false}[0]
11+
12+
func FreeOSMemory() {
13+
}
14+
15+
func Size() uint64 {
16+
return 0
17+
}
18+
19+
func Stats() string {
20+
return ""
21+
}

0 commit comments

Comments
 (0)