From 1d83087c508ff942969c1125c9b6bb9fd8c21cce Mon Sep 17 00:00:00 2001 From: iguazio-deploy Date: Thu, 23 Jul 2020 07:38:08 +0000 Subject: [PATCH] Updated TSDB to v0.10.7 --- .../v3io/v3io-tsdb/.github/workflows/ci.yaml | 132 ++++++++++++++++++ .../github.com/v3io/v3io-tsdb/.gitignore | 1 + .../github.com/v3io/v3io-tsdb/.travis.yml | 9 +- .../github.com/v3io/v3io-tsdb/Jenkinsfile | 59 -------- .../vendor/github.com/v3io/v3io-tsdb/Makefile | 4 +- .../github.com/v3io/v3io-tsdb/README.md | 3 +- .../v3io/v3io-tsdb/pkg/appender/appender.go | 4 - .../v3io/v3io-tsdb/pkg/appender/ingest.go | 58 ++++---- .../v3io/v3io-tsdb/pkg/utils/timeutils.go | 4 +- .../v3io-tsdb/pkg/utils/timeutils_test.go | 29 +++- .../{travis_v3io.yaml => test/ci_v3io.yaml} | 0 .../ci_v3io_bench.yaml} | 0 .../v3io/v3io-tsdb/.github/workflows/ci.yaml | 132 ++++++++++++++++++ .../github.com/v3io/v3io-tsdb/.gitignore | 1 + .../github.com/v3io/v3io-tsdb/.travis.yml | 9 +- .../github.com/v3io/v3io-tsdb/Jenkinsfile | 59 -------- .../vendor/github.com/v3io/v3io-tsdb/Makefile | 4 +- .../github.com/v3io/v3io-tsdb/README.md | 3 +- .../v3io/v3io-tsdb/pkg/appender/appender.go | 4 - .../v3io/v3io-tsdb/pkg/appender/ingest.go | 58 ++++---- .../v3io/v3io-tsdb/pkg/utils/timeutils.go | 4 +- .../v3io-tsdb/pkg/utils/timeutils_test.go | 29 +++- .../{travis_v3io.yaml => test/ci_v3io.yaml} | 0 .../ci_v3io_bench.yaml} | 0 24 files changed, 380 insertions(+), 226 deletions(-) create mode 100644 functions/ingest/vendor/github.com/v3io/v3io-tsdb/.github/workflows/ci.yaml rename functions/ingest/vendor/github.com/v3io/v3io-tsdb/{travis_v3io.yaml => test/ci_v3io.yaml} (100%) rename functions/ingest/vendor/github.com/v3io/v3io-tsdb/{travis_v3io_bench.yaml => test/ci_v3io_bench.yaml} (100%) create mode 100644 functions/query/vendor/github.com/v3io/v3io-tsdb/.github/workflows/ci.yaml rename functions/query/vendor/github.com/v3io/v3io-tsdb/{travis_v3io.yaml => test/ci_v3io.yaml} (100%) rename functions/query/vendor/github.com/v3io/v3io-tsdb/{travis_v3io_bench.yaml => test/ci_v3io_bench.yaml} (100%) diff --git a/functions/ingest/vendor/github.com/v3io/v3io-tsdb/.github/workflows/ci.yaml b/functions/ingest/vendor/github.com/v3io/v3io-tsdb/.github/workflows/ci.yaml new file mode 100644 index 00000000..5b7e0fbe --- /dev/null +++ b/functions/ingest/vendor/github.com/v3io/v3io-tsdb/.github/workflows/ci.yaml @@ -0,0 +1,132 @@ +name: CI + +on: + pull_request: + branches: + - development + - 'v[0-9]+.[0-9]+.x' + + # Run CI also on push to master + push: + branches: + - master + + release: + types: + - created + +jobs: + test: + name: Lint & test + runs-on: ubuntu-latest + env: + TSDB_TEST_TABLE_PATH: TSDB_INTEGRATION_TESTS/${{ github.sha }} + + steps: + - name: Dump github context + run: echo "$GITHUB_CONTEXT" + env: + GITHUB_CONTEXT: ${{ toJson(github) }} + + - name: Dump runner context + run: echo "$RUNNER_CONTEXT" + env: + RUNNER_CONTEXT: ${{ toJson(runner) }} + + - name: Dump github ref + run: echo "$GITHUB_REF" + + - uses: actions/setup-go@v2 + with: + go-version: "^1.14.0" + + - uses: actions/cache@v2 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + + - uses: actions/checkout@v2 + + - name: Lint + run: make lint + + - name: Test short + run: make test + +# Disabled until GH actions will have a solution to using secret on PR triggered workflows. +# Until then, this runs in travis +# - name: Test integration +# run: make integration +# env: +# V3IO_TSDB_CONFIG: ${{ github.workspace }}/test/ci_v3io.yaml +# +# - name: Benchmark +# run: make bench +# env: +# V3IO_TSDB_CONFIG: ${{ github.workspace }}/test/ci_v3io_bench.yaml +# TSDB_BENCH_INGEST_CONFIG: ${{ github.workspace }}/test/benchmark/testdata/tsdb-bench-test-config-ci.yaml + + release: + name: Release + runs-on: ubuntu-latest + if: github.event_name == 'release' + needs: + - test + strategy: + matrix: + go-os: + - linux + - darwin + - windows + + steps: + - name: Dump GitHub context + run: echo "$GITHUB_CONTEXT" + env: + GITHUB_CONTEXT: ${{ toJson(github) }} + + - name: Dump runner context + run: echo "$RUNNER_CONTEXT" + env: + RUNNER_CONTEXT: ${{ toJson(runner) }} + + - name: Dump github ref + run: echo "$GITHUB_REF" + + - name: Extract ref info + id: release_info + run: | + echo ::set-output name=REF_BRANCH::${GITHUB_REF#refs/heads/} + echo ::set-output name=REF_TAG::${GITHUB_REF#refs/tags/} + + - name: Set TSDB_LABEL to release tag + run: | + echo "::set-env name=TSDB_LABEL::${{ steps.release_info.outputs.REF_TAG }}" + + - uses: actions/checkout@v2 + + - uses: actions/setup-go@v2 + with: + go-version: "^1.14.0" + + - uses: actions/cache@v2 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + + - name: Build binaries for ${{ matrix.go-os }} + run: make bin + env: + GOPATH: ${{ github.workspace }}/go + GOARCH: amd64 + GOOS: ${{ matrix.go-os }} + + - name: Upload binaries + uses: AButler/upload-release-assets@v2.0 + with: + files: ${{ github.workspace }}/go/bin/tsdbctl-* + repo-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/functions/ingest/vendor/github.com/v3io/v3io-tsdb/.gitignore b/functions/ingest/vendor/github.com/v3io/v3io-tsdb/.gitignore index ddd81e7f..1fe5870c 100644 --- a/functions/ingest/vendor/github.com/v3io/v3io-tsdb/.gitignore +++ b/functions/ingest/vendor/github.com/v3io/v3io-tsdb/.gitignore @@ -5,3 +5,4 @@ *.sln.iml .idea/ +tsdbctl-* diff --git a/functions/ingest/vendor/github.com/v3io/v3io-tsdb/.travis.yml b/functions/ingest/vendor/github.com/v3io/v3io-tsdb/.travis.yml index c845894d..44baf85d 100644 --- a/functions/ingest/vendor/github.com/v3io/v3io-tsdb/.travis.yml +++ b/functions/ingest/vendor/github.com/v3io/v3io-tsdb/.travis.yml @@ -9,10 +9,5 @@ env: script: - make test - make lint - - V3IO_TSDB_CONFIG="$TRAVIS_BUILD_DIR/travis_v3io.yaml" make integration - - V3IO_TSDB_CONFIG="$TRAVIS_BUILD_DIR/travis_v3io_bench.yaml" TSDB_BENCH_INGEST_CONFIG="$TRAVIS_BUILD_DIR/test/benchmark/testdata/tsdb-bench-test-config-ci.yaml" make bench - -before_deploy: - - GOOS=linux GOARCH=amd64 make build - - GOOS=darwin GOARCH=amd64 make build - - GOOS=windows GOARCH=amd64 make build + - V3IO_TSDB_CONFIG="$TRAVIS_BUILD_DIR/test/ci_v3io.yaml" make integration + - V3IO_TSDB_CONFIG="$TRAVIS_BUILD_DIR/test/ci_v3io_bench.yaml" TSDB_BENCH_INGEST_CONFIG="$TRAVIS_BUILD_DIR/test/benchmark/testdata/tsdb-bench-test-config-ci.yaml" make bench diff --git a/functions/ingest/vendor/github.com/v3io/v3io-tsdb/Jenkinsfile b/functions/ingest/vendor/github.com/v3io/v3io-tsdb/Jenkinsfile index 6b4f15b8..dec17c14 100644 --- a/functions/ingest/vendor/github.com/v3io/v3io-tsdb/Jenkinsfile +++ b/functions/ingest/vendor/github.com/v3io/v3io-tsdb/Jenkinsfile @@ -9,54 +9,6 @@ git_deploy_user_token = "iguazio-prod-git-user-token" git_deploy_user_private_key = "iguazio-prod-git-user-private-key" -def build_v3io_tsdb(TAG_VERSION) { - withCredentials([ - string(credentialsId: git_deploy_user_token, variable: 'GIT_TOKEN') - ]) { - def git_project = 'v3io-tsdb' - stage('prepare sources') { - container('jnlp') { - dir("${BUILD_FOLDER}/src/github.com/${git_project_upstream_user}/${git_project}") { - git(changelog: false, credentialsId: git_deploy_user_private_key, poll: false, url: "git@github.com:${git_project_user}/${git_project}.git") - sh("git checkout ${TAG_VERSION}") - } - } - } - - stage("build ${git_project} binaries in dood") { - container('golang') { - dir("${BUILD_FOLDER}/src/github.com/${git_project_upstream_user}/${git_project}") { - sh """ - mkdir -p /home/jenkins/go/bin - GO111MODULE=on GOOS=linux GOARCH=amd64 TRAVIS_TAG=${TAG_VERSION} make bin - GO111MODULE=on GOOS=darwin GOARCH=amd64 TRAVIS_TAG=${TAG_VERSION} make bin - GO111MODULE=on GOOS=windows GOARCH=amd64 TRAVIS_TAG=${TAG_VERSION} make bin - ls -la /go/bin - mv /go/bin/* /home/jenkins/go/bin/ - ls -la /home/jenkins/go/bin/ - - """ - } - } - } - - stage('upload release assets') { - container('jnlp') { - RELEASE_ID = github.get_release_id(git_project, git_project_user, "${TAG_VERSION}", GIT_TOKEN) - - github.upload_asset(git_project, git_project_user, "tsdbctl-${TAG_VERSION}-linux-amd64", RELEASE_ID, GIT_TOKEN) - github.upload_asset(git_project, git_project_user, "tsdbctl-${TAG_VERSION}-darwin-amd64", RELEASE_ID, GIT_TOKEN) - github.upload_asset(git_project, git_project_user, "tsdbctl-${TAG_VERSION}-windows-amd64", RELEASE_ID, GIT_TOKEN) - withCredentials([ - string(credentialsId: pipelinex.PackagesRepo.ARTIFACTORY_IGUAZIO[2], variable: 'PACKAGES_ARTIFACTORY_PASSWORD') - ]) { - common.upload_file_to_artifactory(pipelinex.PackagesRepo.ARTIFACTORY_IGUAZIO[0], pipelinex.PackagesRepo.ARTIFACTORY_IGUAZIO[1], PACKAGES_ARTIFACTORY_PASSWORD, "iguazio-devops/k8s", "tsdbctl-${TAG_VERSION}-linux-amd64") - } - } - } - } -} - def build_nuclio(V3IO_TSDB_VERSION, internal_status="stable") { withCredentials([ usernamePassword(credentialsId: git_deploy_user, passwordVariable: 'GIT_PASSWORD', usernameVariable: 'GIT_USERNAME'), @@ -413,17 +365,6 @@ podTemplate(label: "${git_project}-${label}", inheritFrom: "jnlp-docker-golang") if (github.check_tag_expiration(git_project, git_project_user, MAIN_TAG_VERSION, GIT_TOKEN)) { parallel( - 'v3io-tsdb': { - podTemplate(label: "v3io-tsdb-${label}", inheritFrom: "jnlp-docker-golang") { - node("v3io-tsdb-${label}") { - withCredentials([ - string(credentialsId: git_deploy_user_token, variable: 'GIT_TOKEN') - ]) { - build_v3io_tsdb(MAIN_TAG_VERSION) - } - } - } - }, 'tsdb-nuclio': { podTemplate(label: "v3io-tsdb-nuclio-${label}", inheritFrom: "jnlp-docker-golang") { node("v3io-tsdb-nuclio-${label}") { diff --git a/functions/ingest/vendor/github.com/v3io/v3io-tsdb/Makefile b/functions/ingest/vendor/github.com/v3io/v3io-tsdb/Makefile index c0bc9855..ebf9b9c1 100644 --- a/functions/ingest/vendor/github.com/v3io/v3io-tsdb/Makefile +++ b/functions/ingest/vendor/github.com/v3io/v3io-tsdb/Makefile @@ -4,8 +4,8 @@ ifeq ($(GIT_BRANCH),) GIT_BRANCH="N/A" endif -ifneq ($(TRAVIS_TAG),) - GIT_REVISION := $(TRAVIS_TAG) +ifneq ($(TSDB_LABEL),) + GIT_REVISION := $(TSDB_LABEL) else GIT_REVISION := $(shell git describe --always) endif diff --git a/functions/ingest/vendor/github.com/v3io/v3io-tsdb/README.md b/functions/ingest/vendor/github.com/v3io/v3io-tsdb/README.md index b99720eb..3d77474a 100644 --- a/functions/ingest/vendor/github.com/v3io/v3io-tsdb/README.md +++ b/functions/ingest/vendor/github.com/v3io/v3io-tsdb/README.md @@ -1,4 +1,5 @@ -[![Build Status](https://travis-ci.org/v3io/v3io-tsdb.svg?branch=master)](https://travis-ci.org/v3io/v3io-tsdb) +[![Travis Build Status](https://travis-ci.org/v3io/v3io-tsdb.svg?branch=master)](https://travis-ci.org/v3io/v3io-tsdb) +[![GH Build Status](https://github.com/v3io/v3io-tsdb/workflows/CI/badge.svg)](https://github.com/v3io/v3io-tsdb/actions) # V3IO-TSDB Iguazio API lib for time-series DB access and Prometheus TSDB storage driver. diff --git a/functions/ingest/vendor/github.com/v3io/v3io-tsdb/pkg/appender/appender.go b/functions/ingest/vendor/github.com/v3io/v3io-tsdb/pkg/appender/appender.go index 19322404..26fd9ea5 100644 --- a/functions/ingest/vendor/github.com/v3io/v3io-tsdb/pkg/appender/appender.go +++ b/functions/ingest/vendor/github.com/v3io/v3io-tsdb/pkg/appender/appender.go @@ -77,10 +77,6 @@ func (m *MetricState) isReady() bool { return m.state == storeStateReady } -func (m *MetricState) isTimeInvalid(t int64) bool { - return !((m.state == storeStateReady || m.state == storeStateUpdate) && t < m.store.maxTime-maxLateArrivalInterval) -} - func (m *MetricState) getState() storeState { return m.state } diff --git a/functions/ingest/vendor/github.com/v3io/v3io-tsdb/pkg/appender/ingest.go b/functions/ingest/vendor/github.com/v3io/v3io-tsdb/pkg/appender/ingest.go index 032c8e0b..eb349e0e 100644 --- a/functions/ingest/vendor/github.com/v3io/v3io-tsdb/pkg/appender/ingest.go +++ b/functions/ingest/vendor/github.com/v3io/v3io-tsdb/pkg/appender/ingest.go @@ -49,7 +49,6 @@ func (mc *MetricsCache) metricFeed(index int) { go func() { inFlight := 0 gotData := false - gotCompletion := false potentialCompletion := false var completeChan chan int @@ -64,14 +63,13 @@ func (mc *MetricsCache) metricFeed(index int) { // If data was sent and the queue is empty, mark as completion if length == 0 && gotData { - if len(mc.asyncAppendChan) == 0 { - gotCompletion = true + switch len(mc.asyncAppendChan) { + case 0: + potentialCompletion = true if completeChan != nil { completeChan <- 0 - gotData = false - potentialCompletion = false } - } else if len(mc.asyncAppendChan) == 1 { + case 1: potentialCompletion = true } } @@ -84,15 +82,9 @@ func (mc *MetricsCache) metricFeed(index int) { if app.metric == nil { // Handle update completion requests (metric == nil) completeChan = app.resp - length := mc.metricQueue.Length() - if length == 0 && len(mc.asyncAppendChan) == 0 { - if gotCompletion || (potentialCompletion && gotData) { - completeChan <- 0 - gotCompletion = false - gotData = false - } + if potentialCompletion { + completeChan <- 0 } - potentialCompletion = false } else { potentialCompletion = false // Handle append requests (Add / AddFast) @@ -100,25 +92,23 @@ func (mc *MetricsCache) metricFeed(index int) { metric := app.metric metric.Lock() - if metric.isTimeInvalid(app.t) { - metric.store.Append(app.t, app.v) - numPushed++ - dataQueued += metric.store.samplesQueueLength() - - // If there are no in-flight requests, add the metric to the queue and update state - if metric.isReady() || metric.getState() == storeStateInit { - - if metric.getState() == storeStateInit { - metric.setState(storeStatePreGet) - } - if metric.isReady() { - metric.setState(storeStateUpdate) - } - - length := mc.metricQueue.Push(metric) - if length < 2*mc.cfg.Workers { - newMetrics++ - } + metric.store.Append(app.t, app.v) + numPushed++ + dataQueued += metric.store.samplesQueueLength() + + // If there are no in-flight requests, add the metric to the queue and update state + if metric.isReady() || metric.getState() == storeStateInit { + + if metric.getState() == storeStateInit { + metric.setState(storeStatePreGet) + } + if metric.isReady() { + metric.setState(storeStateUpdate) + } + + length := mc.metricQueue.Push(metric) + if length < 2*mc.cfg.Workers { + newMetrics++ } } metric.Unlock() @@ -217,7 +207,7 @@ func (mc *MetricsCache) metricsUpdateLoop(index int) { } // Notify the metric feeder when all in-flight tasks are done - if mc.updatesInFlight == 0 && len(mc.asyncAppendChan) == 0 { + if mc.updatesInFlight == 0 { mc.logger.Debug("Return to feed. Metric queue length: %d", mc.metricQueue.Length()) mc.updatesComplete <- 0 } diff --git a/functions/ingest/vendor/github.com/v3io/v3io-tsdb/pkg/utils/timeutils.go b/functions/ingest/vendor/github.com/v3io/v3io-tsdb/pkg/utils/timeutils.go index e646083e..6b0c1f03 100644 --- a/functions/ingest/vendor/github.com/v3io/v3io-tsdb/pkg/utils/timeutils.go +++ b/functions/ingest/vendor/github.com/v3io/v3io-tsdb/pkg/utils/timeutils.go @@ -108,11 +108,11 @@ func Str2unixTime(timeString string) (int64, error) { if err != nil { return 0, errors.Wrap(err, "Invalid time string - not an RFC 3339 time format.") } - return t.Unix() * 1000, nil + return t.UnixNano() / int64(time.Millisecond), nil } func CurrentTimeInMillis() int64 { - return time.Now().Unix() * 1000 + return time.Now().UnixNano() / int64(time.Millisecond) } func GetTimeFromRange(from, to, last, step string) (f int64, t int64, s int64, err error) { diff --git a/functions/ingest/vendor/github.com/v3io/v3io-tsdb/pkg/utils/timeutils_test.go b/functions/ingest/vendor/github.com/v3io/v3io-tsdb/pkg/utils/timeutils_test.go index fa48b4b5..82900b33 100644 --- a/functions/ingest/vendor/github.com/v3io/v3io-tsdb/pkg/utils/timeutils_test.go +++ b/functions/ingest/vendor/github.com/v3io/v3io-tsdb/pkg/utils/timeutils_test.go @@ -80,7 +80,10 @@ func (suite *testTimeSuite) TestStr2unixTime() { suite.Require().Nil(err) startTime, err := Str2unixTime("now-1m") suite.Require().Nil(err) - suite.Require().Equal(expectedDuration, endTime-startTime) + actualDuration := endTime - startTime + // "now" might slightly change from call to call, so we allow a safety range + suite.Require().LessOrEqual(expectedDuration-actualDuration, int64(2)) + suite.Require().GreaterOrEqual(expectedDuration-actualDuration, int64(0)) } func (suite *testTimeSuite) TestStr2unixTimeWithNow() { @@ -90,7 +93,10 @@ func (suite *testTimeSuite) TestStr2unixTimeWithNow() { suite.Require().Nil(err) startTime, err := Str2unixTime("now-1m") suite.Require().Nil(err) - suite.Require().Equal(expectedDuration, endTime-startTime) + actualDuration := endTime - startTime + // "now" might slightly change from call to call, so we allow a safety range + suite.Require().LessOrEqual(expectedDuration-actualDuration, int64(2)) + suite.Require().GreaterOrEqual(expectedDuration-actualDuration, int64(0)) } func (suite *testTimeSuite) TestStr2unixTimeWithNowPlus() { @@ -100,7 +106,10 @@ func (suite *testTimeSuite) TestStr2unixTimeWithNowPlus() { suite.Require().Nil(err) startTime, err := Str2unixTime("now-1m") suite.Require().Nil(err) - suite.Require().Equal(expectedDuration, endTime-startTime) + actualDuration := endTime - startTime + // "now" might slightly change from call to call, so we allow a safety range + suite.Require().LessOrEqual(expectedDuration-actualDuration, int64(2)) + suite.Require().GreaterOrEqual(expectedDuration-actualDuration, int64(0)) } func (suite *testTimeSuite) TestStr2unixTimeWithNowPlusMinus() { @@ -110,7 +119,19 @@ func (suite *testTimeSuite) TestStr2unixTimeWithNowPlusMinus() { suite.Require().Nil(err) startTime, err := Str2unixTime("now-") suite.Require().Nil(err) - suite.Require().Equal(expectedDuration, endTime-startTime) + actualDuration := endTime - startTime + // "now" might slightly change from call to call, so we allow a safety range + suite.Require().LessOrEqual(expectedDuration-actualDuration, int64(2)) + suite.Require().GreaterOrEqual(expectedDuration-actualDuration, int64(0)) +} + +func (suite *testTimeSuite) TestRFCStr2unixTime() { + timeStr := "2020-07-20T14:55:56.770808000Z" + realTime, err := time.Parse(time.RFC3339, timeStr) + suite.Require().NoError(err) + timeInMillis, err := Str2unixTime(timeStr) + suite.Require().NoError(err) + suite.Require().Equal(realTime.UnixNano()/int64(time.Millisecond), timeInMillis) } func TestTimeSuite(t *testing.T) { diff --git a/functions/ingest/vendor/github.com/v3io/v3io-tsdb/travis_v3io.yaml b/functions/ingest/vendor/github.com/v3io/v3io-tsdb/test/ci_v3io.yaml similarity index 100% rename from functions/ingest/vendor/github.com/v3io/v3io-tsdb/travis_v3io.yaml rename to functions/ingest/vendor/github.com/v3io/v3io-tsdb/test/ci_v3io.yaml diff --git a/functions/ingest/vendor/github.com/v3io/v3io-tsdb/travis_v3io_bench.yaml b/functions/ingest/vendor/github.com/v3io/v3io-tsdb/test/ci_v3io_bench.yaml similarity index 100% rename from functions/ingest/vendor/github.com/v3io/v3io-tsdb/travis_v3io_bench.yaml rename to functions/ingest/vendor/github.com/v3io/v3io-tsdb/test/ci_v3io_bench.yaml diff --git a/functions/query/vendor/github.com/v3io/v3io-tsdb/.github/workflows/ci.yaml b/functions/query/vendor/github.com/v3io/v3io-tsdb/.github/workflows/ci.yaml new file mode 100644 index 00000000..5b7e0fbe --- /dev/null +++ b/functions/query/vendor/github.com/v3io/v3io-tsdb/.github/workflows/ci.yaml @@ -0,0 +1,132 @@ +name: CI + +on: + pull_request: + branches: + - development + - 'v[0-9]+.[0-9]+.x' + + # Run CI also on push to master + push: + branches: + - master + + release: + types: + - created + +jobs: + test: + name: Lint & test + runs-on: ubuntu-latest + env: + TSDB_TEST_TABLE_PATH: TSDB_INTEGRATION_TESTS/${{ github.sha }} + + steps: + - name: Dump github context + run: echo "$GITHUB_CONTEXT" + env: + GITHUB_CONTEXT: ${{ toJson(github) }} + + - name: Dump runner context + run: echo "$RUNNER_CONTEXT" + env: + RUNNER_CONTEXT: ${{ toJson(runner) }} + + - name: Dump github ref + run: echo "$GITHUB_REF" + + - uses: actions/setup-go@v2 + with: + go-version: "^1.14.0" + + - uses: actions/cache@v2 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + + - uses: actions/checkout@v2 + + - name: Lint + run: make lint + + - name: Test short + run: make test + +# Disabled until GH actions will have a solution to using secret on PR triggered workflows. +# Until then, this runs in travis +# - name: Test integration +# run: make integration +# env: +# V3IO_TSDB_CONFIG: ${{ github.workspace }}/test/ci_v3io.yaml +# +# - name: Benchmark +# run: make bench +# env: +# V3IO_TSDB_CONFIG: ${{ github.workspace }}/test/ci_v3io_bench.yaml +# TSDB_BENCH_INGEST_CONFIG: ${{ github.workspace }}/test/benchmark/testdata/tsdb-bench-test-config-ci.yaml + + release: + name: Release + runs-on: ubuntu-latest + if: github.event_name == 'release' + needs: + - test + strategy: + matrix: + go-os: + - linux + - darwin + - windows + + steps: + - name: Dump GitHub context + run: echo "$GITHUB_CONTEXT" + env: + GITHUB_CONTEXT: ${{ toJson(github) }} + + - name: Dump runner context + run: echo "$RUNNER_CONTEXT" + env: + RUNNER_CONTEXT: ${{ toJson(runner) }} + + - name: Dump github ref + run: echo "$GITHUB_REF" + + - name: Extract ref info + id: release_info + run: | + echo ::set-output name=REF_BRANCH::${GITHUB_REF#refs/heads/} + echo ::set-output name=REF_TAG::${GITHUB_REF#refs/tags/} + + - name: Set TSDB_LABEL to release tag + run: | + echo "::set-env name=TSDB_LABEL::${{ steps.release_info.outputs.REF_TAG }}" + + - uses: actions/checkout@v2 + + - uses: actions/setup-go@v2 + with: + go-version: "^1.14.0" + + - uses: actions/cache@v2 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + + - name: Build binaries for ${{ matrix.go-os }} + run: make bin + env: + GOPATH: ${{ github.workspace }}/go + GOARCH: amd64 + GOOS: ${{ matrix.go-os }} + + - name: Upload binaries + uses: AButler/upload-release-assets@v2.0 + with: + files: ${{ github.workspace }}/go/bin/tsdbctl-* + repo-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/functions/query/vendor/github.com/v3io/v3io-tsdb/.gitignore b/functions/query/vendor/github.com/v3io/v3io-tsdb/.gitignore index ddd81e7f..1fe5870c 100644 --- a/functions/query/vendor/github.com/v3io/v3io-tsdb/.gitignore +++ b/functions/query/vendor/github.com/v3io/v3io-tsdb/.gitignore @@ -5,3 +5,4 @@ *.sln.iml .idea/ +tsdbctl-* diff --git a/functions/query/vendor/github.com/v3io/v3io-tsdb/.travis.yml b/functions/query/vendor/github.com/v3io/v3io-tsdb/.travis.yml index c845894d..44baf85d 100644 --- a/functions/query/vendor/github.com/v3io/v3io-tsdb/.travis.yml +++ b/functions/query/vendor/github.com/v3io/v3io-tsdb/.travis.yml @@ -9,10 +9,5 @@ env: script: - make test - make lint - - V3IO_TSDB_CONFIG="$TRAVIS_BUILD_DIR/travis_v3io.yaml" make integration - - V3IO_TSDB_CONFIG="$TRAVIS_BUILD_DIR/travis_v3io_bench.yaml" TSDB_BENCH_INGEST_CONFIG="$TRAVIS_BUILD_DIR/test/benchmark/testdata/tsdb-bench-test-config-ci.yaml" make bench - -before_deploy: - - GOOS=linux GOARCH=amd64 make build - - GOOS=darwin GOARCH=amd64 make build - - GOOS=windows GOARCH=amd64 make build + - V3IO_TSDB_CONFIG="$TRAVIS_BUILD_DIR/test/ci_v3io.yaml" make integration + - V3IO_TSDB_CONFIG="$TRAVIS_BUILD_DIR/test/ci_v3io_bench.yaml" TSDB_BENCH_INGEST_CONFIG="$TRAVIS_BUILD_DIR/test/benchmark/testdata/tsdb-bench-test-config-ci.yaml" make bench diff --git a/functions/query/vendor/github.com/v3io/v3io-tsdb/Jenkinsfile b/functions/query/vendor/github.com/v3io/v3io-tsdb/Jenkinsfile index 6b4f15b8..dec17c14 100644 --- a/functions/query/vendor/github.com/v3io/v3io-tsdb/Jenkinsfile +++ b/functions/query/vendor/github.com/v3io/v3io-tsdb/Jenkinsfile @@ -9,54 +9,6 @@ git_deploy_user_token = "iguazio-prod-git-user-token" git_deploy_user_private_key = "iguazio-prod-git-user-private-key" -def build_v3io_tsdb(TAG_VERSION) { - withCredentials([ - string(credentialsId: git_deploy_user_token, variable: 'GIT_TOKEN') - ]) { - def git_project = 'v3io-tsdb' - stage('prepare sources') { - container('jnlp') { - dir("${BUILD_FOLDER}/src/github.com/${git_project_upstream_user}/${git_project}") { - git(changelog: false, credentialsId: git_deploy_user_private_key, poll: false, url: "git@github.com:${git_project_user}/${git_project}.git") - sh("git checkout ${TAG_VERSION}") - } - } - } - - stage("build ${git_project} binaries in dood") { - container('golang') { - dir("${BUILD_FOLDER}/src/github.com/${git_project_upstream_user}/${git_project}") { - sh """ - mkdir -p /home/jenkins/go/bin - GO111MODULE=on GOOS=linux GOARCH=amd64 TRAVIS_TAG=${TAG_VERSION} make bin - GO111MODULE=on GOOS=darwin GOARCH=amd64 TRAVIS_TAG=${TAG_VERSION} make bin - GO111MODULE=on GOOS=windows GOARCH=amd64 TRAVIS_TAG=${TAG_VERSION} make bin - ls -la /go/bin - mv /go/bin/* /home/jenkins/go/bin/ - ls -la /home/jenkins/go/bin/ - - """ - } - } - } - - stage('upload release assets') { - container('jnlp') { - RELEASE_ID = github.get_release_id(git_project, git_project_user, "${TAG_VERSION}", GIT_TOKEN) - - github.upload_asset(git_project, git_project_user, "tsdbctl-${TAG_VERSION}-linux-amd64", RELEASE_ID, GIT_TOKEN) - github.upload_asset(git_project, git_project_user, "tsdbctl-${TAG_VERSION}-darwin-amd64", RELEASE_ID, GIT_TOKEN) - github.upload_asset(git_project, git_project_user, "tsdbctl-${TAG_VERSION}-windows-amd64", RELEASE_ID, GIT_TOKEN) - withCredentials([ - string(credentialsId: pipelinex.PackagesRepo.ARTIFACTORY_IGUAZIO[2], variable: 'PACKAGES_ARTIFACTORY_PASSWORD') - ]) { - common.upload_file_to_artifactory(pipelinex.PackagesRepo.ARTIFACTORY_IGUAZIO[0], pipelinex.PackagesRepo.ARTIFACTORY_IGUAZIO[1], PACKAGES_ARTIFACTORY_PASSWORD, "iguazio-devops/k8s", "tsdbctl-${TAG_VERSION}-linux-amd64") - } - } - } - } -} - def build_nuclio(V3IO_TSDB_VERSION, internal_status="stable") { withCredentials([ usernamePassword(credentialsId: git_deploy_user, passwordVariable: 'GIT_PASSWORD', usernameVariable: 'GIT_USERNAME'), @@ -413,17 +365,6 @@ podTemplate(label: "${git_project}-${label}", inheritFrom: "jnlp-docker-golang") if (github.check_tag_expiration(git_project, git_project_user, MAIN_TAG_VERSION, GIT_TOKEN)) { parallel( - 'v3io-tsdb': { - podTemplate(label: "v3io-tsdb-${label}", inheritFrom: "jnlp-docker-golang") { - node("v3io-tsdb-${label}") { - withCredentials([ - string(credentialsId: git_deploy_user_token, variable: 'GIT_TOKEN') - ]) { - build_v3io_tsdb(MAIN_TAG_VERSION) - } - } - } - }, 'tsdb-nuclio': { podTemplate(label: "v3io-tsdb-nuclio-${label}", inheritFrom: "jnlp-docker-golang") { node("v3io-tsdb-nuclio-${label}") { diff --git a/functions/query/vendor/github.com/v3io/v3io-tsdb/Makefile b/functions/query/vendor/github.com/v3io/v3io-tsdb/Makefile index c0bc9855..ebf9b9c1 100644 --- a/functions/query/vendor/github.com/v3io/v3io-tsdb/Makefile +++ b/functions/query/vendor/github.com/v3io/v3io-tsdb/Makefile @@ -4,8 +4,8 @@ ifeq ($(GIT_BRANCH),) GIT_BRANCH="N/A" endif -ifneq ($(TRAVIS_TAG),) - GIT_REVISION := $(TRAVIS_TAG) +ifneq ($(TSDB_LABEL),) + GIT_REVISION := $(TSDB_LABEL) else GIT_REVISION := $(shell git describe --always) endif diff --git a/functions/query/vendor/github.com/v3io/v3io-tsdb/README.md b/functions/query/vendor/github.com/v3io/v3io-tsdb/README.md index b99720eb..3d77474a 100644 --- a/functions/query/vendor/github.com/v3io/v3io-tsdb/README.md +++ b/functions/query/vendor/github.com/v3io/v3io-tsdb/README.md @@ -1,4 +1,5 @@ -[![Build Status](https://travis-ci.org/v3io/v3io-tsdb.svg?branch=master)](https://travis-ci.org/v3io/v3io-tsdb) +[![Travis Build Status](https://travis-ci.org/v3io/v3io-tsdb.svg?branch=master)](https://travis-ci.org/v3io/v3io-tsdb) +[![GH Build Status](https://github.com/v3io/v3io-tsdb/workflows/CI/badge.svg)](https://github.com/v3io/v3io-tsdb/actions) # V3IO-TSDB Iguazio API lib for time-series DB access and Prometheus TSDB storage driver. diff --git a/functions/query/vendor/github.com/v3io/v3io-tsdb/pkg/appender/appender.go b/functions/query/vendor/github.com/v3io/v3io-tsdb/pkg/appender/appender.go index 19322404..26fd9ea5 100644 --- a/functions/query/vendor/github.com/v3io/v3io-tsdb/pkg/appender/appender.go +++ b/functions/query/vendor/github.com/v3io/v3io-tsdb/pkg/appender/appender.go @@ -77,10 +77,6 @@ func (m *MetricState) isReady() bool { return m.state == storeStateReady } -func (m *MetricState) isTimeInvalid(t int64) bool { - return !((m.state == storeStateReady || m.state == storeStateUpdate) && t < m.store.maxTime-maxLateArrivalInterval) -} - func (m *MetricState) getState() storeState { return m.state } diff --git a/functions/query/vendor/github.com/v3io/v3io-tsdb/pkg/appender/ingest.go b/functions/query/vendor/github.com/v3io/v3io-tsdb/pkg/appender/ingest.go index 032c8e0b..eb349e0e 100644 --- a/functions/query/vendor/github.com/v3io/v3io-tsdb/pkg/appender/ingest.go +++ b/functions/query/vendor/github.com/v3io/v3io-tsdb/pkg/appender/ingest.go @@ -49,7 +49,6 @@ func (mc *MetricsCache) metricFeed(index int) { go func() { inFlight := 0 gotData := false - gotCompletion := false potentialCompletion := false var completeChan chan int @@ -64,14 +63,13 @@ func (mc *MetricsCache) metricFeed(index int) { // If data was sent and the queue is empty, mark as completion if length == 0 && gotData { - if len(mc.asyncAppendChan) == 0 { - gotCompletion = true + switch len(mc.asyncAppendChan) { + case 0: + potentialCompletion = true if completeChan != nil { completeChan <- 0 - gotData = false - potentialCompletion = false } - } else if len(mc.asyncAppendChan) == 1 { + case 1: potentialCompletion = true } } @@ -84,15 +82,9 @@ func (mc *MetricsCache) metricFeed(index int) { if app.metric == nil { // Handle update completion requests (metric == nil) completeChan = app.resp - length := mc.metricQueue.Length() - if length == 0 && len(mc.asyncAppendChan) == 0 { - if gotCompletion || (potentialCompletion && gotData) { - completeChan <- 0 - gotCompletion = false - gotData = false - } + if potentialCompletion { + completeChan <- 0 } - potentialCompletion = false } else { potentialCompletion = false // Handle append requests (Add / AddFast) @@ -100,25 +92,23 @@ func (mc *MetricsCache) metricFeed(index int) { metric := app.metric metric.Lock() - if metric.isTimeInvalid(app.t) { - metric.store.Append(app.t, app.v) - numPushed++ - dataQueued += metric.store.samplesQueueLength() - - // If there are no in-flight requests, add the metric to the queue and update state - if metric.isReady() || metric.getState() == storeStateInit { - - if metric.getState() == storeStateInit { - metric.setState(storeStatePreGet) - } - if metric.isReady() { - metric.setState(storeStateUpdate) - } - - length := mc.metricQueue.Push(metric) - if length < 2*mc.cfg.Workers { - newMetrics++ - } + metric.store.Append(app.t, app.v) + numPushed++ + dataQueued += metric.store.samplesQueueLength() + + // If there are no in-flight requests, add the metric to the queue and update state + if metric.isReady() || metric.getState() == storeStateInit { + + if metric.getState() == storeStateInit { + metric.setState(storeStatePreGet) + } + if metric.isReady() { + metric.setState(storeStateUpdate) + } + + length := mc.metricQueue.Push(metric) + if length < 2*mc.cfg.Workers { + newMetrics++ } } metric.Unlock() @@ -217,7 +207,7 @@ func (mc *MetricsCache) metricsUpdateLoop(index int) { } // Notify the metric feeder when all in-flight tasks are done - if mc.updatesInFlight == 0 && len(mc.asyncAppendChan) == 0 { + if mc.updatesInFlight == 0 { mc.logger.Debug("Return to feed. Metric queue length: %d", mc.metricQueue.Length()) mc.updatesComplete <- 0 } diff --git a/functions/query/vendor/github.com/v3io/v3io-tsdb/pkg/utils/timeutils.go b/functions/query/vendor/github.com/v3io/v3io-tsdb/pkg/utils/timeutils.go index e646083e..6b0c1f03 100644 --- a/functions/query/vendor/github.com/v3io/v3io-tsdb/pkg/utils/timeutils.go +++ b/functions/query/vendor/github.com/v3io/v3io-tsdb/pkg/utils/timeutils.go @@ -108,11 +108,11 @@ func Str2unixTime(timeString string) (int64, error) { if err != nil { return 0, errors.Wrap(err, "Invalid time string - not an RFC 3339 time format.") } - return t.Unix() * 1000, nil + return t.UnixNano() / int64(time.Millisecond), nil } func CurrentTimeInMillis() int64 { - return time.Now().Unix() * 1000 + return time.Now().UnixNano() / int64(time.Millisecond) } func GetTimeFromRange(from, to, last, step string) (f int64, t int64, s int64, err error) { diff --git a/functions/query/vendor/github.com/v3io/v3io-tsdb/pkg/utils/timeutils_test.go b/functions/query/vendor/github.com/v3io/v3io-tsdb/pkg/utils/timeutils_test.go index fa48b4b5..82900b33 100644 --- a/functions/query/vendor/github.com/v3io/v3io-tsdb/pkg/utils/timeutils_test.go +++ b/functions/query/vendor/github.com/v3io/v3io-tsdb/pkg/utils/timeutils_test.go @@ -80,7 +80,10 @@ func (suite *testTimeSuite) TestStr2unixTime() { suite.Require().Nil(err) startTime, err := Str2unixTime("now-1m") suite.Require().Nil(err) - suite.Require().Equal(expectedDuration, endTime-startTime) + actualDuration := endTime - startTime + // "now" might slightly change from call to call, so we allow a safety range + suite.Require().LessOrEqual(expectedDuration-actualDuration, int64(2)) + suite.Require().GreaterOrEqual(expectedDuration-actualDuration, int64(0)) } func (suite *testTimeSuite) TestStr2unixTimeWithNow() { @@ -90,7 +93,10 @@ func (suite *testTimeSuite) TestStr2unixTimeWithNow() { suite.Require().Nil(err) startTime, err := Str2unixTime("now-1m") suite.Require().Nil(err) - suite.Require().Equal(expectedDuration, endTime-startTime) + actualDuration := endTime - startTime + // "now" might slightly change from call to call, so we allow a safety range + suite.Require().LessOrEqual(expectedDuration-actualDuration, int64(2)) + suite.Require().GreaterOrEqual(expectedDuration-actualDuration, int64(0)) } func (suite *testTimeSuite) TestStr2unixTimeWithNowPlus() { @@ -100,7 +106,10 @@ func (suite *testTimeSuite) TestStr2unixTimeWithNowPlus() { suite.Require().Nil(err) startTime, err := Str2unixTime("now-1m") suite.Require().Nil(err) - suite.Require().Equal(expectedDuration, endTime-startTime) + actualDuration := endTime - startTime + // "now" might slightly change from call to call, so we allow a safety range + suite.Require().LessOrEqual(expectedDuration-actualDuration, int64(2)) + suite.Require().GreaterOrEqual(expectedDuration-actualDuration, int64(0)) } func (suite *testTimeSuite) TestStr2unixTimeWithNowPlusMinus() { @@ -110,7 +119,19 @@ func (suite *testTimeSuite) TestStr2unixTimeWithNowPlusMinus() { suite.Require().Nil(err) startTime, err := Str2unixTime("now-") suite.Require().Nil(err) - suite.Require().Equal(expectedDuration, endTime-startTime) + actualDuration := endTime - startTime + // "now" might slightly change from call to call, so we allow a safety range + suite.Require().LessOrEqual(expectedDuration-actualDuration, int64(2)) + suite.Require().GreaterOrEqual(expectedDuration-actualDuration, int64(0)) +} + +func (suite *testTimeSuite) TestRFCStr2unixTime() { + timeStr := "2020-07-20T14:55:56.770808000Z" + realTime, err := time.Parse(time.RFC3339, timeStr) + suite.Require().NoError(err) + timeInMillis, err := Str2unixTime(timeStr) + suite.Require().NoError(err) + suite.Require().Equal(realTime.UnixNano()/int64(time.Millisecond), timeInMillis) } func TestTimeSuite(t *testing.T) { diff --git a/functions/query/vendor/github.com/v3io/v3io-tsdb/travis_v3io.yaml b/functions/query/vendor/github.com/v3io/v3io-tsdb/test/ci_v3io.yaml similarity index 100% rename from functions/query/vendor/github.com/v3io/v3io-tsdb/travis_v3io.yaml rename to functions/query/vendor/github.com/v3io/v3io-tsdb/test/ci_v3io.yaml diff --git a/functions/query/vendor/github.com/v3io/v3io-tsdb/travis_v3io_bench.yaml b/functions/query/vendor/github.com/v3io/v3io-tsdb/test/ci_v3io_bench.yaml similarity index 100% rename from functions/query/vendor/github.com/v3io/v3io-tsdb/travis_v3io_bench.yaml rename to functions/query/vendor/github.com/v3io/v3io-tsdb/test/ci_v3io_bench.yaml