Skip to content

Commit

Permalink
[dbnode] Remove namespaces from example config and integration tests (#…
Browse files Browse the repository at this point in the history
  • Loading branch information
nbroyles committed Nov 10, 2020
1 parent 345a67e commit 8a88279
Show file tree
Hide file tree
Showing 33 changed files with 270 additions and 241 deletions.
10 changes: 1 addition & 9 deletions scripts/docker-integration-tests/aggregator/m3coordinator.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,15 +12,7 @@ carbon:
retention: 6h

clusters:
- namespaces:
- namespace: agg
type: aggregated
resolution: 10s
retention: 6h
- namespace: unagg
type: unaggregated
retention: 1s
client:
- client:
config:
service:
env: default_env
Expand Down
2 changes: 1 addition & 1 deletion scripts/docker-integration-tests/aggregator/test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ function defer {
trap defer EXIT

echo "Setup DB node"
setup_single_m3db_node
AGG_RESOLUTION=10s AGG_RETENTION=6h setup_single_m3db_node

echo "Initializing aggregator topology"
curl -vvvsSf -X POST -H "Cluster-Environment-Name: override_test_env" localhost:7201/api/v1/services/m3aggregator/placement/init -d '{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,15 +12,7 @@ carbon:
retention: 6h

clusters:
- namespaces:
- namespace: agg
type: aggregated
resolution: 10s
retention: 6h
- namespace: unagg
type: unaggregated
retention: 1s
client:
- client:
config:
service:
env: default_env
Expand Down
2 changes: 1 addition & 1 deletion scripts/docker-integration-tests/aggregator_legacy/test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ function defer {
trap defer EXIT

echo "Setup DB node"
setup_single_m3db_node
AGG_RESOLUTION=10s AGG_RETENTION=6h setup_single_m3db_node

echo "Initializing aggregator topology"
curl -vvvsSf -X POST -H "Cluster-Environment-Name: override_test_env" localhost:7201/api/v1/services/m3aggregator/placement/init -d '{
Expand Down
10 changes: 1 addition & 9 deletions scripts/docker-integration-tests/carbon/m3coordinator.yml
Original file line number Diff line number Diff line change
@@ -1,13 +1,5 @@
clusters:
- namespaces:
- namespace: agg
type: aggregated
retention: 10h
resolution: 5s
- namespace: unagg
type: unaggregated
retention: 10m
client:
- client:
config:
service:
env: default_env
Expand Down
2 changes: 1 addition & 1 deletion scripts/docker-integration-tests/carbon/test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ function defer {
}
trap defer EXIT

setup_single_m3db_node
AGG_RESOLUTION=5s setup_single_m3db_node

function read_carbon {
target=$1
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,7 @@ limits:
maxFetchedSeries: 100

clusters:
- namespaces:
- namespace: agg
type: aggregated
retention: 10h
resolution: 15s
- namespace: unagg
type: unaggregated
retention: 10h
client:
- client:
config:
service:
env: default_env
Expand Down
105 changes: 102 additions & 3 deletions scripts/docker-integration-tests/common.sh
Original file line number Diff line number Diff line change
Expand Up @@ -74,10 +74,32 @@ function setup_single_m3db_node_long_namespaces {
]
}'

echo "Updating aggregation options for agg namespace"
curl -vvvsSf -X PUT 0.0.0.0:${coordinator_port}/api/v1/services/m3db/namespace -d '{
"name": "agg",
"options": {
"aggregationOptions": {
"aggregations": [
{
"aggregated": true,
"attributes": {
"resolutionDuration": "30s",
"downsampleOptions": { "all": false }
}
}
]
}
}
}'

echo "Wait until placement is init'd"
ATTEMPTS=10 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff \
'[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/placement | jq .placement.instances.'${dbnode_id}'.id)" == \"'${dbnode_id}'\" ]'

echo "Wait until agg namespace is ready"
ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff \
'[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/namespace/ready -d "{ \"name\": \"agg\"}" | grep -c true)" -eq 1 ]'

wait_for_namespaces

echo "Adding agg2d namespace"
Expand All @@ -86,10 +108,31 @@ function setup_single_m3db_node_long_namespaces {
"retentionTime": "48h"
}'

echo "Updating aggregation options for agg namespace"
curl -vvvsSf -X PUT 0.0.0.0:${coordinator_port}/api/v1/services/m3db/namespace -d '{
"name": "agg2d",
"options": {
"aggregationOptions": {
"aggregations": [
{
"aggregated": true,
"attributes": {
"resolutionDuration": "1m",
"downsampleOptions": { "all": false }
}
}
]
}
}
}'

echo "Wait until agg2d namespace is init'd"
ATTEMPTS=10 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff \
'[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/namespace | jq .registry.namespaces.agg2d.indexOptions.enabled)" == true ]'

echo "Wait until agg2d namespace is ready"
ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff \
'[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/namespace/ready -d "{ \"name\": \"agg2d\"}" | grep -c true)" -eq 1 ]'

echo "Wait until bootstrapped"
ATTEMPTS=100 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff \
Expand All @@ -103,6 +146,8 @@ function setup_single_m3db_node {
local dbnode_id=${DBNODE_ID:-m3db_local}
local coordinator_port=${COORDINATOR_PORT:-7201}
local zone=${ZONE:-embedded}
local agg_resolution=${AGG_RESOLUTION:-15s}
local agg_retention=${AGG_RETENTION:-10h}

echo "Wait for API to be available"
ATTEMPTS=100 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff \
Expand All @@ -112,7 +157,7 @@ function setup_single_m3db_node {
curl -vvvsSf -X POST 0.0.0.0:${coordinator_port}/api/v1/database/create -d '{
"type": "cluster",
"namespaceName": "agg",
"retentionTime": "6h",
"retentionTime": "'${agg_retention}'",
"num_shards": 4,
"replicationFactor": 1,
"hosts": [
Expand All @@ -127,10 +172,31 @@ function setup_single_m3db_node {
]
}'

echo "Updating aggregation options for agg namespace"
curl -vvvsSf -X PUT 0.0.0.0:${coordinator_port}/api/v1/services/m3db/namespace -d '{
"name": "agg",
"options": {
"aggregationOptions": {
"aggregations": [
{
"aggregated": true,
"attributes": {
"resolutionDuration": "'${agg_resolution}'"
}
}
]
}
}
}'

echo "Wait until placement is init'd"
ATTEMPTS=10 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff \
'[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/placement | jq .placement.instances.'${dbnode_id}'.id)" == \"'${dbnode_id}'\" ]'

echo "Wait until agg namespace is ready"
ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff \
'[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/namespace/ready -d "{ \"name\": \"agg\"}" | grep -c true)" -eq 1 ]'

wait_for_namespaces

echo "Wait until bootstrapped"
Expand All @@ -147,6 +213,8 @@ function setup_two_m3db_nodes {
local dbnode_host_1_health_port=${DBNODE_HEALTH_PORT_01:-9012}
local dbnode_host_2_health_port=${DBNODE_HEALTH_PORT_02:-9022}
local coordinator_port=${COORDINATOR_PORT:-7201}
local agg_resolution=${AGG_RESOLUTION:-15s}
local agg_retention=${AGG_RETENTION:-10h}

echo "Wait for API to be available"
ATTEMPTS=100 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff \
Expand All @@ -156,7 +224,7 @@ function setup_two_m3db_nodes {
curl -vvvsSf -X POST 0.0.0.0:${coordinator_port}/api/v1/database/create -d '{
"type": "cluster",
"namespaceName": "agg",
"retentionTime": "6h",
"retentionTime": "'${agg_retention}'",
"num_shards": 2,
"replicationFactor": 2,
"hosts": [
Expand All @@ -179,10 +247,31 @@ function setup_two_m3db_nodes {
]
}'

echo "Updating aggregation options for agg namespace"
curl -vvvsSf -X PUT 0.0.0.0:${coordinator_port}/api/v1/services/m3db/namespace -d '{
"name": "agg",
"options": {
"aggregationOptions": {
"aggregations": [
{
"aggregated": true,
"attributes": {
"resolutionDuration": "'${agg_resolution}'"
}
}
]
}
}
}'

echo "Wait until placement is init'd"
ATTEMPTS=10 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff \
'[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/placement | jq .placement.instances.'"${dbnode_id_1}"'.id)" == \"'"${dbnode_id_1}"'\" ]'

echo "Wait until agg namespace is ready"
ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff \
'[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/namespace/ready -d "{ \"name\": \"agg\"}" | grep -c true)" -eq 1 ]'

wait_for_namespaces

echo "Wait until bootstrapped"
Expand All @@ -194,6 +283,7 @@ function setup_two_m3db_nodes {

function wait_for_namespaces {
local coordinator_port=${COORDINATOR_PORT:-7201}
local unagg_retention=${UNAGG_RETENTION:-10h}

echo "Wait until agg namespace is init'd"
ATTEMPTS=10 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff \
Expand All @@ -202,13 +292,17 @@ function wait_for_namespaces {
echo "Adding unagg namespace"
curl -vvvsSf -X POST 0.0.0.0:${coordinator_port}/api/v1/database/namespace/create -d '{
"namespaceName": "unagg",
"retentionTime": "6h"
"retentionTime": "'${unagg_retention}'"
}'

echo "Wait until unagg namespace is init'd"
ATTEMPTS=10 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff \
'[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/namespace | jq .registry.namespaces.unagg.indexOptions.enabled)" == true ]'

echo "Wait until unagg namespace is ready"
ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff \
'[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/namespace/ready -d "{ \"name\": \"unagg\"}" | grep -c true)" -eq 1 ]'

echo "Adding coldWritesRepairAndNoIndex namespace"
curl -vvvsSf -X POST 0.0.0.0:${coordinator_port}/api/v1/services/m3db/namespace -d '{
"name": "coldWritesRepairAndNoIndex",
Expand All @@ -227,6 +321,11 @@ function wait_for_namespaces {
"bufferPastDuration": "10m",
"blockDataExpiry": true,
"blockDataExpiryAfterNotAccessPeriodDuration": "5m"
},
"aggregationOptions": {
"aggregations": [
{ "aggregated": false }
]
}
}
}'
Expand Down
Original file line number Diff line number Diff line change
@@ -1,21 +1,5 @@
clusters:
- namespaces:
- namespace: agg
type: aggregated
retention: 24h
resolution: 30s
downsample:
all: false
- namespace: agg2d
type: aggregated
retention: 48h
resolution: 1m
downsample:
all: false
- namespace: unagg
type: unaggregated
retention: 10m
client:
- client:
config:
service:
env: default_env
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,7 @@ limits:
maxFetchedSeries: 100

clusters:
- namespaces:
- namespace: agg
type: aggregated
retention: 10h
resolution: 15s
- namespace: unagg
type: unaggregated
retention: 10h
client:
- client:
config:
service:
env: default_env
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,7 @@ limits:
maxFetchedSeries: 100

clusters:
- namespaces:
- namespace: agg
type: aggregated
retention: 10h
resolution: 15s
- namespace: unagg
type: unaggregated
retention: 10h
client:
- client:
config:
service:
env: default_env
Expand Down
10 changes: 1 addition & 9 deletions scripts/docker-integration-tests/prometheus/m3coordinator.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,7 @@ limits:
maxFetchedSeries: 100

clusters:
- namespaces:
- namespace: agg
type: aggregated
retention: 10h
resolution: 15s
- namespace: unagg
type: unaggregated
retention: 10m
client:
- client:
config:
service:
env: default_env
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,7 @@ writeForwarding:
- url: http://coordinator02:7201/api/v1/prom/remote/write

clusters:
- namespaces:
- namespace: agg
type: aggregated
retention: 10h
resolution: 15s
- namespace: unagg
type: unaggregated
retention: 10m
client:
- client:
config:
service:
env: default_env
Expand Down
Loading

0 comments on commit 8a88279

Please sign in to comment.