From 2e43ba7744a91287535ff7098224395b9c501e53 Mon Sep 17 00:00:00 2001 From: MdSahil-oss Date: Mon, 12 Jun 2023 00:46:50 +0530 Subject: [PATCH 1/8] Added Dockerfile.cs.template file Signed-off-by: MdSahil-oss --- Dockerfile.cs.template | 10 ++++++++++ docker-entrypoint-cs.sh | 1 + 2 files changed, 11 insertions(+) create mode 100644 Dockerfile.cs.template create mode 100644 docker-entrypoint-cs.sh diff --git a/Dockerfile.cs.template b/Dockerfile.cs.template new file mode 100644 index 00000000..fe69b923 --- /dev/null +++ b/Dockerfile.cs.template @@ -0,0 +1,10 @@ +# %%MARIADB_VERSION%% will be set by update.sh according to generating mariadb version image. +FROM mariadb:%%MARIADB_VERSION%% + +RUN apt-get update && \ + apt-get install -y mariadb-plugin-columnstore=$MARIADB_VERSION ; \ + rm -rf /var/lib/apt/lists/* + +COPY docker-entrypoint-cs.sh /usr/local/bin/ + +ENTRYPOINT ["docker-entrypoint-cs.sh"] \ No newline at end of file diff --git a/docker-entrypoint-cs.sh b/docker-entrypoint-cs.sh new file mode 100644 index 00000000..c5d58ad2 --- /dev/null +++ b/docker-entrypoint-cs.sh @@ -0,0 +1 @@ +# Add docker-column-store image related scripts. \ No newline at end of file From ffaa2d8de6371cd63989b936611bba48c6e42950 Mon Sep 17 00:00:00 2001 From: sahil Date: Thu, 15 Jun 2023 15:21:31 +0530 Subject: [PATCH 2/8] Added sample docker-entrypoint-cs.sh Signed-off-by: sahil --- docker-entrypoint-cs.sh | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/docker-entrypoint-cs.sh b/docker-entrypoint-cs.sh index c5d58ad2..7d0e512d 100644 --- a/docker-entrypoint-cs.sh +++ b/docker-entrypoint-cs.sh @@ -1 +1,30 @@ -# Add docker-column-store image related scripts. \ No newline at end of file +#!/bin/bash +set -eo pipefail + +# logging functions +mysql_log() { + local type="$1"; shift + printf '%s [%s] [Entrypoint]: %s\n' "$(date --rfc-3339=seconds)" "$type" "$*" +} +mysql_note() { + mysql_log Note "$@" +} +mysql_warn() { + mysql_log Warn "$@" >&2 +} +mysql_error() { + mysql_log ERROR "$@" >&2 + exit 1 +} + +_main() { + if [ "$1" = 'mariadbd' ]; then + # simply start maridb with column-store + elif [ "$1" = 'StorageManager' ]; then + # Run StorageManage of column-store + elif [ "$1" = 'brm' ]; then + # Run brm + fi +} + +_main "$@" \ No newline at end of file From 08f1f27bc22b7d5ac984dfc2902d1149f0a11c47 Mon Sep 17 00:00:00 2001 From: sahil Date: Fri, 16 Jun 2023 15:02:40 +0530 Subject: [PATCH 3/8] Added mariad functionality Signed-off-by: sahil --- docker-entrypoint-cs.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-entrypoint-cs.sh b/docker-entrypoint-cs.sh index 7d0e512d..057c295b 100644 --- a/docker-entrypoint-cs.sh +++ b/docker-entrypoint-cs.sh @@ -20,6 +20,7 @@ mysql_error() { _main() { if [ "$1" = 'mariadbd' ]; then # simply start maridb with column-store + exec docker-entrypoint.sh "$@" elif [ "$1" = 'StorageManager' ]; then # Run StorageManage of column-store elif [ "$1" = 'brm' ]; then @@ -27,4 +28,5 @@ _main() { fi } +# Runs _main() function. _main "$@" \ No newline at end of file From 9302ccfc0032fb5a514db319d561dad6c8a71766 Mon Sep 17 00:00:00 2001 From: Md Sahil Date: Sat, 23 Sep 2023 01:32:08 +0530 Subject: [PATCH 4/8] Updated update.sh to create column store specific directory Signed-off-by: Md Sahil --- docker-entrypoint-cs.sh | 4 +++- update.sh | 25 ++++++++++++++++++++++++- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/docker-entrypoint-cs.sh b/docker-entrypoint-cs.sh index 057c295b..4de12a9f 100644 --- a/docker-entrypoint-cs.sh +++ b/docker-entrypoint-cs.sh @@ -19,10 +19,12 @@ mysql_error() { _main() { if [ "$1" = 'mariadbd' ]; then - # simply start maridb with column-store + # simply start mariadb with column-store exec docker-entrypoint.sh "$@" elif [ "$1" = 'StorageManager' ]; then # Run StorageManage of column-store + # storageManager's directroy :- /var/lib/columnstore/storagemanager + print "$@" elif [ "$1" = 'brm' ]; then # Run brm fi diff --git a/update.sh b/update.sh index b0eb27ab..412c1a17 100755 --- a/update.sh +++ b/update.sh @@ -29,6 +29,25 @@ declare -A suffix=( # For testing with https://downloads.dev.mariadb.org/rest-api typeset -r DOWNLOADS_REST_API="https://downloads.mariadb.org/rest-api" +update_cs_version() +{ + echo "Creating column store specific directory for mariadb version $version: $mariaVersion ($releaseStatus)" + + cp Dockerfile.cs.template "cs-$version/Dockerfile" + + cp docker-entrypoint-cs.sh healthcheck.sh "cs-$version/" + chmod a+x "cs-$version"/healthcheck.sh + sed -i \ + -e 's!%%MARIADB_VERSION%%!'"$fullVersion"'!g' \ + -e 's!%%MARIADB_VERSION_BASIC%%!'"$mariaVersion"'!g' \ + -e 's!%%MARIADB_MAJOR%%!'"$version"'!g' \ + -e 's!%%MARIADB_RELEASE_STATUS%%!'"$releaseStatus"'!g' \ + -e 's!%%MARIADB_SUPPORT_TYPE%%!'"$supportType"'!g' \ + -e 's!%%SUITE%%!'"$suite"'!g' \ + -e 's!%%ARCHES%%! '"$arches"'!g' \ + "cs-$version/Dockerfile" +} + update_version() { echo "$version: $mariaVersion ($releaseStatus)" @@ -163,7 +182,7 @@ fi versions=( "$@" ) for version in "${versions[@]}"; do - if [ "$version" == $development_version ]; then + if [ "$version" == $development_version ] || [ "${versions[0]}" == "--cs" ] && [ "$#" -eq 1 ]; then in_development continue fi @@ -180,3 +199,7 @@ for version in "${versions[@]}"; do update_version done + +if [ "${versions[0]}" == "--cs" ]; then + update_cs_version +fi From c72ad5db42bc869c1811f12110217f4d23db1888 Mon Sep 17 00:00:00 2001 From: Md Sahil Date: Thu, 28 Sep 2023 01:46:07 +0530 Subject: [PATCH 5/8] Updates update.sh Signed-off-by: Md Sahil --- Dockerfile.cs.template | 2 +- update.sh | 41 ++++++++++++++++++++++++----------------- 2 files changed, 25 insertions(+), 18 deletions(-) diff --git a/Dockerfile.cs.template b/Dockerfile.cs.template index fe69b923..cd378997 100644 --- a/Dockerfile.cs.template +++ b/Dockerfile.cs.template @@ -2,7 +2,7 @@ FROM mariadb:%%MARIADB_VERSION%% RUN apt-get update && \ - apt-get install -y mariadb-plugin-columnstore=$MARIADB_VERSION ; \ + apt-get install -y mariadb-plugin-columnstore=%%MARIADB_VERSION%% ; \ rm -rf /var/lib/apt/lists/* COPY docker-entrypoint-cs.sh /usr/local/bin/ diff --git a/update.sh b/update.sh index 412c1a17..42af87cf 100755 --- a/update.sh +++ b/update.sh @@ -33,19 +33,21 @@ update_cs_version() { echo "Creating column store specific directory for mariadb version $version: $mariaVersion ($releaseStatus)" - cp Dockerfile.cs.template "cs-$version/Dockerfile" + mkdir "$cs_dir" + cp Dockerfile.cs.template "$cs_dir/Dockerfile" - cp docker-entrypoint-cs.sh healthcheck.sh "cs-$version/" - chmod a+x "cs-$version"/healthcheck.sh + cp docker-entrypoint-cs.sh healthcheck.sh "$cs_dir/" + chmod a+x "$cs_dir"/healthcheck.sh sed -i \ - -e 's!%%MARIADB_VERSION%%!'"$fullVersion"'!g' \ - -e 's!%%MARIADB_VERSION_BASIC%%!'"$mariaVersion"'!g' \ - -e 's!%%MARIADB_MAJOR%%!'"$version"'!g' \ - -e 's!%%MARIADB_RELEASE_STATUS%%!'"$releaseStatus"'!g' \ - -e 's!%%MARIADB_SUPPORT_TYPE%%!'"$supportType"'!g' \ - -e 's!%%SUITE%%!'"$suite"'!g' \ - -e 's!%%ARCHES%%! '"$arches"'!g' \ - "cs-$version/Dockerfile" + -e 's!%%MARIADB_VERSION%%!'"$mariaVersion"'!g' \ + "$cs_dir/Dockerfile" +} + +createCSImage() { + cs_dir="cs-$version" + if [ "${#versions[@]}" -gt 0 ] && [ "${versions[0]}" == "--cs" ] && [ ! -d "$cs_dir" ]; then + update_cs_version + fi } update_version() @@ -123,6 +125,8 @@ update_version() --arg milestone "$version" --arg version "$mariaVersion" --arg fullVersion "$fullVersion" --arg releaseStatus "$releaseStatus" --arg supportType "$supportType" --arg base "ubuntu:$suite" --arg arches "$arches" \ '.[$milestone] = {"milestone": $milestone, "version": $version, "fullVersion": $fullVersion, "releaseStatus": $releaseStatus, "supportType": $supportType, "base": $base, "arches": $arches|split(" ")}' versions.json)" printf '%s\n' "$versionJson" > versions.json + + createCSImage } update_version_array() @@ -170,19 +174,25 @@ in_development() version=$development_version mariaVersion=${development_version}.0 [ -d "$development_version" ] && update_version + if [ "${#versions[@]}" -gt 0 ] && [ "${versions[0]}" == "--cs" ]; then + createCSImage + fi } +versions=( "$@" ) -if [ $# -eq 0 ]; then +if [ $# -eq 0 ] || ( [ "${versions[0]}" == "--cs" ] && [ "$#" -eq 1 ] ); then all in_development exit 0 fi -versions=( "$@" ) for version in "${versions[@]}"; do - if [ "$version" == $development_version ] || [ "${versions[0]}" == "--cs" ] && [ "$#" -eq 1 ]; then + if [ "${version}" == "--cs" ]; then + continue + fi + if [ "$version" == $development_version ]; then in_development continue fi @@ -200,6 +210,3 @@ for version in "${versions[@]}"; do update_version done -if [ "${versions[0]}" == "--cs" ]; then - update_cs_version -fi From b29f5b72b1ba32ea7c6a22cdd58ecba458384ed9 Mon Sep 17 00:00:00 2001 From: Md Sahil Date: Sat, 30 Sep 2023 01:38:30 +0530 Subject: [PATCH 6/8] Updated Signed-off-by: Md Sahil --- docker-entrypoint-cs.sh | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) mode change 100644 => 100755 docker-entrypoint-cs.sh diff --git a/docker-entrypoint-cs.sh b/docker-entrypoint-cs.sh old mode 100644 new mode 100755 index 4de12a9f..1c58d44c --- a/docker-entrypoint-cs.sh +++ b/docker-entrypoint-cs.sh @@ -18,15 +18,17 @@ mysql_error() { } _main() { - if [ "$1" = 'mariadbd' ]; then - # simply start mariadb with column-store - exec docker-entrypoint.sh "$@" - elif [ "$1" = 'StorageManager' ]; then + if [ "$1" = 'StorageManager' ]; then # Run StorageManage of column-store # storageManager's directroy :- /var/lib/columnstore/storagemanager - print "$@" + echo "$@" elif [ "$1" = 'brm' ]; then # Run brm + echo "$@" + else + # if [ "$1" = 'mariadbd' ] or No argument then by default: + # simply start mariadb with column-store + exec docker-entrypoint.sh "$@" fi } From 7913e662023d9ca94e22536841553bed94aa5f21 Mon Sep 17 00:00:00 2001 From: Md Sahil Date: Tue, 10 Oct 2023 01:21:23 +0530 Subject: [PATCH 7/8] Added get_storage_manager_default_values & validate_args Signed-off-by: Md Sahil --- docker-entrypoint-cs.sh | 83 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/docker-entrypoint-cs.sh b/docker-entrypoint-cs.sh index 1c58d44c..e958e3a0 100755 --- a/docker-entrypoint-cs.sh +++ b/docker-entrypoint-cs.sh @@ -17,10 +17,93 @@ mysql_error() { exit 1 } +check_existence() { + for n in a b c; + do + echo $n + done +} + +validate_args() { + storage_manager_args[0]=service + storage_manager_args[1]=object_size + storage_manager_args[2]=metadata_path + storage_manager_args[3]=max_concurrent_downloads + storage_manager_args[4]=max_concurrent_uploads + storage_manager_args[5]=common_prefix_depth + storage_manager_args[6]=region + storage_manager_args[7]=bucket + storage_manager_args[8]=endpoint + storage_manager_args[9]=aws_access_key_id + storage_manager_args[10]=aws_secret_access_key + storage_manager_args[11]=iam_role_name + storage_manager_args[12]=sts_region + storage_manager_args[13]=sts_endpoint + storage_manager_args[14]=ec2_iam_mode + storage_manager_args[15]=use_http + storage_manager_args[16]=ssl_verify + storage_manager_args[17]=libs3_debug + storage_manager_args[18]=path + storage_manager_args[19]=fake_latency + storage_manager_args[20]=max_latency + storage_manager_args[21]=cache_size + storage_manager_args[22]=cache_path + + # ************Code Here ************ + # Split each arg by `=`. And check whether arg exist in storage_manager_args or not. + # If Not then print an error/warning message to the user but don't fail. + + # If everything is alright then update the default values if arguments ware provided by user relevant to the fields. +} + +get_storage_manager_default_values() { + # For [ObjectStorage] + service='LocalStorage' + object_size='5M' + metadata_path='@ENGINE_DATADIR@/storagemanager/metadata' + journal_path='@ENGINE_DATADIR@/storagemanager/journal' + max_concurrent_downloads=21 + max_concurrent_uploads=21 + common_prefix_depth=3 + + # For [S3] + region='' + bucket='' + endpoint='' + prefix='cs/' + aws_access_key_id='' + aws_secret_access_key='' + iam_role_name= + sts_region= + sts_endpoint= + ec2_iam_mode= + use_http= + ssl_verify= + libs3_debug= + + # The LocalStorage section configures the 'local storage' module + # if specified by ObjectStorage/service. + # [LocalStorage] + + # path specifies where the module should store object data. + path='@ENGINE_DATADIR@/storagemanager/fake-cloud' + fake_latency= + max_latency=50000 + + # [Cache] + cache_size='2g' + cache_path='@ENGINE_DATADIR@/storagemanager/cache' + +} + _main() { if [ "$1" = 'StorageManager' ]; then # Run StorageManage of column-store # storageManager's directroy :- /var/lib/columnstore/storagemanager + shift + validate_args "$@" + get_storage_manager_default_values + echo "$@" elif [ "$1" = 'brm' ]; then # Run brm From 2470204176ba5b78f4c7fd2930fbf4bd2aba1242 Mon Sep 17 00:00:00 2001 From: Md Sahil Date: Thu, 19 Oct 2023 01:30:48 +0530 Subject: [PATCH 8/8] Updated Signed-off-by: Md Sahil --- docker-entrypoint-cs.sh | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/docker-entrypoint-cs.sh b/docker-entrypoint-cs.sh index e958e3a0..94d7a792 100755 --- a/docker-entrypoint-cs.sh +++ b/docker-entrypoint-cs.sh @@ -17,14 +17,12 @@ mysql_error() { exit 1 } -check_existence() { - for n in a b c; - do - echo $n - done -} +# arg_splitter() { + +# } validate_args() { + local storage_manager_args= storage_manager_args[0]=service storage_manager_args[1]=object_size storage_manager_args[2]=metadata_path @@ -49,11 +47,25 @@ validate_args() { storage_manager_args[21]=cache_size storage_manager_args[22]=cache_path - # ************Code Here ************ - # Split each arg by `=`. And check whether arg exist in storage_manager_args or not. - # If Not then print an error/warning message to the user but don't fail. + for arg in $@; do + if [ $arg == '' ]; then + continue + fi + local arr=(${arg//=/ }) + local arg_key=${arr[0]} + local is_exist=false + for storage_manager_arg in "${storage_manager_args[@]}"; do + if [ "$storage_manager_arg" == "$arg_key" ]; then + is_exist=true + fi + done + if [ "$is_exist" != "true" ]; then + echo "argument $arg_key is unknown for storage-manager" >&2 + fi + done - # If everything is alright then update the default values if arguments ware provided by user relevant to the fields. + # *************CODE HERE**************** + # Now, Update the default values if arguments were provided by user relevant to the fields. } get_storage_manager_default_values() {