Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Gh 621 bulk import #815

Closed
wants to merge 20 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
9971e19
trying out spaces
chb0github Sep 26, 2023
aed7081
Update devfile.yaml
chb0github Sep 26, 2023
a5ecbd6
Merge pull request #804 from jfrog/remove-repos-project-key-default
chb0github Sep 15, 2023
14e6def
GH-621 - working for groups, users and repos.
chb0github Oct 1, 2023
9908880
Merge branch 'master' into GH-621-bulk-import
chb0github Oct 2, 2023
6482ab8
GH-621 - all done. No support for docker v1 at all.
chb0github Oct 2, 2023
bd61ddb
remove cruft
chb0github Oct 2, 2023
126cf7d
GH-621 - Jobs done. No support no docker v2. Docker file for containe…
chb0github Oct 2, 2023
9628090
Merge branch 'master' into GH-621-bulk-import
chb0github Oct 2, 2023
587b1f7
Should now work with V1 docker repos, but I have no way to test. Disc…
chb0github Oct 4, 2023
cbf1835
Merge branch 'GH-621-bulk-import' of github.com:chb0github/terraform-…
chb0github Oct 4, 2023
3818082
add a format function for later use
chb0github Oct 4, 2023
4083ecd
Everything works again, and it now includes support for v1 and v2 docker
chb0github Oct 5, 2023
7fb6218
Merge branch 'master' into GH-621-bulk-import
chb0github Oct 6, 2023
5afd562
Merge branch 'master' into GH-621-bulk-import
chb0github Oct 9, 2023
5182422
GH-621 - remove some debug code
chb0github Oct 9, 2023
1574ee7
Merge branch 'master' into GH-621-bulk-import
chb0github Oct 12, 2023
d2bf147
Merge branch 'master' into GH-621-bulk-import
chb0github Oct 13, 2023
d466bde
Merge branch 'master' into GH-621-bulk-import
chb0github Oct 18, 2023
42a8043
Merge branch 'master' into GH-621-bulk-import
chb0github Dec 5, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 36 additions & 22 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,23 +1,37 @@
# Fetch the dependencies
FROM golang:1.15-alpine AS builder

RUN apk add --update ca-certificates git gcc g++ libc-dev
WORKDIR /src/

ENV GO111MODULE=on

COPY go.mod .
COPY go.sum .

FROM alpine AS base
RUN apk add --no-cache git terraform wget make openssh && \
wget -q https://github.com/goreleaser/goreleaser/releases/download/v1.19.2/goreleaser_1.19.2_x86_64.apk && \
wget -q https://go.dev/dl/go1.19.2.linux-amd64.tar.gz && \
rm -rf /usr/local/go && \
tar -C /usr/local -xzf go1.19.2.linux-amd64.tar.gz && \
apk add --allow-untrusted goreleaser_1.19.2_x86_64.apk && \
mkdir -p /src/terraform-provider-artifactory
ENV PATH=$PATH:/usr/local/go/bin
WORKDIR /root

FROM base as builder
COPY . .
RUN go mod download

COPY pkg/ /src/pkg/
COPY main.go /src/

RUN CGO_ENABLED=0 GOOS=linux go build


# Build the final image
FROM hashicorp/terraform:0.13

COPY --from=builder /src/terraform-provider-artifactory /root/.terraform.d/plugins/
RUN make
WORKDIR /root/v5-v6-migrator
RUN make

FROM hashicorp/terraform as plugin
RUN adduser -S jfrog
WORKDIR /home/jfrog
COPY --from=builder /src/terraform-provider-artifactory/terraform-provider-artifactory /home/jfrog/.terraform.d/plugins/

FROM alpine as migrator
RUN adduser -S jfrog
WORKDIR /home/jfrog
COPY --from=builder /src/terraform-provider-artifactory/v5-v6-migrator/tf-v5-migrator /home/jfrog/tf-v5-migrator

alexhung marked this conversation as resolved.
Show resolved Hide resolved
FROM alpine as importer
RUN apk add --no-cache jq curl bash terraform
alexhung marked this conversation as resolved.
Show resolved Hide resolved
RUN adduser -S jfrog
WORKDIR /home/jfrog
COPY scripts/bulkimport.sh /home/jfrog
RUN chown -R jfrog /home/jfrog
USER jfrog
ENTRYPOINT bash
CMD /home/jfrog/bulkimport.sh
316 changes: 316 additions & 0 deletions scripts/bulkimport.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,316 @@
#!/usr/bin/env bash
${DEBUG:+set -xv}
set -e
set -o errexit -o pipefail -o noclobber
shopt -s expand_aliases

# shellcheck disable=SC2139
alias curl="curl -snL${DEBUG:+v}f"

HOME=${HOME:-"~"}

command -v curl >/dev/null || (echo "You must install curl to proceed" >&2 && exit 1)
command -v jq >/dev/null || (echo "You must install jq to proceed" >&2 && exit 1)
command -v terraform >/dev/null || (echo "You must install terraform to proceed" >&2 && exit 1)

function usage {
echo "${0} [{--users |-u} {--repos|-r} | {--groups|-g} | {--all | -a}] -h|--host https://\${host}
duplicate resource declarations are de duped (see below)

example:
${0} -u --repos --users -h https://myartifactory.com > import.tf
terraform plan -no-color -generate-config-out generated.tf -out ${RANDOM}-out -parallelism=10
terraform apply -no-color -parallelism=10

You may enable debug with: DEBUG=1 ${0} ..." >&2
exit 1
}

resources=()
while getopts urdgah:-: OPT; do
# shellcheck disable=SC2154
if [ "$OPT" = "-" ]; then # long option: reformulate OPT and OPTARG
OPT="${OPTARG%%=*}" # extract long option name
OPTARG="${OPTARG#$OPT}" # extract long option argument (may be empty)
OPTARG="${OPTARG#=}" # if long option argument, remove assigning `=`
fi
# shellcheck disable=SC2214
case "${OPT}" in
u | users)
resources+=(users)
;;
r | repos)
resources+=(repos)
;;
g | groups)
resources+=(groups)
;;
h | host)
host="${OPTARG}"
;;
a | all)
resources=(users repos groups)
;;
??*)
usage
;;
*)
usage
;;
esac
done

function netrc_location {
chb0github marked this conversation as resolved.
Show resolved Hide resolved
case "$(uname)" in
CYGWIN* | MSYS* | MINGW*)
echo "$HOME/_netrc"
;;
Darwin* | Linux*)
echo "$HOME/.netrc"
;;
*)
echo "unsupported OS" >&2
return 255
;;
esac
}

function assert_netrc {
local location
location="$(netrc_location)"
test -f "${location}" || touch "${location}"
echo "${location}"
}

function toHost {
local host="${1:?You must supply a host}"
host="${host/https:\/\//}"
echo "${host/http:\/\//}"
}

function hasNetRcEntry {
local h="${1:?No host supplied}"
grep -qE "machine[ ]+$(toHost "${h}")" "$(assert_netrc)"
}

function write_netrc {
local host="${1:?You must supply a host name}"
local netrc
netrc=$(assert_netrc)
host=$(toHost "${host}")
read -r -p "Please enter the username for ${host}: " username
read -rs -p "Enter the api token (will not be echoed): " token
# append only to both files
cat <<-EOF >> "${netrc}"
machine ${host}
login ${username}
password ${token}
EOF
echo "${host}"
}

if ! grep -qE 'http(s)?://.*' <<<"${host:-}"; then
echo "malformed url name: '${host}'. must be of the form http(s)?://.*"
exit 1
fi

# if they have no netrc file at all, create the file and add an entry
if ! hasNetRcEntry "${host}" ; then
cat <<-EOF >&2

added entry
to $(netrc_location)
for $(write_netrc "${host}" )"
EOF
fi

function get_tf_state {
jq -re '.resources |map({type,name})' terraform.tfstate
}

function repos {
# jq '.resources |map({type,name})' terraform.tfstate # make sure to not include anything already in state
# we'll make our internal jq structure match that of the tf state file so we can subtract them easy
local host="${1:?You must supply the artifactory host}"
# literally, usage of jq is 6x faster than bash/read/etc
# GET "${host}/artifactory/api/repositories" returns {key,type,packageType,url} where
# url) points to the UI for that resource??
# packageType) is cased ??
# type) is upcased and in "${host}/artifactory/api/repositories/${key}" it's not AND it's called rclass
local tempJson
tempJson="$(mktemp)-$RANDOM"
local url="${host}/artifactory/api/repositories"
# we have to sort out the wheat from the chaffee. We're normalizing the input while we sort it out
# and we choose to map to 'rclass' from '.type' because that's how the Go code maps it
curl -snLf "${url}" |
jq 'map({
key,
rclass: (.type | ascii_downcase),
packageType : (.packageType | ascii_downcase)
}) |
group_by(.packageType == "docker" and .rclass == "local") |
{
safe: .[0],
docker_remap: .[1]
}
' > "${tempJson}"

# the URL that comes in the original payload refers to the UI endpoint. Dumb
jq -re --arg u "${url}" '.docker_remap[] | "\($u)/\(.key)"' "${tempJson}" |
#grab the docker-local repos. Curl when used this xargs doesn't seem to be picking up the alias
xargs -n 10 -P 10 curl -snLf |
# this was literally the only field we couldn't get from before and, apparently it's no longer possible to
# even set docker V1 in RT (even though there is a check, you get an error if you try). But for legacy reason, we
# have to go fetch them.
jq -sre 'map(.dockerApiVersion |= ascii_downcase)' |
# combined step 1 with the tf state and step 3, and give them saner names. But what if they have no tf file??
cat "${tempJson}" - | jq -sre '
{
safe: .[0].safe,
docker:.[1:][0]
} |
((.safe | map({
type: "artifactory_\(.rclass)_\(.packageType)_repository.\(.key | ascii_downcase)",
name: .key
})
) +
(.docker | map({
type: "artifactory_\(.rclass)_\(.packageType)_\(.dockerApiVersion)_repository.\(.key)",
name: .key
})
)) | .[] |
"import {
to = \(.type)
id = \"\(.name)\"
}"'
}

function accessTokens {
local host="${1:?You must supply the artifactory host}"
return 1
curl "${host}/artifactory/api/repositories/artifactory/api/security/token"
}

function ldapGroups {
local host="${1:?You must supply the artifactory host}"
return 1
curl "${host}/access/api/v1/ldap/groups"
}

function apiKeys {
local host="${1:?You must supply the artifactory host}"
return 1
curl "${host}/artifactory/api/security/apiKey"

}

function groups {
local host="${1:?You must supply the artifactory host}"
curl "${host}/artifactory/api/security/groups" |
jq -re '.[].name |
"import {
to = artifactory_group.\(. | ascii_downcase)
id = \"\(.)\"
}"'
}

function certificates {
local host="${1:?You must supply the artifactory host}"
curl "${host}/artifactory/api/system/security/certificates/" |
jq -re '.[] |
"import {
to = artifactory_certificate.\(.certificateAlias)
id = \"(.certificateAlias)\"
}"'
}

function distributionPublicKeys {
local host="${1:?You must supply the artifactory host}"
# untested
return 1
curl "${host}/artifactory/api/security/keys/trusted" |
jq -re '.keys[] | "
import {
to = artifactory_distribution_public_key.\(.alias)
id = \"\(.kid)\"
}"'
}

function permissions {
# these names have spaces in them
local host="${1:?You must supply the artifactory host}"
return 1 # untested
curl "${host}/artifactory/api/v2/security/permissions/" |
jq -re '.[] | select(.name | startswith("INTERNAL") | not) | "
import {
to = artifactory_permission_target.\(.name)
id = \"\(.name)\"
}"'
}
function keyPairs {
local host="${1:?You must supply the artifactory host}"
return 1 # untested
curl "${host}/artifactory/api/security/keypair/" |
jq -re '.[] | "
import {
to = artifactory_keypair.\(.pairName)
id = \"\(.pairName)\"
}"'
}
function users {
# .name has values in it that artifactory will never accept, like email@. Not sure if in that case it should just be user-$RANDOM
local host="${1:?You must supply the artifactory host}"
curl "${host}/artifactory/api/security/users" | jq -re '.[] |
{
user: .name | capture("(?<user>\\w+)@(?<domain>\\w+)").user,
name
}|
"import {
to = artifactory_user.\(.user)
id = \"\(.name)\"
}"'
}
function format {
local resourceType="${1:?You must supply a resource type}"
local key="${2:?You must supply a key}"
local alias="${3:-${key}}"
cat <<-EOF
import {
to = $resourceType.$alias
id = "$key"
}
EOF
}


function output {
local host="${1:?You must supply artifactory host name}"
# don't touch this heredoc if you want proper output format
cat <<-EOF
terraform {
required_providers {
artifactory = {
source = "registry.terraform.io/jfrog/artifactory"
version = ">= 9.1.0"
}
}
}
provider "artifactory" {
url = "${host}"
}

$(for f in "${@:2}"; do
eval "${f} ${host}"
done)
EOF
}

# shellcheck disable=SC2046
output "${host}" $(echo "${resources[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' ')
out="$RANDOM-out"
# everytime I try to automate this, I get 'This character is not used within the language.' - it's generating something funky
echo "please run :

terraform plan -no-color -generate-config-out generated.tf -out ${out} -parallelism=10
terraform apply -no-color -parallelism=10
" >&2
Loading