commit

Author: AndreiP-og

.gitignore

@@ -0,0 +1 @@
+.sequence

.idea/.gitignore

@@ -0,0 +1,7 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": []
+}

.idea/gitlab-exporter.iml

@@ -0,0 +1,20 @@
+FROM golang:1.16 as build
+
+ENV GO111MODULE=on
+
+COPY ./ /go/src/github.com/aserdean/gitlab-exporter/
+WORKDIR /go/src/github.com/aserdean/gitlab-exporter/
+
+RUN go mod download \
+    && CGO_ENABLED=0 GOOS=linux go build -o /bin/main
+
+FROM alpine:3.11.3
+
+RUN apk --no-cache add ca-certificates \
+     && addgroup exporter \
+     && adduser -S -G exporter exporter
+ADD VERSION .
+USER exporter
+COPY --from=build /bin/main /bin/main
+ENV LISTEN_PORT=8080
+ENTRYPOINT [ "/bin/main" ]

.idea/modules.xml

@@ -0,0 +1,22 @@
+# Prometheus Gitlab Exporter
+
+Exposes metrics for your repositories from the Gitlab API, to a Prometheus compatible endpoint.
+
+## Configuration
+
+This exporter is setup to take input from environment variables. All variables are optional:
+
+* `GROUPS` If supplied, the exporter will enumerate all repositories for that group. Expected in the format "group1, group2".
+* `REPOS` If supplied, The repos you wish to monitor, expected in the format "group/repo1, group/repo2". Can be across different Gitlab users/orgs.
+* `GITLAB_TOKEN` If supplied, enables the user to supply a gitlab authentication token that allows the API to be queried. If none supplied, the exporter will only have access to public repos. The token have `read_api` access.
+* `GITHUB_TOKEN_FILE` If supplied _instead of_ `GITHUB_TOKEN`, enables the user to supply a path to a file containing a gitlab authentication token.
+* `API_URL` Gitlab API URL, shouldn't need to change this. Defaults to `https://gitlab.com`
+* `LOG_LEVEL` The level of logging the exporter will run with, defaults to `debug`
+
+## Install and deploy
+
+Build a docker image:
+```
+docker build -t <image-name> .
+docker tag <built_image_hash> gitlab-exporter_gitlab-exporter:latest
+```

.idea/vcs.xml

@@ -0,0 +1 @@
+1.0.0

.vscode/launch.json

@@ -0,0 +1,121 @@
+package config
+
+import (
+	"fmt"
+	goGitlab "github.com/xanzy/go-gitlab"
+	"io/ioutil"
+	"strings"
+
+	log "github.com/sirupsen/logrus"
+
+	"os"
+)
+
+// GetEnv - Allows us to supply a fallback option if nothing specified
+func GetEnv(key, fallback string) string {
+	value := os.Getenv(key)
+	if len(value) == 0 {
+		return fallback
+	}
+	return value
+}
+
+// Config struct holds all of the runtime confgiguration for the application
+type Config struct {
+	APIURL        string
+	Repositories  []string
+	Organisations []string
+	Users         string
+	APITokenEnv   string
+	APITokenFile  string
+	APIToken      string
+}
+
+// Init populates the Config struct based on environmental runtime configuration
+func Init() Config {
+
+	listenPort := GetEnv("LISTEN_PORT", "9171")
+	os.Setenv("LISTEN_PORT", listenPort)
+	url := "https://gitlab.com"
+	repos := os.Getenv("REPOS")
+	groups := os.Getenv("GROUPS")
+	users := os.Getenv("USERS")
+	tokenEnv := os.Getenv("GITLAB_TOKEN")
+	tokenFile := os.Getenv("GITLAB_TOKEN_FILE")
+	token, err := getAuth(tokenEnv, tokenFile)
+
+	if err != nil {
+		log.Errorf("Error initialising Configuration, Error: %v", err)
+	}
+
+	var reposList []string
+	if repos != "" {
+		rs := strings.Split(repos, ", ")
+		for _, x := range rs {
+			reposList = append(reposList, x)
+		}
+	}
+
+	var groupList []string
+	if groups != "" {
+		gs := strings.Split(groups, ", ")
+		for _, x := range gs {
+			groupList = append(reposList, x)
+			groupRepos := getReposByGroup(x, token)
+			reposList = append(reposList, groupRepos...)
+		}
+	}
+
+
+	appConfig := Config{
+		url,
+		reposList,
+		groupList,
+		users,
+		tokenEnv,
+		tokenFile,
+		token,
+	}
+
+	return appConfig
+}
+
+// getAuth returns oauth2 token as string for usage in http.request
+func getAuth(token string, tokenFile string) (string, error) {
+
+	if token != "" {
+		return token, nil
+	} else if tokenFile != "" {
+		b, err := ioutil.ReadFile(tokenFile)
+		if err != nil {
+			return "", err
+		}
+		return strings.TrimSpace(string(b)), err
+
+	}
+
+	return "", nil
+}
+
+// getReposByGroup returns a list of repositories
+// that belong to a group
+func getReposByGroup(group string, token string) []string {
+	fmt.Println(token)
+	git, err := goGitlab.NewClient(token)
+	if err != nil {
+		log.Fatalf("Failed to create client: %v", err)
+		return nil
+	}
+	groupInfo, _, err := git.Groups.GetGroup(group)
+	if err != nil {
+		log.Errorf("Error obtaining repositories from group, Error: %v", err)
+		return nil
+	}
+	var repos []string
+	for _, p := range groupInfo.Projects {
+		fullName := group + "/" + p.Path
+		repos = append(repos, fullName)
+	}
+
+	return repos
+}

Dockerfile

@@ -0,0 +1,41 @@
+version: "3"
+
+services:
+  gitlab-exporter:
+    expose:
+      - 8080
+    ports:
+      - 8080:8080
+    image: gitlab-exporter_gitlab-exporter:latest
+    environment:
+      # It can use both repositories and organization names separated by comma ", "
+      # eg:
+      # REPOS=RepoName1, RepoName2
+      # GROUPS= ORG1, ORG2
+      - GROUPS=aserdean1
+      - GITLAB_TOKEN=vi4j27pD9fZKCFJTN5Cn
+
+  prometheus:
+    image: docker.io/prom/prometheus:v2.22.2
+    ports:
+      - 9090:9090
+    links:
+      - gitlab-exporter
+    volumes:
+      - ./prometheus/config.yml:/etc/prometheus/prometheus.yml
+
+  grafana:
+    image: docker.io/grafana/grafana:7.3.3
+    ports:
+      - 3000:3000
+    environment:
+      GF_AUTH_ANONYMOUS_ENABLED: 'true'
+      GF_INSTALL_PLUGINS: grafana-polystat-panel,yesoreyeram-boomtable-panel
+    links:
+      - prometheus
+    volumes:
+      - ./grafana/datasources.yml:/etc/grafana/provisioning/datasources/default.yml
+
+networks:
+  default:
+    driver: bridge

README.md

@@ -0,0 +1,129 @@
+package exporter
+
+import (
+	log "github.com/sirupsen/logrus"
+	goGitlab "github.com/xanzy/go-gitlab"
+)
+
+
+
+// XXX gatherData - Collects the data from the API and stores into struct
+func (e *Exporter) gatherData() ([]*Datum, error) {
+
+	data := []*Datum{}
+
+	// this needs to use something returned from getGroups
+	for _, repo := range e.Config.Repositories {
+		git, err := goGitlab.NewClient(e.Config.APIToken)
+		if err != nil {
+			log.Fatalf("Failed to create client: %v", err)
+		}
+
+		d := new(Datum)
+
+		d.RepoName = repo
+
+		d.Branches = getBranches(repo, git)
+
+		d.Releases = getReleases(repo, git)
+
+		d.MergeRequests = getMergeRequests(repo, git)
+
+		for _, b := range d.Branches {
+			c := new(CommitsPerBranch)
+			c.Branch = b.Name
+			c.BranchCommits = getCommits(repo, b.Name, git)
+			d.Commits = append(d.Commits, c)
+		}
+
+		data = append(data, d)
+
+		log.Infof("API data fetched for repository: %s", repo)
+	}
+
+	//return data, rates, err
+	return data, nil
+
+}
+
+type MergeRequests []*goGitlab.MergeRequest
+
+func getMergeRequests(repo string, client *goGitlab.Client) MergeRequests {
+	var mrs MergeRequests
+	opts := &goGitlab.ListProjectMergeRequestsOptions{
+		ListOptions: goGitlab.ListOptions{
+			Page: 0,
+		},
+	}
+	for  {
+		pageMRs, resp, err := client.MergeRequests.ListProjectMergeRequests(repo, opts)
+		if err != nil {
+			log.Errorf("Unable to obtain merge requests from API, Error: %s", err)
+		}
+
+		mrs = append(mrs, pageMRs...)
+
+		// Exit the loop when we've seen all pages.
+		if resp.NextPage == 0 {
+			break
+		}
+
+		// Update the page number to get the next page.
+		opts.Page = resp.NextPage
+	}
+
+	return mrs
+}
+
+type Commits []*goGitlab.Commit
+
+func getCommits(repo string, branch string, client *goGitlab.Client) Commits {
+	var commits Commits
+	opts := &goGitlab.ListCommitsOptions{
+		RefName: &branch,
+		ListOptions: goGitlab.ListOptions{
+			Page: 0,
+		},
+	}
+	for  {
+		pageCs, resp, err := client.Commits.ListCommits(repo, opts)
+		if err != nil {
+			log.Errorf("Unable to obtain commits from API, Error: %s", err)
+		}
+
+		commits = append(commits, pageCs...)
+
+		// Exit the loop when we've seen all pages.
+		if resp.NextPage == 0 {
+			break
+		}
+
+		// Update the page number to get the next page.
+		opts.Page = resp.NextPage
+	}
+
+	return commits
+}
+
+type Releases []*goGitlab.Release
+
+func getReleases(repo string, client *goGitlab.Client) Releases {
+	var releases Releases
+	releases, _, err := client.Releases.ListReleases(repo, &goGitlab.ListReleasesOptions{PerPage: 1000})
+	if err != nil {
+		log.Errorf("Unable to obtain releases from API, Error: %s", err)
+	}
+	return releases
+}
+
+type Branches []*goGitlab.Branch
+
+func getBranches(repo string, client *goGitlab.Client) Branches {
+	var branches Branches
+	pageOpts := goGitlab.ListOptions{PerPage: 1000}
+	branches, _, err := client.Branches.ListBranches(repo, &goGitlab.ListBranchesOptions{ListOptions: pageOpts})
+	if err != nil {
+		log.Errorf("Unable to obtain branches from API, Error: %s", err)
+	}
+	return branches
+}