Skip to content
This repository has been archived by the owner on Jul 16, 2021. It is now read-only.

Change for helm v3 client #643

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions chart/monocular/templates/ingress.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,10 @@ spec:
serviceName: {{ template "fullname" $ }}-ui
servicePort: {{ $.Values.ui.service.externalPort }}
path: /?(.*)
- backend:
serviceName: {{ template "fullname" $ }}-chartsvc
servicePort: {{ $.Values.chartsvc.service.port }}
path: /api/?(.*)
- backend:
serviceName: {{ template "fullname" $ }}-chartsvc
servicePort: {{ $.Values.chartsvc.service.port }}
Expand Down
35 changes: 26 additions & 9 deletions chart/monocular/templates/ui-vhost.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,10 @@ data:
server {{ template "fullname" . }}-prerender;
}

upstream chartsvc {
server {{ template "fullname" . }}-chartsvc:{{ .Values.chartsvc.service.port }};
}

server {
listen {{ .Values.ui.service.internalPort }};

Expand All @@ -21,33 +25,46 @@ data:
gzip_static on;

location / {
try_files $uri @prerender;
try_files $uri @findredirect;
}

location @prerender {
set $prerender 0;
location @findredirect {
set $findredirect 0;

# Intercept some errors, like redirects, and follow them.
proxy_intercept_errors on;

# Look for the Helm user agent. If a Helm client wants the URL we redirect
# to the file being requested.
if ($http_user_agent ~* "helm") {
set $findredirect 2;
}

# Detect bots that want HTML. We send this to a prerender service
if ($http_user_agent ~* "baiduspider|twitterbot|facebookexternalhit|rogerbot|linkedinbot|embedly|quora link preview|showyoubot|outbrain|pinterest|slackbot|vkShare|W3C_Validator") {
set $prerender 1;
set $findredirect 1;
}

if ($args ~ "_escaped_fragment_") {
set $prerender 1;
set $findredirect 1;
}

if ($http_user_agent ~ "Prerender") {
set $prerender 0;
set $findredirect 0;
}

if ($uri ~* "\.(js|css|xml|less|png|jpg|jpeg|gif|pdf|doc|txt|ico|rss|zip|mp3|rar|exe|wmv|doc|avi|ppt|mpg|mpeg|tif|wav|mov|psd|ai|xls|mp4|m4a|swf|dat|dmg|iso|flv|m4v|torrent|ttf|woff|svg|eot)") {
set $prerender 0;
set $findredirect 0;
}

if ($prerender = 1) {
if ($findredirect = 2) {
proxy_pass http://chartsvc/v1/redirect$request_uri;
}
if ($findredirect = 1) {
rewrite .* /https://$host$request_uri? break;
proxy_pass http://target_service;
}
if ($prerender = 0) {
if ($findredirect = 0) {
rewrite .* /index.html break;
}
}
Expand Down
72 changes: 72 additions & 0 deletions cmd/chartsvc/handler.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"math"
"net/http"
"strconv"
"strings"

"github.com/globalsign/mgo/bson"
"github.com/gorilla/mux"
Expand Down Expand Up @@ -459,3 +460,74 @@ func newChartVersionListResponse(c *models.Chart) apiListResponse {

return cvl
}

func redirectToChartVersionPackage(w http.ResponseWriter, req *http.Request) {
// The URL can be in one of two forms:
// - /v1/redirect/charts/stable/aerospike
// - /v1/redirect/charts/stable/aerospike/v1.2.3
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would split the handlers up and extract the shared logic to other functions. We should try to use the router as much as possible to handle path parameters.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So... here is my problem and maybe you can help with it. Let's say I have the path my-repo/my-chart/0.1.0/. Now we also need to get the provenance file for that. This would be my-repo/my-chart/0.1.0/.prov. Notice the placements of the path separators. Or, what if we have the path my-repo/my-chart/0.1.0 and need to get my-repo/my-chart/0.1.0.prov. How do we separate the chart tarball from the provenance one with mux?

There is some variation to the possible inputs which makes handling some of this in the router difficult. Any pointers on that?

// And either of these can optionally have a trailing /

// Make sure the path is valid
ct := strings.TrimPrefix(req.URL.Path, "/v1/redirect/charts/")

// check if URL for provenance
prov := strings.HasSuffix(ct, ".prov")
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we may also want to consider handling the provenance file in a separate handler

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure how to handle that with mux (see setupRoutes). PathPrefix is used because the redirect URL is going to have multiple path parts to it. In the router there is not method for filtering based on suffix, that I'm aware of. Do you have a suggestion?

ct = strings.TrimSuffix(ct, ".prov")

ct = strings.TrimSuffix(ct, "/") // Removing the optional / on the end
parts := strings.Split(ct, "/")

// Not enough parts passed in to the path
if len(parts) < 2 || len(parts) > 3 {
response.NewErrorResponse(http.StatusNotFound, "could not find chart").Write(w)
return
}

// Decide if latest or a version
var version string
if len(parts) == 3 {
version = parts[2]
}

// Look it up. This will be different if there is a version or we are getting
// the latest
db, closer := dbSession.DB()
defer closer()
var chart models.Chart
chartID := fmt.Sprintf("%s/%s", parts[0], parts[1])

if version == "" {
if err := db.C(chartCollection).FindId(chartID).One(&chart); err != nil {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since we're starting to repeat these MongoDB calls, we may want to extract them out to separate functions and call them from here and in getChart (and the one below in getChartVersion)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we do this as part of a separate change? This query is used 4 times by 4 different callbacks. If the logic were to be extracted I would suggest doing it uniformly and updating all of them. It's worth being tested separately from this feature addition IMHO. Repeated the same database query is common here as it stands.

log.WithError(err).Errorf("could not find chart with id %s", chartID)
response.NewErrorResponse(http.StatusNotFound, "could not find chart").Write(w)
return
}
} else {
if err := db.C(chartCollection).Find(bson.M{
"_id": chartID,
"chartversions": bson.M{"$elemMatch": bson.M{"version": version}},
}).Select(bson.M{
"name": 1, "repo": 1, "description": 1, "home": 1, "keywords": 1, "maintainers": 1, "sources": 1,
"chartversions.$": 1,
}).One(&chart); err != nil {
log.WithError(err).Errorf("could not find chart with id %s", chartID)
response.NewErrorResponse(http.StatusNotFound, "could not find chart version").Write(w)
return
}
}

// Respond with proper redirect for tarball and prov
if len(chart.ChartVersions) > 0 {
cv := chart.ChartVersions[0]
if len(cv.URLs) > 0 {
if prov {
http.Redirect(w, req, cv.URLs[0]+".prov", http.StatusTemporaryRedirect)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just a thought, instead of redirecting, it might make sense to stream the file directly back to the client. I know the intention of this is for public repositories, but this could be useful for instances of Monocular pointing to private repositories that might not be accessible by the client.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I thought about that but if we do that for the Helm Hub it means we are eating a lot more egress cost as we stream all the packages out from the hub. It also means more system resources are required. I was attempting to be cost sensitive with this implementation.

} else {
http.Redirect(w, req, cv.URLs[0], http.StatusTemporaryRedirect)
}
return
}
}

response.NewErrorResponse(http.StatusNotFound, "could not find chart version").Write(w)
}
86 changes: 86 additions & 0 deletions cmd/chartsvc/handler_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -907,3 +907,89 @@ func Test_findLatestChart(t *testing.T) {
assert.Equal(t, len(data), 2, "it should return both charts")
})
}

func Test_redirectToChartVersionPackage(t *testing.T) {
tests := []struct {
name string
err error
chart models.Chart
wantCode int
location string
}{
{
"chart does not exist",
errors.New("return an error when checking if chart exists"),
models.Chart{ID: "my-repo/my-chart"},
http.StatusNotFound,
"",
},
{
"chart exists",
nil,
models.Chart{ID: "my-repo/my-chart", ChartVersions: []models.ChartVersion{{Version: "0.1.0", URLs: []string{"https://example.com/my-chart-0.1.0.tgz"}}, {Version: "0.0.1", URLs: []string{"https://example.com/my-chart-0.0.1.tgz"}}}},
http.StatusTemporaryRedirect,
"https://example.com/my-chart-0.1.0.tgz",
},
{
"chart exists with trailing /",
nil,
models.Chart{ID: "my-repo/my-chart/", ChartVersions: []models.ChartVersion{{Version: "0.1.0", URLs: []string{"https://example.com/my-chart-0.1.0.tgz"}}, {Version: "0.0.1", URLs: []string{"https://example.com/my-chart-0.0.1.tgz"}}}},
http.StatusTemporaryRedirect,
"https://example.com/my-chart-0.1.0.tgz",
},
{
"chart with version",
nil,
models.Chart{ID: "my-repo/my-chart/0.1.0", ChartVersions: []models.ChartVersion{{Version: "0.1.0", URLs: []string{"https://example.com/my-chart-0.1.0.tgz"}}}},
http.StatusTemporaryRedirect,
"https://example.com/my-chart-0.1.0.tgz",
},
{
"chart with version with trailing /",
nil,
models.Chart{ID: "my-repo/my-chart/0.1.0/", ChartVersions: []models.ChartVersion{{Version: "0.1.0", URLs: []string{"https://example.com/my-chart-0.1.0.tgz"}}}},
http.StatusTemporaryRedirect,
"https://example.com/my-chart-0.1.0.tgz",
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
var m mock.Mock
dbSession = mockstore.NewMockSession(&m)

if tt.err != nil {
m.On("One", mock.Anything).Return(tt.err)
} else {
m.On("One", &models.Chart{}).Return(nil).Run(func(args mock.Arguments) {
*args.Get(0).(*models.Chart) = tt.chart
})
}

w := httptest.NewRecorder()
req := httptest.NewRequest("GET", "/v1/redirect/charts/"+tt.chart.ID, nil)

redirectToChartVersionPackage(w, req)

m.AssertExpectations(t)
assert.Equal(t, tt.wantCode, w.Code)
if tt.wantCode == http.StatusTemporaryRedirect {
resp := w.Result()
assert.Equal(t, tt.location, resp.Header.Get("Location"), "response header location should be chart url")
}

// Check for provenance file
w = httptest.NewRecorder()
req = httptest.NewRequest("GET", "/v1/redirect/charts/"+tt.chart.ID+".prov", nil)

redirectToChartVersionPackage(w, req)

m.AssertExpectations(t)
assert.Equal(t, tt.wantCode, w.Code)
if tt.wantCode == http.StatusTemporaryRedirect {
resp := w.Result()
assert.Equal(t, tt.location+".prov", resp.Header.Get("Location"), "response header location should be chart provenance url")
}
})
}
}
5 changes: 5 additions & 0 deletions cmd/chartsvc/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,11 @@ func setupRoutes() http.Handler {
apiv1.Methods("GET").Path("/assets/{repo}/{chartName}/versions/{version}/values.yaml").Handler(WithParams(getChartVersionValues))
apiv1.Methods("GET").Path("/assets/{repo}/{chartName}/versions/{version}/values.schema.json").Handler(WithParams(getChartVersionSchema))

// Handle redirects to the root chart. That way you can
// `helm install monocular.example.com/charts/foo/bar` and have monocular
// redirect to the right place.
apiv1.Methods("GET").PathPrefix("/redirect").HandlerFunc(redirectToChartVersionPackage)

n := negroni.Classic()
n.UseHandler(r)
return n
Expand Down