From d1ef85b5dc6c7b4374271f4290dce06bdc0c05fc Mon Sep 17 00:00:00 2001 From: singhalkarun Date: Mon, 14 Aug 2023 17:42:58 +0530 Subject: [PATCH 1/3] 1. Added a new bool value to the FileServer Struct called PreferPrecompressed 2. Start accepting prefer_precompressed subdirective under the fileserver directive --- modules/caddyhttp/fileserver/caddyfile.go | 6 ++++++ modules/caddyhttp/fileserver/staticfiles.go | 2 ++ 2 files changed, 8 insertions(+) diff --git a/modules/caddyhttp/fileserver/caddyfile.go b/modules/caddyhttp/fileserver/caddyfile.go index df56092b031..9131460d8da 100644 --- a/modules/caddyhttp/fileserver/caddyfile.go +++ b/modules/caddyhttp/fileserver/caddyfile.go @@ -147,6 +147,12 @@ func parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error) } fsrv.PassThru = true + case "prefer_precompressed": + if h.NextArg() { + return nil, h.ArgErr() + } + fsrv.PreferPrecompressed = true + default: return nil, h.Errf("unknown subdirective '%s'", h.Val()) } diff --git a/modules/caddyhttp/fileserver/staticfiles.go b/modules/caddyhttp/fileserver/staticfiles.go index 133578cf69f..d029e71963b 100644 --- a/modules/caddyhttp/fileserver/staticfiles.go +++ b/modules/caddyhttp/fileserver/staticfiles.go @@ -132,6 +132,8 @@ type FileServer struct { // clobbering the explicit rewrite with implicit behavior. CanonicalURIs *bool `json:"canonical_uris,omitempty"` + PreferPrecompressed bool `json:"prefer_precompressed,omitempty"` + // Override the status code written when successfully serving a file. // Particularly useful when explicitly serving a file as display for // an error, like a 404 page. A placeholder may be used. By default, From 029a791eb670128ea1228e8d5bcc65155f13cd66 Mon Sep 17 00:00:00 2001 From: singhalkarun Date: Mon, 14 Aug 2023 19:57:24 +0530 Subject: [PATCH 2/3] 1. If PreCompressed is enabed on fileserver, ignore checking the existence of the original file and serve encoded file as per the accepted encoding if available, else return 404. To Do Next: If the file is not available as per accepted encoding, try to decompress the encoded file to original file and serve, if not able to, return 404. --- modules/caddyhttp/fileserver/staticfiles.go | 182 +++++++++++--------- 1 file changed, 96 insertions(+), 86 deletions(-) diff --git a/modules/caddyhttp/fileserver/staticfiles.go b/modules/caddyhttp/fileserver/staticfiles.go index d029e71963b..23400583e81 100644 --- a/modules/caddyhttp/fileserver/staticfiles.go +++ b/modules/caddyhttp/fileserver/staticfiles.go @@ -257,104 +257,109 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c zap.String("request_path", r.URL.Path), zap.String("result", filename)) - // get information about the file - info, err := fs.Stat(fsrv.fileSystem, filename) - if err != nil { - err = fsrv.mapDirOpenError(err, filename) - if errors.Is(err, fs.ErrNotExist) || errors.Is(err, fs.ErrInvalid) { - return fsrv.notFound(w, r, next) - } else if errors.Is(err, fs.ErrPermission) { - return caddyhttp.Error(http.StatusForbidden, err) + if !fsrv.PreferPrecompressed { + // get information about the file + info, err := fs.Stat(fsrv.fileSystem, filename) + if err != nil { + err = fsrv.mapDirOpenError(err, filename) + if errors.Is(err, fs.ErrNotExist) || errors.Is(err, fs.ErrInvalid) { + return fsrv.notFound(w, r, next) + } else if errors.Is(err, fs.ErrPermission) { + return caddyhttp.Error(http.StatusForbidden, err) + } + return caddyhttp.Error(http.StatusInternalServerError, err) } - return caddyhttp.Error(http.StatusInternalServerError, err) - } - // if the request mapped to a directory, see if - // there is an index file we can serve - var implicitIndexFile bool - if info.IsDir() && len(fsrv.IndexNames) > 0 { - for _, indexPage := range fsrv.IndexNames { - indexPage := repl.ReplaceAll(indexPage, "") - indexPath := caddyhttp.SanitizedPathJoin(filename, indexPage) - if fileHidden(indexPath, filesToHide) { - // pretend this file doesn't exist - fsrv.logger.Debug("hiding index file", - zap.String("filename", indexPath), - zap.Strings("files_to_hide", filesToHide)) - continue - } + // if the request mapped to a directory, see if + // there is an index file we can serve + var implicitIndexFile bool + if info.IsDir() && len(fsrv.IndexNames) > 0 { + for _, indexPage := range fsrv.IndexNames { + indexPage := repl.ReplaceAll(indexPage, "") + indexPath := caddyhttp.SanitizedPathJoin(filename, indexPage) + if fileHidden(indexPath, filesToHide) { + // pretend this file doesn't exist + fsrv.logger.Debug("hiding index file", + zap.String("filename", indexPath), + zap.Strings("files_to_hide", filesToHide)) + continue + } - indexInfo, err := fs.Stat(fsrv.fileSystem, indexPath) - if err != nil { - continue - } + indexInfo, err := fs.Stat(fsrv.fileSystem, indexPath) + if err != nil { + continue + } - // don't rewrite the request path to append - // the index file, because we might need to - // do a canonical-URL redirect below based - // on the URL as-is - - // we've chosen to use this index file, - // so replace the last file info and path - // with that of the index file - info = indexInfo - filename = indexPath - implicitIndexFile = true - fsrv.logger.Debug("located index file", zap.String("filename", filename)) - break + // don't rewrite the request path to append + // the index file, because we might need to + // do a canonical-URL redirect below based + // on the URL as-is + + // we've chosen to use this index file, + // so replace the last file info and path + // with that of the index file + info = indexInfo + filename = indexPath + implicitIndexFile = true + fsrv.logger.Debug("located index file", zap.String("filename", filename)) + break + } } - } - // if still referencing a directory, delegate - // to browse or return an error - if info.IsDir() { - fsrv.logger.Debug("no index file in directory", - zap.String("path", filename), - zap.Strings("index_filenames", fsrv.IndexNames)) - if fsrv.Browse != nil && !fileHidden(filename, filesToHide) { - return fsrv.serveBrowse(root, filename, w, r, next) + // if still referencing a directory, delegate + // to browse or return an error + if info.IsDir() { + fsrv.logger.Debug("no index file in directory", + zap.String("path", filename), + zap.Strings("index_filenames", fsrv.IndexNames)) + if fsrv.Browse != nil && !fileHidden(filename, filesToHide) { + return fsrv.serveBrowse(root, filename, w, r, next) + } + return fsrv.notFound(w, r, next) } - return fsrv.notFound(w, r, next) - } - // one last check to ensure the file isn't hidden (we might - // have changed the filename from when we last checked) - if fileHidden(filename, filesToHide) { - fsrv.logger.Debug("hiding file", - zap.String("filename", filename), - zap.Strings("files_to_hide", filesToHide)) - return fsrv.notFound(w, r, next) - } + // one last check to ensure the file isn't hidden (we might + // have changed the filename from when we last checked) + if fileHidden(filename, filesToHide) { + fsrv.logger.Debug("hiding file", + zap.String("filename", filename), + zap.Strings("files_to_hide", filesToHide)) + return fsrv.notFound(w, r, next) + } - // if URL canonicalization is enabled, we need to enforce trailing - // slash convention: if a directory, trailing slash; if a file, no - // trailing slash - not enforcing this can break relative hrefs - // in HTML (see https://github.com/caddyserver/caddy/issues/2741) - if fsrv.CanonicalURIs == nil || *fsrv.CanonicalURIs { - // Only redirect if the last element of the path (the filename) was not - // rewritten; if the admin wanted to rewrite to the canonical path, they - // would have, and we have to be very careful not to introduce unwanted - // redirects and especially redirect loops! - // See https://github.com/caddyserver/caddy/issues/4205. - origReq := r.Context().Value(caddyhttp.OriginalRequestCtxKey).(http.Request) - if path.Base(origReq.URL.Path) == path.Base(r.URL.Path) { - if implicitIndexFile && !strings.HasSuffix(origReq.URL.Path, "/") { - to := origReq.URL.Path + "/" - fsrv.logger.Debug("redirecting to canonical URI (adding trailing slash for directory)", - zap.String("from_path", origReq.URL.Path), - zap.String("to_path", to)) - return redirect(w, r, to) - } else if !implicitIndexFile && strings.HasSuffix(origReq.URL.Path, "/") { - to := origReq.URL.Path[:len(origReq.URL.Path)-1] - fsrv.logger.Debug("redirecting to canonical URI (removing trailing slash for file)", - zap.String("from_path", origReq.URL.Path), - zap.String("to_path", to)) - return redirect(w, r, to) + // if URL canonicalization is enabled, we need to enforce trailing + // slash convention: if a directory, trailing slash; if a file, no + // trailing slash - not enforcing this can break relative hrefs + // in HTML (see https://github.com/caddyserver/caddy/issues/2741) + if fsrv.CanonicalURIs == nil || *fsrv.CanonicalURIs { + // Only redirect if the last element of the path (the filename) was not + // rewritten; if the admin wanted to rewrite to the canonical path, they + // would have, and we have to be very careful not to introduce unwanted + // redirects and especially redirect loops! + // See https://github.com/caddyserver/caddy/issues/4205. + origReq := r.Context().Value(caddyhttp.OriginalRequestCtxKey).(http.Request) + if path.Base(origReq.URL.Path) == path.Base(r.URL.Path) { + if implicitIndexFile && !strings.HasSuffix(origReq.URL.Path, "/") { + to := origReq.URL.Path + "/" + fsrv.logger.Debug("redirecting to canonical URI (adding trailing slash for directory)", + zap.String("from_path", origReq.URL.Path), + zap.String("to_path", to)) + return redirect(w, r, to) + } else if !implicitIndexFile && strings.HasSuffix(origReq.URL.Path, "/") { + to := origReq.URL.Path[:len(origReq.URL.Path)-1] + fsrv.logger.Debug("redirecting to canonical URI (removing trailing slash for file)", + zap.String("from_path", origReq.URL.Path), + zap.String("to_path", to)) + return redirect(w, r, to) + } } } } var file fs.File + var err error + var info fs.FileInfo + var compressedInfo fs.FileInfo // etag is usually unset, but if the user knows what they're doing, let them override it etag := w.Header().Get("Etag") @@ -366,7 +371,7 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c continue } compressedFilename := filename + precompress.Suffix() - compressedInfo, err := fs.Stat(fsrv.fileSystem, compressedFilename) + compressedInfo, err = fs.Stat(fsrv.fileSystem, compressedFilename) if err != nil || compressedInfo.IsDir() { fsrv.logger.Debug("precompressed file not accessible", zap.String("filename", compressedFilename), zap.Error(err)) continue @@ -478,7 +483,12 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c // that errors generated by ServeContent are written immediately // to the response, so we cannot handle them (but errors there // are rare) - http.ServeContent(w, r, info.Name(), info.ModTime(), file.(io.ReadSeeker)) + + if fsrv.PreferPrecompressed { + http.ServeContent(w, r, compressedInfo.Name(), compressedInfo.ModTime(), file.(io.ReadSeeker)) + } else { + http.ServeContent(w, r, info.Name(), info.ModTime(), file.(io.ReadSeeker)) + } return nil } From 4e97c9729c14eb53e7cea747a859b6d0309a4755 Mon Sep 17 00:00:00 2001 From: singhalkarun Date: Mon, 14 Aug 2023 23:27:38 +0530 Subject: [PATCH 3/3] 1. Refactored to use a info variable instead of 2 variables info and compressedInfo 2. Fixed an issue when the PreCompressed parameter is enabled and the original file is also present, need to fetch the FileInfo such that it can be used in ServeContent. --- modules/caddyhttp/fileserver/staticfiles.go | 33 ++++++++++++--------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/modules/caddyhttp/fileserver/staticfiles.go b/modules/caddyhttp/fileserver/staticfiles.go index 23400583e81..769a19a925e 100644 --- a/modules/caddyhttp/fileserver/staticfiles.go +++ b/modules/caddyhttp/fileserver/staticfiles.go @@ -257,9 +257,12 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c zap.String("request_path", r.URL.Path), zap.String("result", filename)) + var info fs.FileInfo + var err error + if !fsrv.PreferPrecompressed { // get information about the file - info, err := fs.Stat(fsrv.fileSystem, filename) + info, err = fs.Stat(fsrv.fileSystem, filename) if err != nil { err = fsrv.mapDirOpenError(err, filename) if errors.Is(err, fs.ErrNotExist) || errors.Is(err, fs.ErrInvalid) { @@ -357,9 +360,6 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c } var file fs.File - var err error - var info fs.FileInfo - var compressedInfo fs.FileInfo // etag is usually unset, but if the user knows what they're doing, let them override it etag := w.Header().Get("Etag") @@ -371,8 +371,8 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c continue } compressedFilename := filename + precompress.Suffix() - compressedInfo, err = fs.Stat(fsrv.fileSystem, compressedFilename) - if err != nil || compressedInfo.IsDir() { + info, err = fs.Stat(fsrv.fileSystem, compressedFilename) + if err != nil || info.IsDir() { fsrv.logger.Debug("precompressed file not accessible", zap.String("filename", compressedFilename), zap.Error(err)) continue } @@ -395,7 +395,7 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c // of transparent; however we do need to set the Etag: // https://caddy.community/t/gzipped-sidecar-file-wrong-same-etag/16793 if etag == "" { - etag = calculateEtag(compressedInfo) + etag = calculateEtag(info) } break @@ -403,9 +403,18 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c // no precompressed file found, use the actual file if file == nil { - fsrv.logger.Debug("opening file", zap.String("filename", filename)) - // open the file + info, err = fs.Stat(fsrv.fileSystem, filename) + if err != nil { + err = fsrv.mapDirOpenError(err, filename) + if errors.Is(err, fs.ErrNotExist) || errors.Is(err, fs.ErrInvalid) { + return fsrv.notFound(w, r, next) + } else if errors.Is(err, fs.ErrPermission) { + return caddyhttp.Error(http.StatusForbidden, err) + } + return caddyhttp.Error(http.StatusInternalServerError, err) + } + file, err = fsrv.openFile(filename, w) if err != nil { if herr, ok := err.(caddyhttp.HandlerError); ok && @@ -484,11 +493,7 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c // to the response, so we cannot handle them (but errors there // are rare) - if fsrv.PreferPrecompressed { - http.ServeContent(w, r, compressedInfo.Name(), compressedInfo.ModTime(), file.(io.ReadSeeker)) - } else { - http.ServeContent(w, r, info.Name(), info.ModTime(), file.(io.ReadSeeker)) - } + http.ServeContent(w, r, info.Name(), info.ModTime(), file.(io.ReadSeeker)) return nil }