[BUG] CompressHandler if combined with http.Fileserver misses "Content-Encoding: deflate" on 404 status codes #259
Labels
Comments
|
More analysis: I guess, it's b/c of this Go std lib code in C:\Program Files\Go\src\net\http\fs.go which explicitly resets the header field "Content-Encoding": // serveError serves an error from ServeFile, ServeFileFS, and ServeContent.
// Because those can all be configured by the caller by setting headers like
// Etag, Last-Modified, and Cache-Control to send on a successful response,
// the error path needs to clear them, since they may not be meant for errors.
func serveError(w ResponseWriter, text string, code int) {
h := w.Header()
nonDefault := false
for _, k := range []string{
"Cache-Control",
"Content-Encoding", // <-------------- this line is "offensive"
"Etag",
"Last-Modified",
} {
if !h.has(k) {
continue
}
if httpservecontentkeepheaders.Value() == "1" {
nonDefault = true
} else {
h.Del(k)
}
}
if nonDefault {
httpservecontentkeepheaders.IncNonDefault()
}
Error(w, text, code)
}So, I'm really unsure what indeed should be the correct behavior for the compress handler. Maybe it's defensive to just turn off compression in case of non OK http Status codes. Or, to add the compression-algorithm-compatible "Content-Encoding" header later, i.e. right before emitting the compressed body. Or as the client app, honor this: // GODEBUG=httpservecontentkeepheaders=1 restores the pre-1.23 behavior of not deleting
// Cache-Control, Content-Encoding, Etag, or Last-Modified headers on ServeContent errors.
var httpservecontentkeepheaders = godebug.New("httpservecontentkeepheaders") |
|
This is a change with Go 1.23; the The previous behavior can be returned by setting |
|
do to this problem i ended up writing my own gzip middleware package middleware
import (
"bufio"
"compress/gzip"
"io"
"net"
"net/http"
"path/filepath"
"strings"
)
// isCompressibleContentType returns true if the given content type is likely text.
func isCompressibleContentType(ct string) bool {
// If empty, assume compressible.
if ct == "" {
return true
}
ct = strings.ToLower(ct)
// Text types and common compressible MIME types.
if strings.HasPrefix(ct, "text/") ||
strings.HasPrefix(ct, "application/json") ||
strings.HasPrefix(ct, "application/javascript") ||
strings.HasPrefix(ct, "application/xml") ||
strings.HasPrefix(ct, "image/svg") ||
strings.HasPrefix(ct, "application/x-javascript") {
return true
}
// Otherwise (e.g. image, video) assume not compressible.
return false
}
// GzipMiddleware is a middleware that compresses the response using gzip.
type gzipMiddlewareStruct struct {
Next http.Handler
}
// GzipMiddleware returns a new GzipMiddleware.
func GzipMiddleware(h http.Handler) http.Handler {
return &gzipMiddlewareStruct{h}
}
// ServeHTTP compresses the response using gzip.
func (gm *gzipMiddlewareStruct) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if gm.Next == nil {
gm.Next = http.DefaultServeMux
}
switch strings.ToUpper(filepath.Ext(r.URL.Path)) {
case ".MP3", ".MP4", ".WMA", ".WAV", ".WMV", ".APK":
gm.Next.ServeHTTP(w, r)
return
}
// Check if the client accepts gzip.
if !strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
gm.Next.ServeHTTP(w, r)
return
}
gzipWriter := gzip.NewWriter(w)
rw := &gzipResponseWriter{
responseWriter: w,
compressor: gzipWriter,
}
defer rw.Close()
gm.Next.ServeHTTP(rw, r)
}
// gzipResponseWriter wraps http.ResponseWriter to intercept the status code
// and the headers so we can decide whether to compress.
type gzipResponseWriter struct {
responseWriter http.ResponseWriter
compressor *gzip.Writer
status int
disableCompression bool
}
func (grw *gzipResponseWriter) Header() http.Header {
return grw.responseWriter.Header()
}
// WriteHeader captures the status code and checks the Content-Type.
// If the status is >= 400 or the Content-Type indicates binary data, disable compression.
func (grw *gzipResponseWriter) WriteHeader(code int) {
grw.status = code
// Disable compression for error responses.
if code >= http.StatusBadRequest {
grw.disableCompression = true
grw.responseWriter.Header().Del("Content-Encoding")
} else {
// Check Content-Type header if available.
ct := grw.responseWriter.Header().Get("Content-Type")
if !isCompressibleContentType(ct) {
grw.disableCompression = true
grw.responseWriter.Header().Del("Content-Encoding")
} else {
// Set Content-Encoding only if compression is enabled.
grw.responseWriter.Header().Set("Content-Encoding", "gzip")
}
}
grw.responseWriter.WriteHeader(code)
}
// Write writes data to the connection. If compression is disabled, write directly.
func (grw *gzipResponseWriter) Write(data []byte) (int, error) {
// if you try to write without WriteHeader() go will write it with out calling WriteHeader()
if grw.status == 0 {
grw.WriteHeader(http.StatusOK)
}
if grw.disableCompression {
return grw.responseWriter.Write(data)
}
return grw.compressor.Write(data)
}
// making a custom closer to run only if the gzip writer was used
func (grw *gzipResponseWriter) Close() error {
// if there is no body the system go will call the 200 and it will send gzip bytes without the gzip headers
if grw.status == 0 {
grw.WriteHeader(http.StatusOK)
}
if grw.disableCompression {
if f, ok := grw.responseWriter.(io.Closer); ok {
return f.Close()
}
return nil
}
return grw.compressor.Close()
}
func (grw *gzipResponseWriter) Flush() {
if f, ok := grw.responseWriter.(http.Flusher); ok {
f.Flush()
}
}
func (grw *gzipResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
if hj, ok := grw.responseWriter.(http.Hijacker); ok {
return hj.Hijack()
}
return nil, nil, http.ErrNotSupported
}
func (grw *gzipResponseWriter) Push(target string, opts *http.PushOptions) error {
if p, ok := grw.responseWriter.(http.Pusher); ok {
return p.Push(target, opts)
}
return http.ErrNotSupported
}
// Ensure our gzipResponseWriter implements the necessary interfaces.
var _ io.Closer = &gzipResponseWriter{}
var _ http.ResponseWriter = &gzipResponseWriter{}
var _ http.Pusher = &gzipResponseWriter{}
var _ http.Hijacker = &gzipResponseWriter{}
var _ http.Flusher = &gzipResponseWriter{} |
|
I also wrote my own middleware, but relied on the incoming Content-Encoding to determine if it should restore an outgoing Content-Encoding. import (
"net/http"
)
// compressHandler is an HTTP handler that adds the Content-Encoding header
// back to responses when removed by the http.FileServer.
//
// handlers.CompressHandler(newCompressHandler(http.FileServer(...)))
type compressHandler struct {
// handler is an HTTP handler, usually an http.FileServer.
handler http.Handler
}
var _ http.Handler = &compressHandler{}
func newCompressHandler(handler http.Handler) *compressHandler {
return &compressHandler{
handler: handler,
}
}
func (h *compressHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// The wrapped response writer saves the incoming content encoding so
// it can be restored when writing the response headers.
cw := &compressedResponseWriter{
encoding: w.Header().Get("Content-Encoding"),
fixed: false,
responseWriter: w,
}
h.handler.ServeHTTP(cw, r)
}
// compressedResponseWriter is an http.ResponseWriter that ensures that a
// previously-set Content-Encoding header is in place before writing the
// response.
type compressedResponseWriter struct {
encoding string
fixed bool
responseWriter http.ResponseWriter
}
var _ http.ResponseWriter = &compressedResponseWriter{}
func (w *compressedResponseWriter) Header() http.Header {
return w.responseWriter.Header()
}
func (w *compressedResponseWriter) fixContentEncoding() {
if w.fixed {
return
}
w.fixed = true
// The Go 1.23 http.FileServer() removes headers like Content-Encoding
// from error responses. This breaks gzip and deflate encoding.
// https://github.com/gorilla/handlers/issues/259
// https://github.com/golang/go/issues/66343
if w.encoding == "gzip" || w.encoding == "deflate" {
if w.Header().Get("Content-Encoding") == "" {
w.Header().Set("Content-Encoding", w.encoding)
}
}
}
func (w *compressedResponseWriter) Write(data []byte) (int, error) {
w.fixContentEncoding()
return w.responseWriter.Write(data)
}
func (w *compressedResponseWriter) WriteHeader(statusCode int) {
w.fixContentEncoding()
w.responseWriter.WriteHeader(statusCode)
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Is there an existing issue for this?
Current Behavior
curl --output - -i --compressed http://localhost:9000/notfound.htmloutputsfor the program listed in "steps to reproduce".
Expected Behavior
It should output either "deflate and len 25 bytes"
or "not compressed and len 19 bytes"
Steps To Reproduce
The "expected behavior" is present if the line
mux.PathPrefix("/").Methods("GET").Handler(http.FileServer(http.Dir(".")))is commented out. It degrades to the "current behavior" if the FileServer is activated.It works in both cases if a normal OK status code is produced (/index.html).
Chrome Dev tools behave similarly but different in details b/c of gzip / deflate request headers, I guess.
Anything else?
Sorry for not investigating more detail for the sample program. My more complex code worked with Go 1.22.x and handlers v1.5.1 and I don't know if it's an issue of the newer Go std lib, of gorilla/mux or gorilla/handlers.
Thanks
Martin
The text was updated successfully, but these errors were encountered: