From 295473aa39c03a6f6476554bcd8b9995338d6f01 Mon Sep 17 00:00:00 2001 From: Stefan Negru Date: Thu, 14 Dec 2023 22:26:29 +0200 Subject: [PATCH] try to fix coordinates --- api/sda/sda.go | 142 +++++++++++++++++++++++++++---------------------- 1 file changed, 79 insertions(+), 63 deletions(-) diff --git a/api/sda/sda.go b/api/sda/sda.go index c86731f..9ace494 100644 --- a/api/sda/sda.go +++ b/api/sda/sda.go @@ -12,7 +12,6 @@ import ( "time" "github.com/gin-gonic/gin" - "github.com/neicnordic/crypt4gh/model/headers" "github.com/neicnordic/crypt4gh/streaming" "github.com/neicnordic/sda-download/api/middleware" "github.com/neicnordic/sda-download/internal/config" @@ -176,15 +175,6 @@ func Download(c *gin.Context) { return } - // Get coordinates - coordinates, err := parseCoordinates(c.Request) - if err != nil { - log.Errorf("parsing of query param coordinates to crypt4gh format failed, reason: %v", err) - c.String(http.StatusBadRequest, err.Error()) - - return - } - c.Header("Content-Length", fmt.Sprint(fileDetails.DecryptedSize)) c.Header("Content-Type", "application/octet-stream") if c.GetBool("S3") { @@ -207,7 +197,7 @@ func Download(c *gin.Context) { } // Stitch file and prepare it for streaming - fileStream, err := stitchFile(fileDetails.Header, file, coordinates) + fileStream, err := stitchFile(fileDetails.Header, file) if err != nil { log.Errorf("could not prepare file for streaming, %s", err) c.String(http.StatusInternalServerError, "file stream error") @@ -215,81 +205,107 @@ func Download(c *gin.Context) { return } - sendStream(c.Writer, fileStream) -} - -// stitchFile stitches the header and file body together for Crypt4GHReader -// and returns a streamable Reader -var stitchFile = func(header []byte, file io.ReadCloser, coordinates *headers.DataEditListHeaderPacket) (*streaming.Crypt4GHReader, error) { - log.Debugf("stitching header to file %s for streaming", file) - // Stitch header and file body together - hr := bytes.NewReader(header) - mr := io.MultiReader(hr, file) - c4ghr, err := streaming.NewCrypt4GHReader(mr, *config.Config.App.Crypt4GHKey, coordinates) - if err != nil { - log.Errorf("failed to create Crypt4GH stream reader, %v", err) - - return nil, err - } - log.Debugf("file stream for %s constructed", file) - - return c4ghr, nil -} - -// parseCoordinates takes query param coordinates and converts them to -// Crypt4GH reader format -var parseCoordinates = func(r *http.Request) (*headers.DataEditListHeaderPacket, error) { - - coordinates := &headers.DataEditListHeaderPacket{} - // Get query params - qStart := r.URL.Query().Get("startCoordinate") - qEnd := r.URL.Query().Get("endCoordinate") + qStart := c.Request.URL.Query().Get("startCoordinate") + qEnd := c.Request.URL.Query().Get("endCoordinate") // Parse and verify coordinates are valid if len(qStart) > 0 && len(qEnd) > 0 { - start, err := strconv.ParseUint(qStart, 10, 64) + start, err := strconv.ParseInt(qStart, 10, 0) if err != nil { log.Errorf("failed to convert start coordinate %d to integer, %s", start, err) + c.String(http.StatusInternalServerError, "startCoordinate must be an integer") - return nil, errors.New("startCoordinate must be an integer") + return } - end, err := strconv.ParseUint(qEnd, 10, 64) + end, err := strconv.ParseInt(qEnd, 10, 0) if err != nil { log.Errorf("failed to convert end coordinate %d to integer, %s", end, err) - return nil, errors.New("endCoordinate must be an integer") + c.String(http.StatusInternalServerError, "endCoordinate must be an integer") + + return } if end < start { log.Errorf("endCoordinate=%d must be greater than startCoordinate=%d", end, start) - return nil, errors.New("endCoordinate must be greater than startCoordinate") + c.String(http.StatusInternalServerError, "endCoordinate must be greater than startCoordinate") + + return } - // API query params take a coordinate range to read "start...end" - // But Crypt4GHReader takes a start byte and number of bytes to read "start...(end-start)" - bytesToRead := end - start - coordinates.NumberLengths = 2 - coordinates.Lengths = []uint64{start, bytesToRead} + + if start != 0 { + // We don't want to read from start, skip ahead to where we should be + if _, err := fileStream.Seek(int64(start), 0); err != nil { + + c.String(http.StatusInternalServerError, "endCoordinate must be greater than startCoordinate") + + return + } + } + + // Calculate how much we should read (if given) + togo := end - start + + buf := make([]byte, 4096) + + // Loop until we've read what we should (if no/faulty end given, that's EOF) + for end == 0 || togo > 0 { + rbuf := buf[:] + + if end != 0 && togo < 4096 { + // If we don't want to read as much as 4096 bytes + rbuf = buf[:togo] + } + r, err := fileStream.Read(rbuf) + togo -= int64(r) + + // Nothing more to read? + if err == io.EOF && r == 0 { + // Fall out without error if we had EOF (if we got any data, do one + // more lap in the loop) + return + } + + if err != nil && err != io.EOF { + // An error we want to signal? + return + } + + wbuf := rbuf[:r] + for len(wbuf) > 0 { + // Loop until we've written all that we could read, + // fall out on error + w, err := c.Writer.Write(wbuf) + + if err != nil { + return + } + wbuf = wbuf[w:] + } + } + } else { - coordinates = nil + return } - return coordinates, nil } -// sendStream streams file contents from a reader -var sendStream = func(w http.ResponseWriter, file io.Reader) { - log.Debug("begin data stream") - - n, err := io.Copy(w, file) - log.Debug("end data stream") - +// stitchFile stitches the header and file body together for Crypt4GHReader +// and returns a streamable Reader +var stitchFile = func(header []byte, file io.ReadCloser) (*streaming.Crypt4GHReader, error) { + log.Debugf("stitching header to file %s for streaming", file) + // Stitch header and file body together + hr := bytes.NewReader(header) + mr := io.MultiReader(hr, file) + c4ghr, err := streaming.NewCrypt4GHReader(mr, *config.Config.App.Crypt4GHKey, nil) if err != nil { - log.Errorf("file streaming failed, reason: %v", err) - http.Error(w, "file streaming failed", 500) + log.Errorf("failed to create Crypt4GH stream reader, %v", err) - return + return nil, err } - log.Debugf("Sent %d bytes", n) + log.Debugf("file stream for %s constructed", file) + + return c4ghr, nil }