From f4f5fa93583f0f49bd88a4c384ffe0999568b514 Mon Sep 17 00:00:00 2001 From: Stefan Negru Date: Tue, 19 Dec 2023 18:08:45 +0200 Subject: [PATCH 1/6] adjust for download with coordinates --- api/sda/sda.go | 147 ++++++++-------- api/sda/sda_test.go | 414 +------------------------------------------- 2 files changed, 77 insertions(+), 484 deletions(-) diff --git a/api/sda/sda.go b/api/sda/sda.go index c86731f..4c5c215 100644 --- a/api/sda/sda.go +++ b/api/sda/sda.go @@ -12,7 +12,6 @@ import ( "time" "github.com/gin-gonic/gin" - "github.com/neicnordic/crypt4gh/model/headers" "github.com/neicnordic/crypt4gh/streaming" "github.com/neicnordic/sda-download/api/middleware" "github.com/neicnordic/sda-download/internal/config" @@ -176,16 +175,6 @@ func Download(c *gin.Context) { return } - // Get coordinates - coordinates, err := parseCoordinates(c.Request) - if err != nil { - log.Errorf("parsing of query param coordinates to crypt4gh format failed, reason: %v", err) - c.String(http.StatusBadRequest, err.Error()) - - return - } - - c.Header("Content-Length", fmt.Sprint(fileDetails.DecryptedSize)) c.Header("Content-Type", "application/octet-stream") if c.GetBool("S3") { lastModified, err := time.Parse(time.RFC3339, fileDetails.LastModified) @@ -206,90 +195,106 @@ func Download(c *gin.Context) { return } - // Stitch file and prepare it for streaming - fileStream, err := stitchFile(fileDetails.Header, file, coordinates) + hr := bytes.NewReader(fileDetails.Header) + mr := io.MultiReader(hr, file) + c4ghr, err := streaming.NewCrypt4GHReader(mr, *config.Config.App.Crypt4GHKey, nil) if err != nil { log.Errorf("could not prepare file for streaming, %s", err) c.String(http.StatusInternalServerError, "file stream error") return } + defer c4ghr.Close() - sendStream(c.Writer, fileStream) -} + // Get query params + qStart := c.DefaultQuery("startCoordinate", "0") + qEnd := c.DefaultQuery("endCoordinate", "0") -// stitchFile stitches the header and file body together for Crypt4GHReader -// and returns a streamable Reader -var stitchFile = func(header []byte, file io.ReadCloser, coordinates *headers.DataEditListHeaderPacket) (*streaming.Crypt4GHReader, error) { - log.Debugf("stitching header to file %s for streaming", file) - // Stitch header and file body together - hr := bytes.NewReader(header) - mr := io.MultiReader(hr, file) - c4ghr, err := streaming.NewCrypt4GHReader(mr, *config.Config.App.Crypt4GHKey, coordinates) + // Parse and verify coordinates are valid + start, err := strconv.ParseInt(qStart, 10, 0) if err != nil { - log.Errorf("failed to create Crypt4GH stream reader, %v", err) + log.Errorf("failed to convert start coordinate %d to integer, %s", start, err) + c.String(http.StatusBadRequest, "startCoordinate must be an integer") - return nil, err + return } - log.Debugf("file stream for %s constructed", file) - - return c4ghr, nil -} - -// parseCoordinates takes query param coordinates and converts them to -// Crypt4GH reader format -var parseCoordinates = func(r *http.Request) (*headers.DataEditListHeaderPacket, error) { + end, err := strconv.ParseInt(qEnd, 10, 0) + if err != nil { + log.Errorf("failed to convert end coordinate %d to integer, %s", end, err) - coordinates := &headers.DataEditListHeaderPacket{} + c.String(http.StatusBadRequest, "endCoordinate must be an integer") - // Get query params - qStart := r.URL.Query().Get("startCoordinate") - qEnd := r.URL.Query().Get("endCoordinate") + return + } + if end < start { + log.Errorf("endCoordinate=%d must be greater than startCoordinate=%d", end, start) - // Parse and verify coordinates are valid - if len(qStart) > 0 && len(qEnd) > 0 { - start, err := strconv.ParseUint(qStart, 10, 64) - if err != nil { - log.Errorf("failed to convert start coordinate %d to integer, %s", start, err) + c.String(http.StatusBadRequest, "endCoordinate must be greater than startCoordinate") - return nil, errors.New("startCoordinate must be an integer") - } - end, err := strconv.ParseUint(qEnd, 10, 64) - if err != nil { - log.Errorf("failed to convert end coordinate %d to integer, %s", end, err) - - return nil, errors.New("endCoordinate must be an integer") - } - if end < start { - log.Errorf("endCoordinate=%d must be greater than startCoordinate=%d", end, start) + return + } - return nil, errors.New("endCoordinate must be greater than startCoordinate") - } - // API query params take a coordinate range to read "start...end" - // But Crypt4GHReader takes a start byte and number of bytes to read "start...(end-start)" - bytesToRead := end - start - coordinates.NumberLengths = 2 - coordinates.Lengths = []uint64{start, bytesToRead} + if start == 0 && end == 0 { + c.Header("Content-Length", fmt.Sprint(fileDetails.DecryptedSize)) } else { - coordinates = nil + // Calculate how much we should read (if given) + togo := end - start + c.Header("Content-Length", fmt.Sprint(togo)) } - return coordinates, nil + err = sendStream(c4ghr, c.Writer, start, end) + if err != nil { + log.Errorf("error occurred while sending stream: %v", err) + c.String(http.StatusInternalServerError, "an error occurred") + + return + } } -// sendStream streams file contents from a reader -var sendStream = func(w http.ResponseWriter, file io.Reader) { - log.Debug("begin data stream") +// sendStream +// used from: https://github.com/neicnordic/crypt4gh/blob/master/examples/reader/main.go#L48C1-L113C1 +var sendStream = func(reader *streaming.Crypt4GHReader, writer http.ResponseWriter, start, end int64) error { - n, err := io.Copy(w, file) - log.Debug("end data stream") + // Calculate how much we should read (if given) + togo := end - start - if err != nil { - log.Errorf("file streaming failed, reason: %v", err) - http.Error(w, "file streaming failed", 500) + buf := make([]byte, 4096) - return + // Loop until we've read what we should (if no/faulty end given, that's EOF) + for end == 0 || togo > 0 { + rbuf := buf + + if end != 0 && togo < 4096 { + // If we don't want to read as much as 4096 bytes + rbuf = buf[:togo] + } + r, err := reader.Read(rbuf) + togo -= int64(r) + + // Nothing more to read? + if err == io.EOF && r == 0 { + // Fall out without error if we had EOF (if we got any data, do one + // more lap in the loop) + return nil + } + + if err != nil && err != io.EOF { + // An error we want to signal? + return err + } + + wbuf := rbuf[:r] + for len(wbuf) > 0 { + // Loop until we've written all that we could read, + // fall out on error + w, err := writer.Write(wbuf) + + if err != nil { + return err + } + wbuf = wbuf[w:] + } } - log.Debugf("Sent %d bytes", n) + return nil } diff --git a/api/sda/sda_test.go b/api/sda/sda_test.go index 148a777..3d3f2c4 100644 --- a/api/sda/sda_test.go +++ b/api/sda/sda_test.go @@ -4,14 +4,11 @@ import ( "bytes" "errors" "io" - "net/http" "net/http/httptest" - "os" "testing" "github.com/gin-gonic/gin" - "github.com/neicnordic/crypt4gh/model/headers" - "github.com/neicnordic/crypt4gh/streaming" + "github.com/neicnordic/sda-download/api/middleware" "github.com/neicnordic/sda-download/internal/config" "github.com/neicnordic/sda-download/internal/database" @@ -323,103 +320,6 @@ func TestFiles_Success(t *testing.T) { } -func TestParseCoordinates_Fail_Start(t *testing.T) { - - // Test case - // startCoordinate must be an integer - r := httptest.NewRequest("GET", "https://testing.fi?startCoordinate=x&endCoordinate=100", nil) - - // Run test target - coordinates, err := parseCoordinates(r) - - // Expected results - expectedError := "startCoordinate must be an integer" - - if err.Error() != expectedError { - t.Errorf("TestParseCoordinates_Fail_Start failed, got %s expected %s", err.Error(), expectedError) - } - if coordinates != nil { - t.Errorf("TestParseCoordinates_Fail_Start failed, got %v expected nil", coordinates) - } - -} - -func TestParseCoordinates_Fail_End(t *testing.T) { - - // Test case - // endCoordinate must be an integer - r := httptest.NewRequest("GET", "https://testing.fi?startCoordinate=0&endCoordinate=y", nil) - - // Run test target - coordinates, err := parseCoordinates(r) - - // Expected results - expectedError := "endCoordinate must be an integer" - - if err.Error() != expectedError { - t.Errorf("TestParseCoordinates_Fail_End failed, got %s expected %s", err.Error(), expectedError) - } - if coordinates != nil { - t.Errorf("TestParseCoordinates_Fail_End failed, got %v expected nil", coordinates) - } - -} - -func TestParseCoordinates_Fail_SizeComparison(t *testing.T) { - - // Test case - // endCoordinate must be greater than startCoordinate - r := httptest.NewRequest("GET", "https://testing.fi?startCoordinate=50&endCoordinate=100", nil) - - // Run test target - coordinates, err := parseCoordinates(r) - - // Expected results - expectedLength := uint32(2) - expectedStart := uint64(50) - expectedBytesToRead := uint64(50) - - if err != nil { - t.Errorf("TestParseCoordinates_Fail_SizeComparison failed, got %v expected nil", err) - } - // nolint:staticcheck - if coordinates == nil { - t.Error("TestParseCoordinates_Fail_SizeComparison failed, got nil expected not nil") - } - // nolint:staticcheck - if coordinates.NumberLengths != expectedLength { - t.Errorf("TestParseCoordinates_Fail_SizeComparison failed, got %d expected %d", coordinates.Lengths, expectedLength) - } - if coordinates.Lengths[0] != expectedStart { - t.Errorf("TestParseCoordinates_Fail_SizeComparison failed, got %d expected %d", coordinates.Lengths, expectedLength) - } - if coordinates.Lengths[1] != expectedBytesToRead { - t.Errorf("TestParseCoordinates_Fail_SizeComparison failed, got %d expected %d", coordinates.Lengths, expectedLength) - } - -} - -func TestParseCoordinates_Success(t *testing.T) { - - // Test case - // endCoordinate must be greater than startCoordinate - r := httptest.NewRequest("GET", "https://testing.fi?startCoordinate=100&endCoordinate=50", nil) - - // Run test target - coordinates, err := parseCoordinates(r) - - // Expected results - expectedError := "endCoordinate must be greater than startCoordinate" - - if err.Error() != expectedError { - t.Errorf("TestParseCoordinates_Fail_SizeComparison failed, got %s expected %s", err.Error(), expectedError) - } - if coordinates != nil { - t.Errorf("TestParseCoordinates_Fail_SizeComparison failed, got %v expected nil", coordinates) - } - -} - func TestDownload_Fail_FileNotFound(t *testing.T) { // Save original to-be-mocked functions @@ -604,315 +504,3 @@ func TestDownload_Fail_OpenFile(t *testing.T) { database.GetFile = originalGetFile } - -func TestDownload_Fail_ParseCoordinates(t *testing.T) { - - // Save original to-be-mocked functions - originalCheckFilePermission := database.CheckFilePermission - originalGetCacheFromContext := middleware.GetCacheFromContext - originalGetFile := database.GetFile - originalParseCoordinates := parseCoordinates - config.Config.Archive.Posix.Location = "." - Backend, _ = storage.NewBackend(config.Config.Archive) - - // Substitute mock functions - database.CheckFilePermission = func(fileID string) (string, error) { - return "dataset1", nil - } - middleware.GetCacheFromContext = func(ctx *gin.Context) session.Cache { - return session.Cache{ - Datasets: []string{"dataset1"}, - } - } - database.GetFile = func(fileID string) (*database.FileDownload, error) { - fileDetails := &database.FileDownload{ - ArchivePath: "../../README.md", - ArchiveSize: 0, - Header: []byte{}, - } - - return fileDetails, nil - } - parseCoordinates = func(r *http.Request) (*headers.DataEditListHeaderPacket, error) { - return nil, errors.New("bad params") - } - - // Mock request and response holders - w := httptest.NewRecorder() - c, _ := gin.CreateTestContext(w) - - // Test the outcomes of the handler - Download(c) - response := w.Result() - defer response.Body.Close() - body, _ := io.ReadAll(response.Body) - expectedStatusCode := 400 - expectedBody := []byte("bad params") - - if response.StatusCode != expectedStatusCode { - t.Errorf("TestDownload_Fail_ParseCoordinates failed, got %d expected %d", response.StatusCode, expectedStatusCode) - } - if !bytes.Equal(body, []byte(expectedBody)) { - // visual byte comparison in terminal (easier to find string differences) - t.Error(body) - t.Error([]byte(expectedBody)) - t.Errorf("TestDownload_Fail_ParseCoordinates failed, got %s expected %s", string(body), string(expectedBody)) - } - - // Return mock functions to originals - database.CheckFilePermission = originalCheckFilePermission - middleware.GetCacheFromContext = originalGetCacheFromContext - database.GetFile = originalGetFile - parseCoordinates = originalParseCoordinates - -} - -func TestDownload_Fail_StreamFile(t *testing.T) { - - // Save original to-be-mocked functions - originalCheckFilePermission := database.CheckFilePermission - originalGetCacheFromContext := middleware.GetCacheFromContext - originalGetFile := database.GetFile - originalParseCoordinates := parseCoordinates - originalStitchFile := stitchFile - config.Config.Archive.Posix.Location = "." - Backend, _ = storage.NewBackend(config.Config.Archive) - - // Substitute mock functions - database.CheckFilePermission = func(fileID string) (string, error) { - return "dataset1", nil - } - middleware.GetCacheFromContext = func(ctx *gin.Context) session.Cache { - return session.Cache{ - Datasets: []string{"dataset1"}, - } - } - database.GetFile = func(fileID string) (*database.FileDownload, error) { - fileDetails := &database.FileDownload{ - ArchivePath: "../../README.md", - ArchiveSize: 0, - DecryptedSize: 0, - Header: []byte{}, - } - - return fileDetails, nil - } - parseCoordinates = func(r *http.Request) (*headers.DataEditListHeaderPacket, error) { - return nil, nil - } - stitchFile = func(header []byte, file io.ReadCloser, coordinates *headers.DataEditListHeaderPacket) (*streaming.Crypt4GHReader, error) { - return nil, errors.New("file stream error") - } - - // Mock request and response holders - w := httptest.NewRecorder() - c, _ := gin.CreateTestContext(w) - c.Request = httptest.NewRequest("GET", "/", nil) - - // Test the outcomes of the handler - Download(c) - response := w.Result() - defer response.Body.Close() - body, _ := io.ReadAll(response.Body) - expectedStatusCode := 500 - expectedBody := []byte("file stream error") - - if response.StatusCode != expectedStatusCode { - t.Errorf("TestDownload_Fail_StreamFile failed, got %d expected %d", response.StatusCode, expectedStatusCode) - } - if !bytes.Equal(body, []byte(expectedBody)) { - // visual byte comparison in terminal (easier to find string differences) - t.Error(body) - t.Error([]byte(expectedBody)) - t.Errorf("TestDownload_Fail_StreamFile failed, got %s expected %s", string(body), string(expectedBody)) - } - - // Return mock functions to originals - database.CheckFilePermission = originalCheckFilePermission - middleware.GetCacheFromContext = originalGetCacheFromContext - database.GetFile = originalGetFile - parseCoordinates = originalParseCoordinates - stitchFile = originalStitchFile - -} - -func TestDownload_Success(t *testing.T) { - - // Save original to-be-mocked functions - originalCheckFilePermission := database.CheckFilePermission - originalGetCacheFromContext := middleware.GetCacheFromContext - originalGetFile := database.GetFile - originalParseCoordinates := parseCoordinates - originalStitchFile := stitchFile - originalSendStream := sendStream - config.Config.Archive.Posix.Location = "." - Backend, _ = storage.NewBackend(config.Config.Archive) - - // Substitute mock functions - database.CheckFilePermission = func(fileID string) (string, error) { - return "dataset1", nil - } - middleware.GetCacheFromContext = func(ctx *gin.Context) session.Cache { - return session.Cache{ - Datasets: []string{"dataset1"}, - } - } - database.GetFile = func(fileID string) (*database.FileDownload, error) { - fileDetails := &database.FileDownload{ - ArchivePath: "../../README.md", - ArchiveSize: 0, - DecryptedSize: 0, - Header: []byte{}, - } - - return fileDetails, nil - } - parseCoordinates = func(r *http.Request) (*headers.DataEditListHeaderPacket, error) { - return nil, nil - } - stitchFile = func(header []byte, file io.ReadCloser, coordinates *headers.DataEditListHeaderPacket) (*streaming.Crypt4GHReader, error) { - return nil, nil - } - sendStream = func(w http.ResponseWriter, file io.Reader) { - fileReader := bytes.NewReader([]byte("hello\n")) - _, _ = io.Copy(w, fileReader) - } - - // Mock request and response holders - w := httptest.NewRecorder() - c, _ := gin.CreateTestContext(w) - c.Request = httptest.NewRequest("GET", "/", nil) - - // Test the outcomes of the handler - Download(c) - response := w.Result() - defer response.Body.Close() - body, _ := io.ReadAll(response.Body) - expectedStatusCode := 200 - expectedBody := []byte("hello\n") - - if response.StatusCode != expectedStatusCode { - t.Errorf("TestDownload_Success failed, got %d expected %d", response.StatusCode, expectedStatusCode) - } - if !bytes.Equal(body, []byte(expectedBody)) { - // visual byte comparison in terminal (easier to find string differences) - t.Error(body) - t.Error([]byte(expectedBody)) - t.Errorf("TestDownload_Success failed, got %s expected %s", string(body), string(expectedBody)) - } - - // Return mock functions to originals - database.CheckFilePermission = originalCheckFilePermission - middleware.GetCacheFromContext = originalGetCacheFromContext - database.GetFile = originalGetFile - parseCoordinates = originalParseCoordinates - stitchFile = originalStitchFile - sendStream = originalSendStream - -} - -func TestSendStream(t *testing.T) { - // Mock file - file := []byte("hello\n") - fileReader := bytes.NewReader(file) - - // Mock stream response - w := httptest.NewRecorder() - w.Header().Add("Content-Length", "5") - - // Send file to streamer - sendStream(w, fileReader) - response := w.Result() - defer response.Body.Close() - body, _ := io.ReadAll(response.Body) - expectedContentLen := "5" - expectedBody := []byte("hello\n") - - // Verify that stream received contents - if contentLen := response.Header.Get("Content-Length"); contentLen != expectedContentLen { - t.Errorf("TestSendStream failed, got %s, expected %s", contentLen, expectedContentLen) - } - if !bytes.Equal(body, []byte(expectedBody)) { - t.Errorf("TestSendStream failed, got %s, expected %s", string(body), string(expectedBody)) - } -} - -func TestStitchFile_Fail(t *testing.T) { - - // Set test decryption key - config.Config.App.Crypt4GHKey = &[32]byte{} - - // Test header - header := []byte("header") - - // Test file body - testFile, err := os.CreateTemp("/tmp", "_sda_download_test_file") - if err != nil { - t.Errorf("TestStitchFile_Fail failed to create temp file, %v", err) - } - defer os.Remove(testFile.Name()) - defer testFile.Close() - const data = "hello, here is some test data\n" - _, _ = io.WriteString(testFile, data) - - // Test - fileStream, err := stitchFile(header, testFile, nil) - - // Expected results - expectedError := "not a Crypt4GH file" - - if err.Error() != expectedError { - t.Errorf("TestStitchFile_Fail failed, got %s expected %s", err.Error(), expectedError) - } - if fileStream != nil { - t.Errorf("TestStitchFile_Fail failed, got %v expected nil", fileStream) - } - -} - -func TestStitchFile_Success(t *testing.T) { - - // Set test decryption key - config.Config.App.Crypt4GHKey = &[32]byte{104, 35, 143, 159, 198, 120, 0, 145, 227, 124, 101, 127, 223, - 22, 252, 57, 224, 114, 205, 70, 150, 10, 28, 79, 192, 242, 151, 202, 44, 51, 36, 97} - - // Test header - header := []byte{99, 114, 121, 112, 116, 52, 103, 104, 1, 0, 0, 0, 1, 0, 0, 0, 108, 0, 0, 0, 0, 0, 0, 0, - 44, 219, 36, 17, 144, 78, 250, 192, 85, 103, 229, 122, 90, 11, 223, 131, 246, 165, 142, 191, 83, 97, - 206, 225, 206, 114, 10, 235, 239, 160, 206, 82, 55, 101, 76, 39, 217, 91, 249, 206, 122, 241, 69, 142, - 155, 97, 24, 47, 112, 45, 165, 197, 159, 60, 92, 214, 160, 112, 21, 129, 73, 31, 159, 54, 210, 4, 44, - 147, 108, 119, 178, 95, 194, 195, 11, 249, 60, 53, 133, 77, 93, 62, 31, 218, 29, 65, 143, 123, 208, 234, - 249, 34, 58, 163, 32, 149, 156, 110, 68, 49} - - // Test file body - testFile, err := os.CreateTemp("/tmp", "_sda_download_test_file") - if err != nil { - t.Errorf("TestStitchFile_Fail failed to create temp file, %v", err) - } - defer os.Remove(testFile.Name()) - defer testFile.Close() - testData := []byte{237, 0, 67, 9, 203, 239, 12, 187, 86, 6, 195, 174, 56, 234, 44, 78, 140, 2, 195, 5, 252, - 199, 244, 189, 150, 209, 144, 197, 61, 72, 73, 155, 205, 210, 206, 160, 226, 116, 242, 134, 63, 224, 178, - 153, 13, 181, 78, 210, 151, 219, 156, 18, 210, 70, 194, 76, 152, 178} - _, _ = testFile.Write(testData) - - // Test - // The decryption passes, but for some reason the temp test file doesn't return any data, so we can just check for error here - _, err = stitchFile(header, testFile, nil) - // fileStream, err := stitchFile(header, testFile, nil) - // data, err := io.ReadAll(fileStream) - - // Expected results - // expectedData := "hello, here is some test data" - - if err != nil { - t.Errorf("TestStitchFile_Success failed, got %v expected nil", err) - } - // if !bytes.Equal(data, []byte(expectedData)) { - // // visual byte comparison in terminal (easier to find string differences) - // t.Error(data) - // t.Error([]byte(expectedData)) - // t.Errorf("TestStitchFile_Success failed, got %s expected %s", string(data), string(expectedData)) - // } - -} From 13b4d1528ca7d3c04d830e2a8d654ac266d5b428 Mon Sep 17 00:00:00 2001 From: Stefan Negru Date: Tue, 19 Dec 2023 18:08:49 +0200 Subject: [PATCH 2/6] disable logging health endpoint --- api/api.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/api/api.go b/api/api.go index caadb6d..9699bef 100644 --- a/api/api.go +++ b/api/api.go @@ -32,7 +32,11 @@ func Setup() *http.Server { // Set up routing log.Info("(2/5) Registering endpoint handlers") - router := gin.Default() + router := gin.New() + router.Use( + gin.LoggerWithWriter(gin.DefaultWriter, "/health"), + gin.Recovery(), + ) router.HandleMethodNotAllowed = true From e8ca1fe48a507166682e471b67bddb6063f6c13c Mon Sep 17 00:00:00 2001 From: Stefan Negru Date: Tue, 19 Dec 2023 18:09:02 +0200 Subject: [PATCH 3/6] add integration tests for coordiantes --- .../tests/common/50_check_endpoint.sh | 24 +++++++++++++++++++ .../tests/s3notls/52_check_endpoint.sh | 24 +++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/.github/integration/tests/common/50_check_endpoint.sh b/.github/integration/tests/common/50_check_endpoint.sh index 05886a3..edbff75 100755 --- a/.github/integration/tests/common/50_check_endpoint.sh +++ b/.github/integration/tests/common/50_check_endpoint.sh @@ -90,6 +90,30 @@ else echo "Files are different" fi +curl --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/files/urn:neic:001-002?startCoordinate=0&endCoordinate=2" --output test-part.txt + +echo "TH" > old-part.txt + +cmp --silent old-part.txt test-part.txt +status=$? +if [[ $status = 0 ]]; then + echo "Files are the same" +else + echo "Files are different" +fi + +curl --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/files/urn:neic:001-002?startCoordinate=7&endCoordinate=14" --output test-part2.txt + +echo "LE IS J" > old-part2.txt + +cmp --silent old-part2.txt test-part2.txt +status=$? +if [[ $status = 0 ]]; then + echo "Files are the same" +else + echo "Files are different" +fi + # ------------------ # Test get visas failed diff --git a/.github/integration/tests/s3notls/52_check_endpoint.sh b/.github/integration/tests/s3notls/52_check_endpoint.sh index d936dbd..b49e0c3 100644 --- a/.github/integration/tests/s3notls/52_check_endpoint.sh +++ b/.github/integration/tests/s3notls/52_check_endpoint.sh @@ -86,6 +86,30 @@ else echo "Files are different" fi +curl -H "Authorization: Bearer $token" "http://localhost:8080/files/urn:neic:001-002?startCoordinate=0&endCoordinate=2" --output test-part.txt + +echo "TH" > old-part.txt + +cmp --silent old-part.txt test-part.txt +status=$? +if [[ $status = 0 ]]; then + echo "Files are the same" +else + echo "Files are different" +fi + +curl -H "Authorization: Bearer $token" "http://localhost:8080/files/urn:neic:001-002?startCoordinate=7&endCoordinate=14" --output test-part2.txt + +echo "LE IS J" > old-part2.txt + +cmp --silent old-part2.txt test-part2.txt +status=$? +if [[ $status = 0 ]]; then + echo "Files are the same" +else + echo "Files are different" +fi + # ------------------ # Test get visas failed From 28b1705d47c2eddbe03b52a49e0782e694cf02e1 Mon Sep 17 00:00:00 2001 From: Stefan Negru Date: Tue, 2 Jan 2024 14:38:43 +0200 Subject: [PATCH 4/6] match latest dependencies crypt4gh v1.8.7 --- go.mod | 6 +++--- go.sum | 15 ++++++++++----- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 0d3cdfd..a5bd85f 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/spf13/viper v1.18.2 github.com/stretchr/testify v1.8.4 - golang.org/x/exp v0.0.0-20230905200255-921286631fa9 + golang.org/x/exp v0.0.0-20231226003508-02704c960a9b ) require ( @@ -58,7 +58,7 @@ require ( github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/segmentio/asm v1.2.0 // indirect - github.com/shabbyrobe/gocovmerge v0.0.0-20180507124511-f6ea450bfb63 // indirect + github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500 // indirect github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.6.0 // indirect @@ -73,7 +73,7 @@ require ( golang.org/x/net v0.19.0 // indirect golang.org/x/sys v0.15.0 // indirect golang.org/x/text v0.14.0 // indirect - golang.org/x/tools v0.13.0 // indirect + golang.org/x/tools v0.16.0 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index 2703956..649854f 100644 --- a/go.sum +++ b/go.sum @@ -124,8 +124,9 @@ github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6g github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= -github.com/shabbyrobe/gocovmerge v0.0.0-20180507124511-f6ea450bfb63 h1:J6qvD6rbmOil46orKqJaRPG+zTpoGlBTUdyv8ki63L0= github.com/shabbyrobe/gocovmerge v0.0.0-20180507124511-f6ea450bfb63/go.mod h1:n+VKSARF5y/tS9XFSP7vWDfS+GUC5vs/YT7M5XDTUEM= +github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500 h1:WnNuhiq+FOY3jNj6JXFT+eLN3CQ/oPIsDPRanvwsmbI= +github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500/go.mod h1:+njLrG5wSeoG4Ds61rFgEzKvenR2UHbjMoDHsczxly0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= @@ -170,12 +171,14 @@ golang.org/x/arch v0.4.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= -golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= -golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= +golang.org/x/exp v0.0.0-20231226003508-02704c960a9b h1:kLiC65FbiHWFAOu+lxwNPujcsl8VYyTYYEZnsOO1WK4= +golang.org/x/exp v0.0.0-20231226003508-02704c960a9b/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -189,8 +192,10 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20190308174544-00c44ba9c14f/go.mod h1:25r3+/G6/xytQM8iWZKq3Hn0kr0rgFKPUNVEL/dr3z4= -golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ= -golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.0.0-20190829051458-42f498d34c4d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.16.0 h1:GO788SKMRunPIBCXiQyo2AaexLstOrVhuAL5YwsckQM= +golang.org/x/tools v0.16.0/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= From 7b8771ebf476d46904ddb207ac2544173c083dd2 Mon Sep 17 00:00:00 2001 From: Stefan Negru Date: Thu, 11 Jan 2024 16:27:24 +0200 Subject: [PATCH 5/6] update to crypt4gh 1.8.8 --- go.mod | 6 +++--- go.sum | 14 +++++++------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index a5bd85f..720dbe8 100644 --- a/go.mod +++ b/go.mod @@ -11,11 +11,11 @@ require ( github.com/johannesboyne/gofakes3 v0.0.0-20230129080941-f6a8a9ae6fd3 github.com/lestrrat-go/jwx/v2 v2.0.19 github.com/lib/pq v1.10.9 - github.com/neicnordic/crypt4gh v1.8.6 + github.com/neicnordic/crypt4gh v1.8.8 github.com/sirupsen/logrus v1.9.3 github.com/spf13/viper v1.18.2 github.com/stretchr/testify v1.8.4 - golang.org/x/exp v0.0.0-20231226003508-02704c960a9b + golang.org/x/exp v0.0.0-20240110193028-0dcbfd608b1e ) require ( @@ -71,7 +71,7 @@ require ( golang.org/x/arch v0.4.0 // indirect golang.org/x/crypto v0.17.0 // indirect golang.org/x/net v0.19.0 // indirect - golang.org/x/sys v0.15.0 // indirect + golang.org/x/sys v0.16.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/tools v0.16.0 // indirect google.golang.org/protobuf v1.31.0 // indirect diff --git a/go.sum b/go.sum index 649854f..3150420 100644 --- a/go.sum +++ b/go.sum @@ -106,8 +106,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/neicnordic/crypt4gh v1.8.6 h1:ElDWW/jXTRhBxHfA253rJFEpWCShRWceSVT+giLQ8xs= -github.com/neicnordic/crypt4gh v1.8.6/go.mod h1:U3QKgd/TDY1g1qFAx2/AxEtzzLsyzHshGGO50ZJmOv0= +github.com/neicnordic/crypt4gh v1.8.8 h1:mP0VSAspADLIJPC+a0o4lfzX49O4clkMkYoXKRHh6MM= +github.com/neicnordic/crypt4gh v1.8.8/go.mod h1:Wq3Y/xgaEA2rDoYiqT91uIXIlZO2IJOWnwtcQg4RiqU= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -171,8 +171,8 @@ golang.org/x/arch v0.4.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= -golang.org/x/exp v0.0.0-20231226003508-02704c960a9b h1:kLiC65FbiHWFAOu+lxwNPujcsl8VYyTYYEZnsOO1WK4= -golang.org/x/exp v0.0.0-20231226003508-02704c960a9b/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= +golang.org/x/exp v0.0.0-20240110193028-0dcbfd608b1e h1:723BNChdd0c2Wk6WOE320qGBiPtYx0F0Bbm1kriShfE= +golang.org/x/exp v0.0.0-20240110193028-0dcbfd608b1e/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -185,9 +185,9 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= +golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= From 03449f45189ec780782d07a1c3358b827ae835e3 Mon Sep 17 00:00:00 2001 From: Stefan Negru Date: Tue, 16 Jan 2024 13:34:17 +0200 Subject: [PATCH 6/6] fix wrong assumption to file start Co-authored-by: Alex Aperis <76202622+aaperis@users.noreply.github.com> --- .../tests/common/50_check_endpoint.sh | 6 +- .../tests/s3notls/52_check_endpoint.sh | 6 +- api/sda/sda.go | 7 +++ go.mod | 10 +-- go.sum | 61 +++++++++++++------ 5 files changed, 64 insertions(+), 26 deletions(-) diff --git a/.github/integration/tests/common/50_check_endpoint.sh b/.github/integration/tests/common/50_check_endpoint.sh index edbff75..3f21f83 100755 --- a/.github/integration/tests/common/50_check_endpoint.sh +++ b/.github/integration/tests/common/50_check_endpoint.sh @@ -92,7 +92,7 @@ fi curl --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/files/urn:neic:001-002?startCoordinate=0&endCoordinate=2" --output test-part.txt -echo "TH" > old-part.txt +dd if=old-file.txt ibs=1 skip=0 count=2 > old-part.txt cmp --silent old-part.txt test-part.txt status=$? @@ -100,11 +100,12 @@ if [[ $status = 0 ]]; then echo "Files are the same" else echo "Files are different" + exit 1 fi curl --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/files/urn:neic:001-002?startCoordinate=7&endCoordinate=14" --output test-part2.txt -echo "LE IS J" > old-part2.txt +dd if=old-file.txt ibs=1 skip=7 count=7 > old-part2.txt cmp --silent old-part2.txt test-part2.txt status=$? @@ -112,6 +113,7 @@ if [[ $status = 0 ]]; then echo "Files are the same" else echo "Files are different" + exit 1 fi # ------------------ diff --git a/.github/integration/tests/s3notls/52_check_endpoint.sh b/.github/integration/tests/s3notls/52_check_endpoint.sh index b49e0c3..67bdff9 100644 --- a/.github/integration/tests/s3notls/52_check_endpoint.sh +++ b/.github/integration/tests/s3notls/52_check_endpoint.sh @@ -88,7 +88,7 @@ fi curl -H "Authorization: Bearer $token" "http://localhost:8080/files/urn:neic:001-002?startCoordinate=0&endCoordinate=2" --output test-part.txt -echo "TH" > old-part.txt +dd if=old-file.txt ibs=1 skip=0 count=2 > old-part.txt cmp --silent old-part.txt test-part.txt status=$? @@ -96,11 +96,12 @@ if [[ $status = 0 ]]; then echo "Files are the same" else echo "Files are different" + exit 1 fi curl -H "Authorization: Bearer $token" "http://localhost:8080/files/urn:neic:001-002?startCoordinate=7&endCoordinate=14" --output test-part2.txt -echo "LE IS J" > old-part2.txt +dd if=old-file.txt ibs=1 skip=7 count=7 > old-part2.txt cmp --silent old-part2.txt test-part2.txt status=$? @@ -108,6 +109,7 @@ if [[ $status = 0 ]]; then echo "Files are the same" else echo "Files are different" + exit 1 fi # ------------------ diff --git a/api/sda/sda.go b/api/sda/sda.go index 4c5c215..2f7b137 100644 --- a/api/sda/sda.go +++ b/api/sda/sda.go @@ -255,6 +255,13 @@ func Download(c *gin.Context) { // used from: https://github.com/neicnordic/crypt4gh/blob/master/examples/reader/main.go#L48C1-L113C1 var sendStream = func(reader *streaming.Crypt4GHReader, writer http.ResponseWriter, start, end int64) error { + if start != 0 { + // We don't want to read from start, skip ahead to where we should be + if _, err := reader.Seek(start, 0); err != nil { + return err + } + } + // Calculate how much we should read (if given) togo := end - start diff --git a/go.mod b/go.mod index 720dbe8..264fa3b 100644 --- a/go.mod +++ b/go.mod @@ -8,14 +8,14 @@ require ( github.com/dgraph-io/ristretto v0.1.1 github.com/gin-gonic/gin v1.9.1 github.com/google/uuid v1.5.0 - github.com/johannesboyne/gofakes3 v0.0.0-20230129080941-f6a8a9ae6fd3 + github.com/johannesboyne/gofakes3 v0.0.0-20230914150226-f005f5cc03aa github.com/lestrrat-go/jwx/v2 v2.0.19 github.com/lib/pq v1.10.9 github.com/neicnordic/crypt4gh v1.8.8 github.com/sirupsen/logrus v1.9.3 github.com/spf13/viper v1.18.2 github.com/stretchr/testify v1.8.4 - golang.org/x/exp v0.0.0-20240110193028-0dcbfd608b1e + golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 ) require ( @@ -69,11 +69,11 @@ require ( go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect golang.org/x/arch v0.4.0 // indirect - golang.org/x/crypto v0.17.0 // indirect - golang.org/x/net v0.19.0 // indirect + golang.org/x/crypto v0.18.0 // indirect + golang.org/x/net v0.20.0 // indirect golang.org/x/sys v0.16.0 // indirect golang.org/x/text v0.14.0 // indirect - golang.org/x/tools v0.16.0 // indirect + golang.org/x/tools v0.17.0 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index 3150420..997e6f6 100644 --- a/go.sum +++ b/go.sum @@ -2,7 +2,7 @@ filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= -github.com/aws/aws-sdk-go v1.33.0/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= +github.com/aws/aws-sdk-go v1.44.256/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-sdk-go v1.49.21 h1:Rl8KW6HqkwzhATwvXhyr7vD4JFUMi7oXGAw9SrxxIFY= github.com/aws/aws-sdk-go v1.49.21/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= @@ -49,7 +49,6 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.15.2 h1:Ra5cll2/eF8X0Ff2+8SMD7euo2nenQ8WEpgqfy4NhHU= github.com/go-playground/validator/v10 v10.15.2/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= -github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -63,13 +62,12 @@ github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/johannesboyne/gofakes3 v0.0.0-20230129080941-f6a8a9ae6fd3 h1:aTscQmvmU/1AS3PqVaNtUtJUwyMexxqVErkhwsWoEpw= -github.com/johannesboyne/gofakes3 v0.0.0-20230129080941-f6a8a9ae6fd3/go.mod h1:Cnosl0cRZIfKjTMuH49sQog2LeNsU5Hf4WnPIDWIDV0= +github.com/johannesboyne/gofakes3 v0.0.0-20230914150226-f005f5cc03aa h1:a6Hc6Hlq6MxPNBW53/S/HnVwVXKc0nbdD/vgnQYuxG0= +github.com/johannesboyne/gofakes3 v0.0.0-20230914150226-f005f5cc03aa/go.mod h1:AxgWC4DDX54O2WDoQO1Ceabtn6IbktjU/7bigor+66g= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE= @@ -124,7 +122,6 @@ github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6g github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= -github.com/shabbyrobe/gocovmerge v0.0.0-20180507124511-f6ea450bfb63/go.mod h1:n+VKSARF5y/tS9XFSP7vWDfS+GUC5vs/YT7M5XDTUEM= github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500 h1:WnNuhiq+FOY3jNj6JXFT+eLN3CQ/oPIsDPRanvwsmbI= github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500/go.mod h1:+njLrG5wSeoG4Ds61rFgEzKvenR2UHbjMoDHsczxly0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= @@ -160,6 +157,7 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -169,32 +167,61 @@ golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUu golang.org/x/arch v0.4.0 h1:A8WCeEWhLwPBKNbFi5Wv5UTCBx5zzubnXDlMOFAzFMc= golang.org/x/arch v0.4.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= -golang.org/x/exp v0.0.0-20240110193028-0dcbfd608b1e h1:723BNChdd0c2Wk6WOE320qGBiPtYx0F0Bbm1kriShfE= -golang.org/x/exp v0.0.0-20240110193028-0dcbfd608b1e/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= +golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= +golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 h1:hNQpMuAJe5CtcUqCXaWga3FHu+kQvCqcsoVaQgSV60o= +golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= -golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= +golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/tools v0.0.0-20190308174544-00c44ba9c14f/go.mod h1:25r3+/G6/xytQM8iWZKq3Hn0kr0rgFKPUNVEL/dr3z4= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190829051458-42f498d34c4d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.16.0 h1:GO788SKMRunPIBCXiQyo2AaexLstOrVhuAL5YwsckQM= -golang.org/x/tools v0.16.0/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= +golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc= +golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=