diff --git a/README.md b/README.md index 6b5c6ebe..7822f9cc 100644 --- a/README.md +++ b/README.md @@ -257,6 +257,8 @@ go run ./cmd/... -c ~/credentials -l ./load_spec.json -i /Users/gopa.padmanabhan # recursive with file pattern regex match go run ./cmd/... -c ~/credentials -l ./load_spec.json -I /Users/gopa.padmanabhan/scratch/data/MET/tc_data/tc_data/ -r ".tcst" go run ./cmd/... -c ~/credentials.MET -d gopa01 -l ./load_spec.json -I /home/amb-verif/MET_data/tc_data/ -r ".tcst" +# load data from S3, transform, write back to S3 +go run ./cmd/... -c ~/credentials.MET -d gopa01 -l ./load_spec.json -b s3://metin/s3test.jar -B s3://metout/s3test.jar # if -f,-F,-i OR -I options are specified, ignores load_spec input files # update metadata diff --git a/cmd/metjson2db/main.go b/cmd/metjson2db/main.go index aaeaedb7..86df5d83 100644 --- a/cmd/metjson2db/main.go +++ b/cmd/metjson2db/main.go @@ -38,6 +38,12 @@ func main() { flag.StringVar(&inputFile, "f", "", "stat file full path") var inputFiles []string + // s3://bucket/path/to/file.zip (b/B for s3 bundle) + var s3inputFile string + flag.StringVar(&s3inputFile, "b", "", "s3 stat zip input path") + var s3outputFile string + flag.StringVar(&s3outputFile, "B", "", "s3 stat zip output path") + var inputFolder string flag.StringVar(&inputFolder, "i", "", "input stat files folder") @@ -99,6 +105,11 @@ func main() { return } + if len(s3inputFile) > 0 && len(s3outputFile) > 0 { + core.ProcessS3Files(s3inputFile, s3outputFile, nil) + return + } + if len(state.LoadSpec.DatasetName) > 10 { slog.Error("Dataset name must be less than 10 characters!") return diff --git a/docs/s3.doc b/docs/s3.doc new file mode 100644 index 00000000..29378102 --- /dev/null +++ b/docs/s3.doc @@ -0,0 +1,45 @@ +// from URL: +// https://stackoverflow.com/questions/49266516/reading-files-from-aws-s3-in-golang + + +go install github.com/seaweedfs/seaweedfs/weed@latest +go/bin/weed server -dir=~/scratch/ -s3 +https://github.com/seaweedfs/seaweedfs/wiki/s3cmd-with-SeaweedFS + +brew install localstack/tap/localstack-cli +brew install rancher +https://docs.localstack.cloud/aws/integrations/containers/rancher-desktop/ +ls /Applications/Rancher\ Desktop.app/Contents/Resources/resources/darwin/bin +/Applications/Rancher\ Desktop.app/Contents/Resources/resources/darwin/bin/rdctl shutdown +/Applications/Rancher\ Desktop.app/Contents/Resources/resources/darwin/bin/rdctl start + +localstack start +https://docs.localstack.cloud/aws/getting-started/quickstart/ +brew install awscli-local +brew install awscli + +--Create buckets +awslocal s3 mb s3://gopa0 +awslocal s3 mb s3://gopa1 +awslocal s3 mb s3://metin +awslocal s3 mb s3://metout +awslocal s3 ls +aws --profile localstack --endpoint-url=http://localhost:4566 s3 ls + +-- Add item to bucket +awslocal s3 cp /Users/gopa.padmanabhan/git/work/MATS.gopa/GoS3Test/test_data/test_data_0.txt s3://gopa0/test_data_0.txt +awslocal s3 cp /Users/gopa.padmanabhan/git/dtcenter/METjson2db/test_data/testTar0.tar s3://metin/testTar0.tar +awslocal s3 ls metin + +$cat ~/.aws/credentials +[localstack] +aws_access_key_id=test +aws_secret_access_key=test + +$export AWS_ACCESS_KEY_ID=test +$export AWS_SECRET_ACCESS_KEY=test +$export AWS_DEFAULT_REGION=us-east-1 + +go mod tidy +go run . + diff --git a/example.tar.gz b/example.tar.gz new file mode 100644 index 00000000..97675b3a Binary files /dev/null and b/example.tar.gz differ diff --git a/go.mod b/go.mod index 405d22f9..1b596b82 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,25 @@ require ( ) require ( + github.com/aws/aws-sdk-go v1.55.8 // indirect + github.com/aws/aws-sdk-go-v2 v1.39.6 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3 // indirect + github.com/aws/aws-sdk-go-v2/config v1.31.17 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.18.21 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.13 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.13 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.13 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.13 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.13 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.13 // indirect + github.com/aws/aws-sdk-go-v2/service/s3 v1.90.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.1 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.5 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.39.1 // indirect + github.com/aws/smithy-go v1.23.2 // indirect github.com/couchbase/gocbcore/v10 v10.7.0 // indirect github.com/couchbase/gocbcoreps v0.1.3 // indirect github.com/couchbase/goprotostellar v1.0.2 // indirect diff --git a/go.sum b/go.sum index 8e22fd99..5d281628 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,43 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ= +github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk= +github.com/aws/aws-sdk-go-v2 v1.39.6 h1:2JrPCVgWJm7bm83BDwY5z8ietmeJUbh3O2ACnn+Xsqk= +github.com/aws/aws-sdk-go-v2 v1.39.6/go.mod h1:c9pm7VwuW0UPxAEYGyTmyurVcNrbF6Rt/wixFqDhcjE= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3 h1:DHctwEM8P8iTXFxC/QK0MRjwEpWQeM9yzidCRjldUz0= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3/go.mod h1:xdCzcZEtnSTKVDOmUZs4l/j3pSV6rpo1WXl5ugNsL8Y= +github.com/aws/aws-sdk-go-v2/config v1.31.17 h1:QFl8lL6RgakNK86vusim14P2k8BFSxjvUkcWLDjgz9Y= +github.com/aws/aws-sdk-go-v2/config v1.31.17/go.mod h1:V8P7ILjp/Uef/aX8TjGk6OHZN6IKPM5YW6S78QnRD5c= +github.com/aws/aws-sdk-go-v2/credentials v1.18.21 h1:56HGpsgnmD+2/KpG0ikvvR8+3v3COCwaF4r+oWwOeNA= +github.com/aws/aws-sdk-go-v2/credentials v1.18.21/go.mod h1:3YELwedmQbw7cXNaII2Wywd+YY58AmLPwX4LzARgmmA= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.13 h1:T1brd5dR3/fzNFAQch/iBKeX07/ffu/cLu+q+RuzEWk= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.13/go.mod h1:Peg/GBAQ6JDt+RoBf4meB1wylmAipb7Kg2ZFakZTlwk= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.13 h1:a+8/MLcWlIxo1lF9xaGt3J/u3yOZx+CdSveSNwjhD40= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.13/go.mod h1:oGnKwIYZ4XttyU2JWxFrwvhF6YKiK/9/wmE3v3Iu9K8= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.13 h1:HBSI2kDkMdWz4ZM7FjwE7e/pWDEZ+nR95x8Ztet1ooY= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.13/go.mod h1:YE94ZoDArI7awZqJzBAZ3PDD2zSfuP7w6P2knOzIn8M= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.13 h1:eg/WYAa12vqTphzIdWMzqYRVKKnCboVPRlvaybNCqPA= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.13/go.mod h1:/FDdxWhz1486obGrKKC1HONd7krpk38LBt+dutLcN9k= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3 h1:x2Ibm/Af8Fi+BH+Hsn9TXGdT+hKbDd5XOTZxTMxDk7o= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3/go.mod h1:IW1jwyrQgMdhisceG8fQLmQIydcT/jWY21rFhzgaKwo= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.4 h1:NvMjwvv8hpGUILarKw7Z4Q0w1H9anXKsesMxtw++MA4= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.4/go.mod h1:455WPHSwaGj2waRSpQp7TsnpOnBfw8iDfPfbwl7KPJE= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.13 h1:kDqdFvMY4AtKoACfzIGD8A0+hbT41KTKF//gq7jITfM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.13/go.mod h1:lmKuogqSU3HzQCwZ9ZtcqOc5XGMqtDK7OIc2+DxiUEg= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.13 h1:zhBJXdhWIFZ1acfDYIhu4+LCzdUS2Vbcum7D01dXlHQ= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.13/go.mod h1:JaaOeCE368qn2Hzi3sEzY6FgAZVCIYcC2nwbro2QCh8= +github.com/aws/aws-sdk-go-v2/service/s3 v1.90.0 h1:ef6gIJR+xv/JQWwpa5FYirzoQctfSJm7tuDe3SZsUf8= +github.com/aws/aws-sdk-go-v2/service/s3 v1.90.0/go.mod h1:+wArOOrcHUevqdto9k1tKOF5++YTe9JEcPSc9Tx2ZSw= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.1 h1:0JPwLz1J+5lEOfy/g0SURC9cxhbQ1lIMHMa+AHZSzz0= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.1/go.mod h1:fKvyjJcz63iL/ftA6RaM8sRCtN4r4zl4tjL3qw5ec7k= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.5 h1:OWs0/j2UYR5LOGi88sD5/lhN6TDLG6SfA7CqsQO9zF0= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.5/go.mod h1:klO+ejMvYsB4QATfEOIXk8WAEwN4N0aBfJpvC+5SZBo= +github.com/aws/aws-sdk-go-v2/service/sts v1.39.1 h1:mLlUgHn02ue8whiR4BmxxGJLR2gwU6s6ZzJ5wDamBUs= +github.com/aws/aws-sdk-go-v2/service/sts v1.39.1/go.mod h1:E19xDjpzPZC7LS2knI9E6BaRFDK43Eul7vd6rSq2HWk= +github.com/aws/smithy-go v1.23.2 h1:Crv0eatJUQhaManss33hS5r40CG3ZFH+21XSkqMrIUM= +github.com/aws/smithy-go v1.23.2/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= diff --git a/load_spec.json b/load_spec.json index f6c8f0f4..c18e3655 100644 --- a/load_spec.json +++ b/load_spec.json @@ -8,7 +8,7 @@ "runMode0" : "CREATE_JSON_DOC_ARCHIVE", "runMode" : "DIRECT_LOAD_TO_DB", "runMode_doc" : ["DIRECT_LOAD_TO_DB", "CREATE_JSON_DOC_ARCHIVE", "METADATA_UPDATE"], - "jsonArchiveFilePathAndPrefix" :"/scratch/METjson2db_out_", + "jsonArchiveFilePathAndPrefix" :"/Users/gopa.padmanabhan/scratch/METjson2db_out_", "maxDocIdLength": 200, "progressiveLargeDocIdPolicies": [ "SHA256", "REJECT_WIH_ERROR_LOG"], "overWriteData": true, diff --git a/manifest.json b/manifest.json new file mode 100644 index 00000000..bc465554 --- /dev/null +++ b/manifest.json @@ -0,0 +1,21 @@ +{ + "timestamp": "2025-10-29T10:36:46.172154-06:00", + "files": [ + { + "name": "my_dir", + "path": "my_dir/", + "size": 0, + "mode": 493, + "modTime": "2025-10-29T10:36:46-06:00", + "isDir": true + }, + { + "name": "file1.txt", + "path": "my_dir/file1.txt", + "size": 11, + "mode": 420, + "modTime": "2025-10-29T10:36:46-06:00", + "isDir": false + } + ] +} \ No newline at end of file diff --git a/pkg/core/procesdInput.go b/pkg/core/procesdInput.go index 9d395afb..f238098e 100644 --- a/pkg/core/procesdInput.go +++ b/pkg/core/procesdInput.go @@ -116,7 +116,7 @@ func ProcessInputFiles(inputFiles []string, preDbLoadCallback func()) error { } case "CREATE_JSON_DOC_ARCHIVE": // home, _ := os.UserHomeDir() - err := parser.WriteJsonToCompressedFile(state.CbDocs, state.LoadSpec.JsonArchiveFilePathAndPrefix+time.Now().Format(time.RFC3339)) + err := parser.WriteJsonToCompressedFile(state.CbDocs, state.LoadSpec.JsonArchiveFilePathAndPrefix+time.Now().Format(time.RFC3339)+".gz") if err != nil { slog.Error("Expected no error, got:", slog.Any("error", err)) } diff --git a/pkg/core/processS3.go b/pkg/core/processS3.go new file mode 100644 index 00000000..812528d5 --- /dev/null +++ b/pkg/core/processS3.go @@ -0,0 +1,434 @@ +package core + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log" + "log/slog" + "net/url" + "os" + "path/filepath" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/smithy-go" + "github.com/dtcenter/METjson2db/pkg/state" +) + +// ManifestEntry represents a single entry in your manifest.json +type ManifestEntry struct { + Name string `json:"name"` + Path string `json:"path"` + Size int64 `json:"size"` + Mode os.FileMode `json:"mode"` + ModTime time.Time `json:"modTime"` + IsDir bool `json:"isDir"` +} + +// Manifest represents the overall structure of your manifest.json +type Manifest struct { + Timestamp time.Time `json:"timestamp"` + Files []ManifestEntry `json:"files"` +} + +// init runs before main() is evaluated +func init() { + slog.Debug("ProcessInput:init()") +} + +func ProcessS3Files(inputS3Path string, outputS3Path string, preDbLoadCallback func()) error { + slog.Info(fmt.Sprintf("ProcessInputFiles(%s,%s)", inputS3Path, outputS3Path)) + + // start := time.Now() + state.StateReset() + + _, s3client, err := iniS3Client() + if err != nil { + log.Fatalf("Unable to init s3 %v", err) + } + bucketIn, keyIn, _ := extractS3InfoFromS3Path(inputS3Path) + bucketOut, keyOut, _ := extractS3InfoFromS3Path(outputS3Path) + slog.Info(fmt.Sprintf("bucketIn=%s,keyIn=%s bucketOut=%s,keyOut=%s", bucketIn, keyIn, bucketOut, keyOut)) + + // Get the first page of results for ListObjectsV2 for a bucket + output, err := s3client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{ + Bucket: aws.String(bucketIn), + }) + if err != nil { + log.Fatal(err) + } + + log.Println("first page results") + for _, object := range output.Contents { + log.Printf("key=%s size=%d", aws.ToString(object.Key), *object.Size) + } + + result, err := s3client.GetObject(context.TODO(), &s3.GetObjectInput{ + Bucket: aws.String(bucketIn), + Key: aws.String(keyIn), + }) + + if err != nil { + var nsk *types.NoSuchKey + if errors.As(err, &nsk) { + // handle NoSuchKey error + return nil + } + var apiErr smithy.APIError + if errors.As(err, &apiErr) { + code := apiErr.ErrorCode() + message := apiErr.ErrorMessage() + log.Printf("key=%s message=%d", code, message) + return nil + } + // handle error + return nil + } + defer result.Body.Close() + + contentType := *result.ContentType + contentLength := *result.ContentLength + eTag := *result.ETag + + log.Printf("contentType=%s,contentLength=%d,eTag:%s", contentType, contentLength, eTag) + + body, err := io.ReadAll(result.Body) + _, err = io.ReadAll(result.Body) + if err != nil { + log.Fatalf("failed to read: %v", err) + } + + // Convert the byte slice to a string for printing + // fmt.Println(string(body)) + + /* + // Simulate a tar archive in a byte slice + // In a real scenario, this byte slice would come from a network, memory, etc. + tarData := createTarInBytes() + + // Write the JSON data to a file + err = os.WriteFile("testTar0.tar", tarData, 0644) + if err != nil { + log.Fatalf("Error writing testTar0.tar: %v\n", err) + } + + // Convert the byte slice to an io.Reader + tarReaderFromBytes := bytes.NewReader(tarData) + */ + + // Convert the byte slice to an io.Reader + tarReaderFromBytes := bytes.NewReader(body) + + // Create a tar.Reader + tarReader := tar.NewReader(tarReaderFromBytes) + + var buf bytes.Buffer + tarWriter := tar.NewWriter(&buf) + + manifest := &Manifest{ + Timestamp: time.Now(), + Files: []ManifestEntry{}, + } + + for { + header, err := tarReader.Next() + if err == io.EOF { + break // End of archive + } + if err != nil { + log.Fatalf("Error reading tar header: %v", err) + } + + log.Printf("header: %s", header.Name) + + entry := ManifestEntry{ + Name: filepath.Base(header.Name), + Path: header.Name, + Size: header.Size, + Mode: os.FileMode(header.Mode), + ModTime: header.ModTime, + IsDir: header.Typeflag == tar.TypeDir, + } + manifest.Files = append(manifest.Files, entry) + + switch header.Typeflag { + case tar.TypeDir: + log.Println("TypeDir, Mode:%v", header.Mode) + dirHeader := &tar.Header{ + Name: header.Name, + Mode: header.Mode, + Typeflag: tar.TypeDir, + } + if err := tarWriter.WriteHeader(dirHeader); err != nil { + log.Fatalf("Failed to write header for %s: %v", header.Name, err) + } + case tar.TypeReg: + log.Println("TypeReg:%v", header.Mode) + if header.Typeflag == tar.TypeReg { + fmt.Printf("--- Contents of %s ---\n", header.Name) + var buf bytes.Buffer + if _, err := io.Copy(&buf, tarReader); err != nil { + log.Fatalf("Error copying file content: %v", err) + } + fileContent := buf.String() + fmt.Println(fileContent) + fmt.Println("\n----------------------") + + file2Header := &tar.Header{ + Name: header.Name, + Mode: header.Mode, + Size: int64(len(fileContent)), + } + if err := tarWriter.WriteHeader(file2Header); err != nil { + log.Fatalf("Failed to write header for %s: %v", header.Name, err) + } + if _, err := tarWriter.Write([]byte(fileContent)); err != nil { + log.Fatalf("Failed to write content for %s: %v", header.Name, err) + } + + } + default: + fmt.Printf("Skipping unknown tar entry type: %s, %v\n", header.Name, header.Typeflag) + } + } + + // Marshal the manifest to JSON + jsonData, err := json.MarshalIndent(manifest, "", " ") + if err != nil { + fmt.Printf("Error marshaling manifest to JSON: %v\n", err) + return err + } + // Add manifest.json to tar + file2Header := &tar.Header{ + Name: "manifest.json", + Mode: 0644, + Size: int64(len(jsonData)), + } + if err := tarWriter.WriteHeader(file2Header); err != nil { + log.Fatalf("Failed to write header for manifest.json: %v", err) + } + if _, err := tarWriter.Write([]byte(jsonData)); err != nil { + log.Fatalf("Failed to write content for manifest.json: %v", err) + } + + // createTarManifestTest() + + if err := tarWriter.Close(); err != nil { + log.Fatalf("Failed to close tar writer: %v", err) + } + + // write modified tar out + input := &s3.PutObjectInput{ + Bucket: aws.String(bucketOut), + Key: aws.String(keyOut), + Body: bytes.NewReader(buf.Bytes()), + ChecksumAlgorithm: types.ChecksumAlgorithmSha256, + } + + _, err = s3client.PutObject(context.TODO(), input) + if err != nil { + log.Fatalf("Error in PutObject: %v", err) + } + + return nil +} + +func iniS3Client() (aws.Config, *s3.Client, error) { + cfg, err := config.LoadDefaultConfig(context.TODO(), + config.WithRegion("us-east-1"), + config.WithBaseEndpoint("http://localhost:4566"), + ) + if err != nil { + log.Fatalf("Unable to init s3: %v", err) + } + + s3client := s3.NewFromConfig(cfg, func(o *s3.Options) { + o.UsePathStyle = true + }) + return cfg, s3client, nil +} + +func extractS3InfoFromS3Path(s3URI string) (string, string, error) { + u, err := url.Parse(s3URI) + if err != nil { + fmt.Printf("Error parsing S3 URI: %v\n", err) + return "", "", err + } + + // Extract the bucket name + bucketName := u.Host + + // Extract the object key (path) + // The u.Path will include a leading slash, which is typically removed for S3 keys. + objectKey := strings.TrimPrefix(u.Path, "/") + + return bucketName, objectKey, nil +} + +func createTarManifestTest() error { + tarFilePath := "example.tar.gz" // Replace with your tar file path + manifestOutputPath := "manifest.json" + + // Create a dummy tar.gz file for demonstration + createDummyTarGz(tarFilePath) + + manifest, err := createManifestFromTar(tarFilePath) + if err != nil { + fmt.Printf("Error creating manifest: %v\n", err) + return err + } + + // Marshal the manifest to JSON + jsonData, err := json.MarshalIndent(manifest, "", " ") + if err != nil { + fmt.Printf("Error marshaling manifest to JSON: %v\n", err) + return err + } + + // Write the JSON data to a file + err = os.WriteFile(manifestOutputPath, jsonData, 0644) + if err != nil { + fmt.Printf("Error writing manifest.json: %v\n", err) + return err + } + + fmt.Printf("Manifest created successfully at %s\n", manifestOutputPath) + return nil +} + +func createManifestFromTar(tarFilePath string) (*Manifest, error) { + file, err := os.Open(tarFilePath) + if err != nil { + return nil, fmt.Errorf("failed to open tar file: %w", err) + } + defer file.Close() + + var reader io.Reader = file + + // Check if it's a gzipped tar file + if filepath.Ext(tarFilePath) == ".gz" { + gzipReader, err := gzip.NewReader(file) + if err != nil { + return nil, fmt.Errorf("failed to create gzip reader: %w", err) + } + defer gzipReader.Close() + reader = gzipReader + } + + tarReader := tar.NewReader(reader) + + manifest := &Manifest{ + Timestamp: time.Now(), + Files: []ManifestEntry{}, + } + + for { + header, err := tarReader.Next() + if err == io.EOF { + break // End of archive + } + if err != nil { + return nil, fmt.Errorf("failed to read tar entry: %w", err) + } + + entry := ManifestEntry{ + Name: filepath.Base(header.Name), + Path: header.Name, + Size: header.Size, + Mode: os.FileMode(header.Mode), + ModTime: header.ModTime, + IsDir: header.Typeflag == tar.TypeDir, + } + manifest.Files = append(manifest.Files, entry) + } + + return manifest, nil +} + +// createDummyTarGz creates a sample tar.gz file for testing +func createDummyTarGz(filename string) { + file, _ := os.Create(filename) + defer file.Close() + + gw := gzip.NewWriter(file) + defer gw.Close() + + tw := tar.NewWriter(gw) + defer tw.Close() + + // Add a directory + tw.WriteHeader(&tar.Header{ + Name: "my_dir/", + Mode: 0755, + ModTime: time.Now(), + Typeflag: tar.TypeDir, + }) + + // Add a file + header := &tar.Header{ + Name: "my_dir/file1.txt", + Mode: 0644, + Size: int64(len("hello world")), + ModTime: time.Now(), + } + tw.WriteHeader(header) + tw.Write([]byte("hello world")) +} + +// createTarInBytes creates a simple tar archive in a byte slice for demonstration +func createTarInBytes() []byte { + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + + // Add a file + header := &tar.Header{ + Name: "file1.txt", + Mode: 0644, + Size: int64(len("Hello from file1!")), + } + if err := tw.WriteHeader(header); err != nil { + log.Fatalf("Failed to write header for file1.txt: %v", err) + } + if _, err := tw.Write([]byte("Hello from file1!")); err != nil { + log.Fatalf("Failed to write content for file1.txt: %v", err) + } + + // Add a directory + dirHeader := &tar.Header{ + Name: "my_directory/", + Mode: 0755, + Typeflag: tar.TypeDir, + } + if err := tw.WriteHeader(dirHeader); err != nil { + log.Fatalf("Failed to write header for my_directory: %v", err) + } + + // Add another file inside the directory + file2Header := &tar.Header{ + Name: "my_directory/file2.txt", + Mode: 0644, + Size: int64(len("Content of file2.")), + } + if err := tw.WriteHeader(file2Header); err != nil { + log.Fatalf("Failed to write header for file2.txt: %v", err) + } + if _, err := tw.Write([]byte("Content of file2.")); err != nil { + log.Fatalf("Failed to write content for file2.txt: %v", err) + } + + if err := tw.Close(); err != nil { + log.Fatalf("Failed to close tar writer: %v", err) + } + + return buf.Bytes() +} diff --git a/test_data/testTar0.tar b/test_data/testTar0.tar new file mode 100644 index 00000000..88557807 Binary files /dev/null and b/test_data/testTar0.tar differ