Updated UserInfo method to accept a stats flag.

Updated integration tests to allow for uploading random file
to seed statistics, which unfortunately requires another dep.

Author: nathanejohnson

adminapi_integration_test.go

@@ -4,11 +4,20 @@ package radosgwadmin
 
 import (
 	"context"
+	"io"
 	"io/ioutil"
+	"math/rand"
 	"os"
 	"strconv"
 	"testing"
 
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+
 	"github.com/BurntSushi/toml"
 	"github.com/davecgh/go-spew/spew"
 	"github.com/stretchr/testify/suite"
@@ -19,7 +28,7 @@ type IntegrationsSuite struct {
 	aa           *AdminAPI
 	randFilePath string
 	lf           *os.File
-	i            *Integration
+	ic           *IntegrationConfig
 }
 
 type IntegrationConfig struct {
@@ -61,8 +70,9 @@ func (is *IntegrationsSuite) SetupSuite() {
 		is.T().Logf("cannot parse config file at location '%s' : %s", cfgFile, err)
 		os.Exit(1)
 	}
+
+	is.ic = cfg
 	is.aa, err = NewAdminAPI(cfg.RGW)
-	is.i = cfg.Integration
 	if err != nil {
 		is.T().Logf("Error initializing AdminAPI: %s", err)
 		os.Exit(1)
@@ -80,7 +90,7 @@ func (is *IntegrationsSuite) Test01Usage() {
 	usage, err := is.aa.Usage(context.Background(), nil)
 	is.NoError(err, "Got error getting Usage")
 	is.T().Logf("usage: %#v", usage)
-	err = is.aa.UsageTrim(context.Background(), &TrimUsageRequest{UID: is.i.TestUID})
+	err = is.aa.UsageTrim(context.Background(), &TrimUsageRequest{UID: is.ic.Integration.TestUID})
 	is.NoError(err, "Got error trimming usage")
 }
 
@@ -92,24 +102,24 @@ func (is *IntegrationsSuite) Test02Metadata() {
 
 func (is *IntegrationsSuite) Test03UserCreate() {
 	ur := new(UserCreateRequest)
-	ur.UID = is.i.TestUID
-	ur.Email = is.i.TestEmail
-	ur.DisplayName = is.i.TestDisplayName
+	ur.UID = is.ic.Integration.TestUID
+	ur.Email = is.ic.Integration.TestEmail
+	ur.DisplayName = is.ic.Integration.TestDisplayName
 	ur.UserCaps = []UserCap{{"users", "*"}, {"metadata", "*"}, {"buckets", "read"}}
 
 	resp, err := is.aa.UserCreate(context.Background(), ur)
 	is.NoError(err, "Got error running UserCreate")
 	is.T().Logf("%#v", resp)
 	sur := new(SubUserCreateModifyRequest)
-	sur.UID = is.i.TestUID
+	sur.UID = is.ic.Integration.TestUID
 	sur.Access = "full"
 	sur.KeyType = "s3"
-	sur.SubUser = is.i.TestSubUser
+	sur.SubUser = is.ic.Integration.TestSubUser
 	sur.GenerateSecret = true
 	nresp, err := is.aa.SubUserCreate(context.Background(), sur)
 	is.NoError(err)
 	is.T().Logf("%#v", nresp)
-	sur.SubUser = is.i.TesTSubUser2
+	sur.SubUser = is.ic.Integration.TesTSubUser2
 	sur.Access = "read"
 	nresp, err = is.aa.SubUserCreate(context.Background(), sur)
 	is.NoError(err)
@@ -120,21 +130,37 @@ func (is *IntegrationsSuite) Test04Quota() {
 	qsr := new(QuotaSetRequest)
 	qsr.Enabled = true
 	qsr.MaximumObjects = -1 // unlimited
-	qsr.MaximumSizeKb = 8192
+	qsr.MaximumSizeKb = 61440
 	qsr.QuotaType = "user"
-	qsr.UID = is.i.TestUID
+	qsr.UID = is.ic.Integration.TestUID
 	err := is.aa.QuotaSet(context.Background(), qsr)
 	is.NoError(err, "Got error running SetQuota")
 	// read it back
-	qresp, err := is.aa.QuotaUser(context.Background(), is.i.TestUID)
+	qresp, err := is.aa.QuotaUser(context.Background(), is.ic.Integration.TestUID)
 	is.T().Logf("%#v", qresp)
 	is.NoError(err, "Got error fetching user quota")
 	is.True(qresp.Enabled == true, "quota not enabled")
 	is.Equal(qresp.MaxObjects, int64(-1), "MaxObjects not -1")
-	is.Equal(qresp.MaxSizeKb, int64(8192), "MaxSizeKb not 8192")
+	is.Equal(qresp.MaxSizeKb, int64(61440), "MaxSizeKb not 61440")
 }
 
 func (is *IntegrationsSuite) Test05Bucket() {
+	// Write 50mb to it.
+	is.writeRandomFile("bigrandomfile.bin", 1024*10)
+	is.aa.BucketIndex(context.Background(), &BucketIndexRequest{
+		Bucket:       is.ic.Integration.TestBucket,
+		CheckObjects: true,
+		Fix:          true,
+	})
+
+	// see if we can now get stats.
+
+	ui, err := is.aa.UserInfo(context.Background(), is.ic.Integration.TestUID, true)
+
+	is.NoError(err)
+	is.NotNil(ui.Stats)
+
+	is.T().Logf("%#v", ui)
 	bucketnames, err := is.aa.BucketList(context.Background(), "")
 	is.NoError(err, "Got error fetching bucket names")
 	is.T().Logf("bucket names: %#v\n", bucketnames)
@@ -144,26 +170,25 @@ func (is *IntegrationsSuite) Test05Bucket() {
 	is.T().Log(spew.Sdump(bucketstats))
 
 	// bucketstats with bucket filter
-	bucketstatsf, err := is.aa.BucketStats(context.Background(), "", is.i.TestBucket)
+	bucketstatsf, err := is.aa.BucketStats(context.Background(), "", is.ic.Integration.TestBucket)
 	is.NoError(err, "got error fetching bucket stats filtered by bucket")
 	is.T().Log(spew.Sdump(bucketstatsf))
 
 	// TODO: - make code that creates a bucket and does stuff to test
 	// bucket index code. -- for now, do one I know already exists
 
 	bireq := &BucketIndexRequest{}
-	bireq.Bucket = is.i.TestBucket
+	bireq.Bucket = is.ic.Integration.TestBucket
 	bireq.CheckObjects = true
 	bireq.Fix = true
 	bucketindresp, err := is.aa.BucketIndex(context.Background(), bireq)
 	is.NoError(err, "Got error from BucketIndex()")
 	is.T().Logf(spew.Sdump(bucketindresp))
-
 }
 
 func (is *IntegrationsSuite) Test06Caps() {
 	ucr := &UserCapsRequest{
-		UID:      is.i.TestUID,
+		UID:      is.ic.Integration.TestUID,
 		UserCaps: []UserCap{{"usage", "read"}},
 	}
 	newcaps, err := is.aa.CapsAdd(context.Background(), ucr)
@@ -202,15 +227,14 @@ func (is *IntegrationsSuite) Test06Caps() {
 		}
 	}
 	is.Equal(goodct, 2, "not expected removal of perms")
-
 }
 
 func (is *IntegrationsSuite) Test07Keys() {
 	accessKey := "TESTACCESSKEY"
 	secretKey := "TESTSECRETKEY"
 	generateKey := false
 	kc := &KeyCreateRequest{
-		UID:         is.i.TestUID,
+		UID:         is.ic.Integration.TestUID,
 		AccessKey:   accessKey,
 		SecretKey:   secretKey,
 		GenerateKey: &generateKey,
@@ -236,7 +260,7 @@ func (is *IntegrationsSuite) Test07Keys() {
 	is.NoError(err1, "Unexpected error deleting key")
 
 	// check if the key was really deleted
-	userInfo, err2 := is.aa.UserInfo(context.Background(), is.i.TestUID)
+	userInfo, err2 := is.aa.UserInfo(context.Background(), is.ic.Integration.TestUID, false)
 	is.NoError(err2, "Unexpected error reading user info")
 	found = false
 	for i := range userInfo.Keys {
@@ -257,20 +281,111 @@ func (is *IntegrationsSuite) Test08RmUser() {
 			return
 		}
 	}
-	err := is.aa.UserRm(context.Background(), is.i.TestUID, true)
+	err := is.aa.UserRm(context.Background(), is.ic.Integration.TestUID, true)
 	is.NoError(err, "got error removing user")
 	users, err := is.aa.MListUsers(context.Background())
 	is.NoError(err, "got error listing users")
 	found := false
 	for _, user := range users {
-		if user == is.i.TestUID {
+		if user == is.ic.Integration.TestUID {
 			found = true
 			break
 		}
 	}
 	is.False(found, "user not successfully deleted")
 }
 
+func (is *IntegrationsSuite) writeRandomFile(path string, sizekb int) {
+	var (
+		s3c   *s3.S3
+		err   error
+		ui    *UserInfoResponse
+		creds *credentials.Credentials
+	)
+
+	// grab creds for subuser s3 user
+	ui, err = is.aa.UserInfo(context.Background(), is.ic.Integration.TestUID, false)
+
+	is.NoError(err)
+	for _, k := range ui.Keys {
+		if k.User == is.ic.Integration.TestUID {
+			creds = credentials.NewStaticCredentials(k.AccessKey, k.SecretKey, "")
+			break
+		}
+	}
+
+	is.NotNil(creds, "credentials not found")
+	if creds == nil {
+		return
+	}
+	cfg := aws.NewConfig()
+	cfg.Region = aws.String("us-east")
+	cfg.Endpoint = aws.String(is.ic.RGW.ServerURL)
+	cfg.WithS3ForcePathStyle(true)
+
+	cfg.Credentials = creds
+	sess := session.Must(session.NewSession(cfg))
+	s3c = s3.New(sess)
+
+	_, err = s3c.CreateBucket(&s3.CreateBucketInput{
+		Bucket: aws.String(is.ic.Integration.TestBucket),
+		CreateBucketConfiguration: &s3.CreateBucketConfiguration{
+			LocationConstraint: aws.String(""),
+		},
+	})
+	if err != nil {
+
+		// ignore non-fatal creation errors
+		if awsErr, ok := err.(awserr.Error); ok {
+			if awsErr.Code() != s3.ErrCodeBucketAlreadyExists &&
+				awsErr.Code() != s3.ErrCodeBucketAlreadyOwnedByYou {
+				is.FailNow(err.Error())
+			}
+		} else {
+			is.FailNow(err.Error())
+		}
+	}
+
+	rr := NewRandoReader(0, int64(1024*sizekb))
+
+	uploader := s3manager.NewUploader(sess)
+
+	_, err = uploader.Upload(&s3manager.UploadInput{
+		Bucket: aws.String(is.ic.Integration.TestBucket),
+		Key:    aws.String(path),
+		Body:   rr,
+	})
+	if err != nil {
+		is.Fail(err.Error())
+	}
+
+}
+
+type RandoReader struct {
+	r          *rand.Rand
+	max        int64
+	read       int64
+}
+
+func NewRandoReader(seed int64, bytes int64) *RandoReader {
+	return &RandoReader{
+		r:    rand.New(rand.NewSource(seed)),
+		max:  bytes,
+		read: 0,
+	}
+}
+
+func (rr *RandoReader) Read(p []byte) (n int, err error) {
+	if rr.read >= rr.max {
+		return 0, io.EOF
+	} else if rr.max < (rr.read + int64(len(p))) {
+		p = p[:(rr.max - rr.read)]
+	}
+	c, err := rr.r.Read(p)
+	rr.read += int64(c)
+	return c, err
+}
+
 func TestIntegrations(t *testing.T) {
 	suite.Run(t, new(IntegrationsSuite))
 }

adminapi_test.go

@@ -180,6 +180,20 @@ func (ms *ModelsSuite) Test05Quotas() {
 	ms.Equal(resp.UserQuota.MaxSizeKb, int64(-1), "Value not expected")
 }
 
+func (ms *ModelsSuite) Test06User() {
+	userjson := ms.dbags["user"]
+	userstatjson := ms.dbags["userstat"]
+	resp := &UserInfoResponse{}
+	err := json.Unmarshal(userjson, resp)
+	ms.NoError(err, "Error unmarshaling user json")
+	ms.Nil(resp.Stats, "stats not nil as expected")
+	resp = &UserInfoResponse{}
+	err = json.Unmarshal(userstatjson, resp)
+	ms.NoError(err, "Error unmsrshaling userstat json")
+	ms.NotNil(resp.Stats, "userstat Stats is nil when it shouldn't be")
+
+}
+
 func TestAdminAPI(t *testing.T) {
 	suite.Run(t, new(ModelsSuite))
 }

glide.lock

@@ -2,8 +2,12 @@ package: github.com/myENA/radosgwadmin
 import:
 - package: github.com/BurntSushi/toml
   version: ^0.2.0
+- package: github.com/google/go-querystring
+- package: gopkg.in/go-playground/validator.v9
+  version: ^9.6.0
 - package: github.com/smartystreets/go-aws-auth
+testImport:
 - package: github.com/stretchr/testify
   version: ^1.1.4
-- package: github.com/google/go-querystring
-- package: gopkg.in/go-playground/validator.v9
+- package: github.com/aws/aws-sdk-go
+  version: ^1.10.38

glide.yaml

@@ -12,3 +12,4 @@
     AdminPath = "admin"
     AccessKeyID = "ABC123BLAHBLAHBLAH"
     SecretAccessKey = "IMASUPERSECRETACCESSKEY"
+    DisableSSL = true