From 2b6402df86ae7ed9de28c39dff35045cc3ad90c9 Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Fri, 17 Mar 2017 16:51:00 -0700 Subject: [PATCH 01/69] Add option to start DB test as Docker container --- dbtest/dbserver.go | 74 +++++++++++++++++++++++++++++++++++++++------- 1 file changed, 63 insertions(+), 11 deletions(-) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index 16b7b5841..1eb85f3f7 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -13,6 +13,13 @@ import ( "gopkg.in/tomb.v2" ) +// Constants to define how the DB test instance should be executed +type ExecType int +const ( + LocalProcess ExecType = 0 // Run MongoDB as local process + Docker ExecType = 1 // Run MongoDB within a docker container +) + // DBServer controls a MongoDB server process to be used within test suites. // // The test server is started when Session is called the first time and should @@ -28,6 +35,8 @@ type DBServer struct { server *exec.Cmd dbpath string host string + version string // The request MongoDB version, when running within a container + eType ExecType tomb tomb.Tomb } @@ -38,6 +47,51 @@ func (dbs *DBServer) SetPath(dbpath string) { dbs.dbpath = dbpath } +// SetVersion defines the desired MongoDB version to run within a container. +// The attribute is ignored when running MongoDB outside a container. +func (dbs *DBServer) SetVersion(version string) { + dbs.version = version +} + +// SetExecType specifies if the DB instance should run locally or as a container. +func (dbs *DBServer) SetExecType(execType ExecType) { + dbs.eType = execType +} + +// Start Mongo DB within Docker container on host. +// It assumes Docker is already installed +func (dbs *DBServer) execContainer(port int) { + if dbs.version == "" { + dbs.version = "latest" + } + args := []string{ + "run", + "-p", + fmt.Sprintf("%d:%d", port, 27017), + "--rm", // Automatically remove the container when it exits + "--hostname", + dbs.host, + "--name", + dbs.host, + fmt.Sprintf("mongo:%s", dbs.version), + } + return exec.Command("docker", args...) +} + +// Start Mongo DB as process on host. It assumes Mongo is already installed +func (dbs *DBServer) execLocal(port int) (*exec.Cmd) { + args := []string{ + "--dbpath", dbs.dbpath, + "--bind_ip", "127.0.0.1", + "--port", strconv.Itoa(port), + "--nssize", "1", + "--noprealloc", + "--smallfiles", + "--nojournal", + } + return exec.Command("mongod", args...) +} + func (dbs *DBServer) start() { if dbs.server != nil { panic("DBServer already started") @@ -53,18 +107,16 @@ func (dbs *DBServer) start() { addr := l.Addr().(*net.TCPAddr) l.Close() dbs.host = addr.String() - - args := []string{ - "--dbpath", dbs.dbpath, - "--bind_ip", "127.0.0.1", - "--port", strconv.Itoa(addr.Port), - "--nssize", "1", - "--noprealloc", - "--smallfiles", - "--nojournal", - } + dbs.tomb = tomb.Tomb{} - dbs.server = exec.Command("mongod", args...) + switch dbs.eType { + case LocalProcess: + dbs.server = execLocal(addr.Port) + case Docker: + dbs.server = execContainer(addr.Port) + default: + panic("unsupported exec type") + } dbs.server.Stdout = &dbs.output dbs.server.Stderr = &dbs.output err = dbs.server.Start() From e54a35098b1aed1c1735c8056296c350822481a5 Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Fri, 17 Mar 2017 17:45:36 -0700 Subject: [PATCH 02/69] add unit test --- dbtest/dbserver.go | 17 +++++++++-------- dbtest/dbserver_test.go | 12 ++++++++++++ 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index 1eb85f3f7..198141177 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -14,10 +14,11 @@ import ( ) // Constants to define how the DB test instance should be executed -type ExecType int const ( - LocalProcess ExecType = 0 // Run MongoDB as local process - Docker ExecType = 1 // Run MongoDB within a docker container + // Run MongoDB as local process + LocalProcess = "local" + // Run MongoDB within a docker container + Docker = "docker" ) // DBServer controls a MongoDB server process to be used within test suites. @@ -36,7 +37,7 @@ type DBServer struct { dbpath string host string version string // The request MongoDB version, when running within a container - eType ExecType + eType string tomb tomb.Tomb } @@ -54,13 +55,13 @@ func (dbs *DBServer) SetVersion(version string) { } // SetExecType specifies if the DB instance should run locally or as a container. -func (dbs *DBServer) SetExecType(execType ExecType) { +func (dbs *DBServer) SetExecType(execType string) { dbs.eType = execType } // Start Mongo DB within Docker container on host. // It assumes Docker is already installed -func (dbs *DBServer) execContainer(port int) { +func (dbs *DBServer) execContainer(port int) (*exec.Cmd) { if dbs.version == "" { dbs.version = "latest" } @@ -111,9 +112,9 @@ func (dbs *DBServer) start() { dbs.tomb = tomb.Tomb{} switch dbs.eType { case LocalProcess: - dbs.server = execLocal(addr.Port) + dbs.server = dbs.execLocal(addr.Port) case Docker: - dbs.server = execContainer(addr.Port) + dbs.server = dbs.execContainer(addr.Port) default: panic("unsupported exec type") } diff --git a/dbtest/dbserver_test.go b/dbtest/dbserver_test.go index 79812fde3..5b7f290d2 100644 --- a/dbtest/dbserver_test.go +++ b/dbtest/dbserver_test.go @@ -32,6 +32,18 @@ func (s *S) TearDownTest(c *C) { os.Setenv("CHECK_SESSIONS", s.oldCheckSessions) } +func (s *S) TestRunAsDocker(c *C) { + var server dbtest.DBServer + server.SetPath(c.MkDir()) + server.SetExecType(dbtest.LocalProcess) + defer server.Stop() + + session := server.Session() + err := session.DB("mydb").C("mycoll").Insert(M{"a": 1}) + session.Close() + c.Assert(err, IsNil) +} + func (s *S) TestWipeData(c *C) { var server dbtest.DBServer server.SetPath(c.MkDir()) From 6c8a117498b3668bbc409c98a20a0a273dd5748f Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Fri, 17 Mar 2017 22:22:56 -0700 Subject: [PATCH 03/69] change go imports --- .travis.yml | 2 +- auth.go | 4 ++-- auth_test.go | 2 +- bson/bson_test.go | 2 +- bson/decimal_test.go | 2 +- bson/json.go | 2 +- bson/json_test.go | 2 +- bulk.go | 2 +- bulk_test.go | 2 +- cluster.go | 2 +- cluster_test.go | 4 ++-- dbtest/dbserver.go | 2 +- dbtest/dbserver_test.go | 4 ++-- gridfs.go | 2 +- gridfs_test.go | 4 ++-- internal/scram/scram_test.go | 2 +- saslimpl.go | 2 +- server.go | 2 +- session.go | 2 +- session_test.go | 4 ++-- socket.go | 2 +- suite_test.go | 4 ++-- txn/debug.go | 2 +- txn/flusher.go | 4 ++-- txn/sim_test.go | 8 ++++---- txn/tarjan.go | 2 +- txn/tarjan_test.go | 2 +- txn/txn.go | 4 ++-- txn/txn_test.go | 8 ++++---- 29 files changed, 43 insertions(+), 43 deletions(-) diff --git a/.travis.yml b/.travis.yml index 45b38cf13..a946a619d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ language: go -go_import_path: gopkg.in/mgo.v2 +go_import_path: gopkg.in/CiscoM31/mgo.v2 addons: apt: diff --git a/auth.go b/auth.go index dc26e52f5..3a16ec403 100644 --- a/auth.go +++ b/auth.go @@ -34,8 +34,8 @@ import ( "fmt" "sync" - "gopkg.in/mgo.v2/bson" - "gopkg.in/mgo.v2/internal/scram" + "gopkg.in/CiscoM31/mgo.v2/bson" + "gopkg.in/CiscoM31/mgo.v2/internal/scram" ) type authCmd struct { diff --git a/auth_test.go b/auth_test.go index 995273475..2e8d6d3eb 100644 --- a/auth_test.go +++ b/auth_test.go @@ -39,7 +39,7 @@ import ( "time" . "gopkg.in/check.v1" - "gopkg.in/mgo.v2" + "gopkg.in/CiscoM31/mgo.v2" ) func (s *S) TestAuthLoginDatabase(c *C) { diff --git a/bson/bson_test.go b/bson/bson_test.go index 37451f9fd..b999c73e3 100644 --- a/bson/bson_test.go +++ b/bson/bson_test.go @@ -40,7 +40,7 @@ import ( "time" . "gopkg.in/check.v1" - "gopkg.in/mgo.v2/bson" + "gopkg.in/CiscoM31/mgo.v2/bson" "gopkg.in/yaml.v2" ) diff --git a/bson/decimal_test.go b/bson/decimal_test.go index a29728094..ee346aa13 100644 --- a/bson/decimal_test.go +++ b/bson/decimal_test.go @@ -33,7 +33,7 @@ import ( "regexp" "strings" - "gopkg.in/mgo.v2/bson" + "gopkg.in/CiscoM31/mgo.v2/bson" . "gopkg.in/check.v1" ) diff --git a/bson/json.go b/bson/json.go index 09df8260a..255c22338 100644 --- a/bson/json.go +++ b/bson/json.go @@ -4,7 +4,7 @@ import ( "bytes" "encoding/base64" "fmt" - "gopkg.in/mgo.v2/internal/json" + "gopkg.in/CiscoM31/mgo.v2/internal/json" "strconv" "time" ) diff --git a/bson/json_test.go b/bson/json_test.go index 866f51c34..c68c89286 100644 --- a/bson/json_test.go +++ b/bson/json_test.go @@ -1,7 +1,7 @@ package bson_test import ( - "gopkg.in/mgo.v2/bson" + "gopkg.in/CiscoM31/mgo.v2/bson" . "gopkg.in/check.v1" "reflect" diff --git a/bulk.go b/bulk.go index 072a5206a..35b870c24 100644 --- a/bulk.go +++ b/bulk.go @@ -4,7 +4,7 @@ import ( "bytes" "sort" - "gopkg.in/mgo.v2/bson" + "gopkg.in/CiscoM31/mgo.v2/bson" ) // Bulk represents an operation that can be prepared with several diff --git a/bulk_test.go b/bulk_test.go index cb280bbfa..bbab1c97e 100644 --- a/bulk_test.go +++ b/bulk_test.go @@ -28,7 +28,7 @@ package mgo_test import ( . "gopkg.in/check.v1" - "gopkg.in/mgo.v2" + "gopkg.in/CiscoM31/mgo.v2" ) func (s *S) TestBulkInsert(c *C) { diff --git a/cluster.go b/cluster.go index c3bf8b013..574ac7b8e 100644 --- a/cluster.go +++ b/cluster.go @@ -35,7 +35,7 @@ import ( "sync" "time" - "gopkg.in/mgo.v2/bson" + "gopkg.in/CiscoM31/mgo.v2/bson" ) // --------------------------------------------------------------------------- diff --git a/cluster_test.go b/cluster_test.go index 54ec86762..beb0dc1d8 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -35,8 +35,8 @@ import ( "time" . "gopkg.in/check.v1" - "gopkg.in/mgo.v2" - "gopkg.in/mgo.v2/bson" + "gopkg.in/CiscoM31/mgo.v2" + "gopkg.in/CiscoM31/mgo.v2/bson" ) func (s *S) TestNewSession(c *C) { diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index 198141177..f2809ca3d 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -9,7 +9,7 @@ import ( "strconv" "time" - "gopkg.in/mgo.v2" + "gopkg.in/CiscoM31/mgo.v2" "gopkg.in/tomb.v2" ) diff --git a/dbtest/dbserver_test.go b/dbtest/dbserver_test.go index 5b7f290d2..f4f80b282 100644 --- a/dbtest/dbserver_test.go +++ b/dbtest/dbserver_test.go @@ -7,8 +7,8 @@ import ( . "gopkg.in/check.v1" - "gopkg.in/mgo.v2" - "gopkg.in/mgo.v2/dbtest" + "gopkg.in/CiscoM31/mgo.v2" + "gopkg.in/CiscoM31/mgo.v2/dbtest" ) type M map[string]interface{} diff --git a/gridfs.go b/gridfs.go index 421472095..2bc3c0286 100644 --- a/gridfs.go +++ b/gridfs.go @@ -36,7 +36,7 @@ import ( "sync" "time" - "gopkg.in/mgo.v2/bson" + "gopkg.in/CiscoM31/mgo.v2/bson" ) type GridFS struct { diff --git a/gridfs_test.go b/gridfs_test.go index 5a6ed5559..2aa90fd7b 100644 --- a/gridfs_test.go +++ b/gridfs_test.go @@ -32,8 +32,8 @@ import ( "time" . "gopkg.in/check.v1" - "gopkg.in/mgo.v2" - "gopkg.in/mgo.v2/bson" + "gopkg.in/CiscoM31/mgo.v2" + "gopkg.in/CiscoM31/mgo.v2/bson" ) func (s *S) TestGridFSCreate(c *C) { diff --git a/internal/scram/scram_test.go b/internal/scram/scram_test.go index 9c20fdfc4..79b11e2f1 100644 --- a/internal/scram/scram_test.go +++ b/internal/scram/scram_test.go @@ -5,7 +5,7 @@ import ( "testing" . "gopkg.in/check.v1" - "gopkg.in/mgo.v2/internal/scram" + "gopkg.in/CiscoM31/mgo.v2/internal/scram" "strings" ) diff --git a/saslimpl.go b/saslimpl.go index 0d25f25cb..20665767b 100644 --- a/saslimpl.go +++ b/saslimpl.go @@ -3,7 +3,7 @@ package mgo import ( - "gopkg.in/mgo.v2/internal/sasl" + "gopkg.in/CiscoM31/mgo.v2/internal/sasl" ) func saslNew(cred Credential, host string) (saslStepper, error) { diff --git a/server.go b/server.go index 392598691..86d1f7776 100644 --- a/server.go +++ b/server.go @@ -33,7 +33,7 @@ import ( "sync" "time" - "gopkg.in/mgo.v2/bson" + "gopkg.in/CiscoM31/mgo.v2/bson" ) // --------------------------------------------------------------------------- diff --git a/session.go b/session.go index 3dccf364e..75100145f 100644 --- a/session.go +++ b/session.go @@ -41,7 +41,7 @@ import ( "sync" "time" - "gopkg.in/mgo.v2/bson" + "gopkg.in/CiscoM31/mgo.v2/bson" ) type Mode int diff --git a/session_test.go b/session_test.go index 492f21078..5ba4350d1 100644 --- a/session_test.go +++ b/session_test.go @@ -38,8 +38,8 @@ import ( "time" . "gopkg.in/check.v1" - "gopkg.in/mgo.v2" - "gopkg.in/mgo.v2/bson" + "gopkg.in/CiscoM31/mgo.v2" + "gopkg.in/CiscoM31/mgo.v2/bson" ) func (s *S) TestRunString(c *C) { diff --git a/socket.go b/socket.go index 8891dd5d7..a1a14c3b0 100644 --- a/socket.go +++ b/socket.go @@ -33,7 +33,7 @@ import ( "sync" "time" - "gopkg.in/mgo.v2/bson" + "gopkg.in/CiscoM31/mgo.v2/bson" ) type replyFunc func(err error, reply *replyOp, docNum int, docData []byte) diff --git a/suite_test.go b/suite_test.go index bac5d3f4a..94edfa868 100644 --- a/suite_test.go +++ b/suite_test.go @@ -39,8 +39,8 @@ import ( "time" . "gopkg.in/check.v1" - "gopkg.in/mgo.v2" - "gopkg.in/mgo.v2/bson" + "gopkg.in/CiscoM31/mgo.v2" + "gopkg.in/CiscoM31/mgo.v2/bson" ) var fast = flag.Bool("fast", false, "Skip slow tests") diff --git a/txn/debug.go b/txn/debug.go index 8224bb313..72634a51d 100644 --- a/txn/debug.go +++ b/txn/debug.go @@ -6,7 +6,7 @@ import ( "sort" "sync/atomic" - "gopkg.in/mgo.v2/bson" + "gopkg.in/CiscoM31/mgo.v2/bson" ) var ( diff --git a/txn/flusher.go b/txn/flusher.go index f640a4380..1b7d9b681 100644 --- a/txn/flusher.go +++ b/txn/flusher.go @@ -3,8 +3,8 @@ package txn import ( "fmt" - "gopkg.in/mgo.v2" - "gopkg.in/mgo.v2/bson" + "gopkg.in/CiscoM31/mgo.v2" + "gopkg.in/CiscoM31/mgo.v2/bson" ) func flush(r *Runner, t *transaction) error { diff --git a/txn/sim_test.go b/txn/sim_test.go index a369ded7c..8a39fe0d8 100644 --- a/txn/sim_test.go +++ b/txn/sim_test.go @@ -2,10 +2,10 @@ package txn_test import ( "flag" - "gopkg.in/mgo.v2" - "gopkg.in/mgo.v2/bson" - "gopkg.in/mgo.v2/dbtest" - "gopkg.in/mgo.v2/txn" + "gopkg.in/CiscoM31/mgo.v2" + "gopkg.in/CiscoM31/mgo.v2/bson" + "gopkg.in/CiscoM31/mgo.v2/dbtest" + "gopkg.in/CiscoM31/mgo.v2/txn" . "gopkg.in/check.v1" "math/rand" "time" diff --git a/txn/tarjan.go b/txn/tarjan.go index e56541c9b..3e7f8e0a4 100644 --- a/txn/tarjan.go +++ b/txn/tarjan.go @@ -1,7 +1,7 @@ package txn import ( - "gopkg.in/mgo.v2/bson" + "gopkg.in/CiscoM31/mgo.v2/bson" "sort" ) diff --git a/txn/tarjan_test.go b/txn/tarjan_test.go index 79745c39b..31fe5f584 100644 --- a/txn/tarjan_test.go +++ b/txn/tarjan_test.go @@ -2,7 +2,7 @@ package txn import ( "fmt" - "gopkg.in/mgo.v2/bson" + "gopkg.in/CiscoM31/mgo.v2/bson" . "gopkg.in/check.v1" ) diff --git a/txn/txn.go b/txn/txn.go index 204b3cf1d..ea2fa4199 100644 --- a/txn/txn.go +++ b/txn/txn.go @@ -14,8 +14,8 @@ import ( "strings" "sync" - "gopkg.in/mgo.v2" - "gopkg.in/mgo.v2/bson" + "gopkg.in/CiscoM31/mgo.v2" + "gopkg.in/CiscoM31/mgo.v2/bson" crand "crypto/rand" mrand "math/rand" diff --git a/txn/txn_test.go b/txn/txn_test.go index 12923ca12..f6b4a63c9 100644 --- a/txn/txn_test.go +++ b/txn/txn_test.go @@ -8,10 +8,10 @@ import ( "time" . "gopkg.in/check.v1" - "gopkg.in/mgo.v2" - "gopkg.in/mgo.v2/bson" - "gopkg.in/mgo.v2/dbtest" - "gopkg.in/mgo.v2/txn" + "gopkg.in/CiscoM31/mgo.v2" + "gopkg.in/CiscoM31/mgo.v2/bson" + "gopkg.in/CiscoM31/mgo.v2/dbtest" + "gopkg.in/CiscoM31/mgo.v2/txn" ) func TestAll(t *testing.T) { From 512995d8d8261c49cc7434f4cfd83f2eb11a9d7d Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Fri, 17 Mar 2017 22:29:22 -0700 Subject: [PATCH 04/69] change type to integer --- dbtest/dbserver.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index f2809ca3d..3d18c87b0 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -16,9 +16,9 @@ import ( // Constants to define how the DB test instance should be executed const ( // Run MongoDB as local process - LocalProcess = "local" + LocalProcess = 0 // Run MongoDB within a docker container - Docker = "docker" + Docker = 1 ) // DBServer controls a MongoDB server process to be used within test suites. @@ -37,7 +37,7 @@ type DBServer struct { dbpath string host string version string // The request MongoDB version, when running within a container - eType string + eType int tomb tomb.Tomb } @@ -55,7 +55,7 @@ func (dbs *DBServer) SetVersion(version string) { } // SetExecType specifies if the DB instance should run locally or as a container. -func (dbs *DBServer) SetExecType(execType string) { +func (dbs *DBServer) SetExecType(execType int) { dbs.eType = execType } @@ -116,7 +116,7 @@ func (dbs *DBServer) start() { case Docker: dbs.server = dbs.execContainer(addr.Port) default: - panic("unsupported exec type") + panic(fmt.Sprintf("unsupported exec type: %d", dbs.eType)) } dbs.server.Stdout = &dbs.output dbs.server.Stderr = &dbs.output From 94e118f3560bc17305da94c1021d445cc7970c85 Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Fri, 17 Mar 2017 22:37:05 -0700 Subject: [PATCH 05/69] remove hostname and name arguments --- dbtest/dbserver.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index 3d18c87b0..fcc5917df 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -70,10 +70,6 @@ func (dbs *DBServer) execContainer(port int) (*exec.Cmd) { "-p", fmt.Sprintf("%d:%d", port, 27017), "--rm", // Automatically remove the container when it exits - "--hostname", - dbs.host, - "--name", - dbs.host, fmt.Sprintf("mongo:%s", dbs.version), } return exec.Command("docker", args...) From c141f81b69c9782dfb2c320699e9bf3843ecc6d9 Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Sat, 18 Mar 2017 07:38:19 -0700 Subject: [PATCH 06/69] change import back to mgo.v2 --- .travis.yml | 2 +- auth.go | 4 ++-- auth_test.go | 2 +- bson/bson_test.go | 2 +- bson/decimal_test.go | 2 +- bson/json.go | 2 +- bson/json_test.go | 2 +- bulk.go | 2 +- bulk_test.go | 2 +- cluster.go | 2 +- cluster_test.go | 4 ++-- dbtest/dbserver.go | 2 +- dbtest/dbserver_test.go | 4 ++-- gridfs.go | 2 +- gridfs_test.go | 4 ++-- internal/scram/scram_test.go | 2 +- saslimpl.go | 2 +- server.go | 2 +- session.go | 2 +- session_test.go | 4 ++-- socket.go | 2 +- suite_test.go | 4 ++-- txn/debug.go | 2 +- txn/flusher.go | 4 ++-- txn/sim_test.go | 8 ++++---- txn/tarjan.go | 2 +- txn/tarjan_test.go | 2 +- txn/txn.go | 4 ++-- txn/txn_test.go | 8 ++++---- 29 files changed, 43 insertions(+), 43 deletions(-) diff --git a/.travis.yml b/.travis.yml index a946a619d..45b38cf13 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ language: go -go_import_path: gopkg.in/CiscoM31/mgo.v2 +go_import_path: gopkg.in/mgo.v2 addons: apt: diff --git a/auth.go b/auth.go index 3a16ec403..dc26e52f5 100644 --- a/auth.go +++ b/auth.go @@ -34,8 +34,8 @@ import ( "fmt" "sync" - "gopkg.in/CiscoM31/mgo.v2/bson" - "gopkg.in/CiscoM31/mgo.v2/internal/scram" + "gopkg.in/mgo.v2/bson" + "gopkg.in/mgo.v2/internal/scram" ) type authCmd struct { diff --git a/auth_test.go b/auth_test.go index 2e8d6d3eb..995273475 100644 --- a/auth_test.go +++ b/auth_test.go @@ -39,7 +39,7 @@ import ( "time" . "gopkg.in/check.v1" - "gopkg.in/CiscoM31/mgo.v2" + "gopkg.in/mgo.v2" ) func (s *S) TestAuthLoginDatabase(c *C) { diff --git a/bson/bson_test.go b/bson/bson_test.go index b999c73e3..37451f9fd 100644 --- a/bson/bson_test.go +++ b/bson/bson_test.go @@ -40,7 +40,7 @@ import ( "time" . "gopkg.in/check.v1" - "gopkg.in/CiscoM31/mgo.v2/bson" + "gopkg.in/mgo.v2/bson" "gopkg.in/yaml.v2" ) diff --git a/bson/decimal_test.go b/bson/decimal_test.go index ee346aa13..a29728094 100644 --- a/bson/decimal_test.go +++ b/bson/decimal_test.go @@ -33,7 +33,7 @@ import ( "regexp" "strings" - "gopkg.in/CiscoM31/mgo.v2/bson" + "gopkg.in/mgo.v2/bson" . "gopkg.in/check.v1" ) diff --git a/bson/json.go b/bson/json.go index 255c22338..09df8260a 100644 --- a/bson/json.go +++ b/bson/json.go @@ -4,7 +4,7 @@ import ( "bytes" "encoding/base64" "fmt" - "gopkg.in/CiscoM31/mgo.v2/internal/json" + "gopkg.in/mgo.v2/internal/json" "strconv" "time" ) diff --git a/bson/json_test.go b/bson/json_test.go index c68c89286..866f51c34 100644 --- a/bson/json_test.go +++ b/bson/json_test.go @@ -1,7 +1,7 @@ package bson_test import ( - "gopkg.in/CiscoM31/mgo.v2/bson" + "gopkg.in/mgo.v2/bson" . "gopkg.in/check.v1" "reflect" diff --git a/bulk.go b/bulk.go index 35b870c24..072a5206a 100644 --- a/bulk.go +++ b/bulk.go @@ -4,7 +4,7 @@ import ( "bytes" "sort" - "gopkg.in/CiscoM31/mgo.v2/bson" + "gopkg.in/mgo.v2/bson" ) // Bulk represents an operation that can be prepared with several diff --git a/bulk_test.go b/bulk_test.go index bbab1c97e..cb280bbfa 100644 --- a/bulk_test.go +++ b/bulk_test.go @@ -28,7 +28,7 @@ package mgo_test import ( . "gopkg.in/check.v1" - "gopkg.in/CiscoM31/mgo.v2" + "gopkg.in/mgo.v2" ) func (s *S) TestBulkInsert(c *C) { diff --git a/cluster.go b/cluster.go index 574ac7b8e..c3bf8b013 100644 --- a/cluster.go +++ b/cluster.go @@ -35,7 +35,7 @@ import ( "sync" "time" - "gopkg.in/CiscoM31/mgo.v2/bson" + "gopkg.in/mgo.v2/bson" ) // --------------------------------------------------------------------------- diff --git a/cluster_test.go b/cluster_test.go index beb0dc1d8..54ec86762 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -35,8 +35,8 @@ import ( "time" . "gopkg.in/check.v1" - "gopkg.in/CiscoM31/mgo.v2" - "gopkg.in/CiscoM31/mgo.v2/bson" + "gopkg.in/mgo.v2" + "gopkg.in/mgo.v2/bson" ) func (s *S) TestNewSession(c *C) { diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index fcc5917df..1df83ae5e 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -9,7 +9,7 @@ import ( "strconv" "time" - "gopkg.in/CiscoM31/mgo.v2" + "gopkg.in/mgo.v2" "gopkg.in/tomb.v2" ) diff --git a/dbtest/dbserver_test.go b/dbtest/dbserver_test.go index f4f80b282..5b7f290d2 100644 --- a/dbtest/dbserver_test.go +++ b/dbtest/dbserver_test.go @@ -7,8 +7,8 @@ import ( . "gopkg.in/check.v1" - "gopkg.in/CiscoM31/mgo.v2" - "gopkg.in/CiscoM31/mgo.v2/dbtest" + "gopkg.in/mgo.v2" + "gopkg.in/mgo.v2/dbtest" ) type M map[string]interface{} diff --git a/gridfs.go b/gridfs.go index 2bc3c0286..421472095 100644 --- a/gridfs.go +++ b/gridfs.go @@ -36,7 +36,7 @@ import ( "sync" "time" - "gopkg.in/CiscoM31/mgo.v2/bson" + "gopkg.in/mgo.v2/bson" ) type GridFS struct { diff --git a/gridfs_test.go b/gridfs_test.go index 2aa90fd7b..5a6ed5559 100644 --- a/gridfs_test.go +++ b/gridfs_test.go @@ -32,8 +32,8 @@ import ( "time" . "gopkg.in/check.v1" - "gopkg.in/CiscoM31/mgo.v2" - "gopkg.in/CiscoM31/mgo.v2/bson" + "gopkg.in/mgo.v2" + "gopkg.in/mgo.v2/bson" ) func (s *S) TestGridFSCreate(c *C) { diff --git a/internal/scram/scram_test.go b/internal/scram/scram_test.go index 79b11e2f1..9c20fdfc4 100644 --- a/internal/scram/scram_test.go +++ b/internal/scram/scram_test.go @@ -5,7 +5,7 @@ import ( "testing" . "gopkg.in/check.v1" - "gopkg.in/CiscoM31/mgo.v2/internal/scram" + "gopkg.in/mgo.v2/internal/scram" "strings" ) diff --git a/saslimpl.go b/saslimpl.go index 20665767b..0d25f25cb 100644 --- a/saslimpl.go +++ b/saslimpl.go @@ -3,7 +3,7 @@ package mgo import ( - "gopkg.in/CiscoM31/mgo.v2/internal/sasl" + "gopkg.in/mgo.v2/internal/sasl" ) func saslNew(cred Credential, host string) (saslStepper, error) { diff --git a/server.go b/server.go index 86d1f7776..392598691 100644 --- a/server.go +++ b/server.go @@ -33,7 +33,7 @@ import ( "sync" "time" - "gopkg.in/CiscoM31/mgo.v2/bson" + "gopkg.in/mgo.v2/bson" ) // --------------------------------------------------------------------------- diff --git a/session.go b/session.go index 75100145f..3dccf364e 100644 --- a/session.go +++ b/session.go @@ -41,7 +41,7 @@ import ( "sync" "time" - "gopkg.in/CiscoM31/mgo.v2/bson" + "gopkg.in/mgo.v2/bson" ) type Mode int diff --git a/session_test.go b/session_test.go index 5ba4350d1..492f21078 100644 --- a/session_test.go +++ b/session_test.go @@ -38,8 +38,8 @@ import ( "time" . "gopkg.in/check.v1" - "gopkg.in/CiscoM31/mgo.v2" - "gopkg.in/CiscoM31/mgo.v2/bson" + "gopkg.in/mgo.v2" + "gopkg.in/mgo.v2/bson" ) func (s *S) TestRunString(c *C) { diff --git a/socket.go b/socket.go index a1a14c3b0..8891dd5d7 100644 --- a/socket.go +++ b/socket.go @@ -33,7 +33,7 @@ import ( "sync" "time" - "gopkg.in/CiscoM31/mgo.v2/bson" + "gopkg.in/mgo.v2/bson" ) type replyFunc func(err error, reply *replyOp, docNum int, docData []byte) diff --git a/suite_test.go b/suite_test.go index 94edfa868..bac5d3f4a 100644 --- a/suite_test.go +++ b/suite_test.go @@ -39,8 +39,8 @@ import ( "time" . "gopkg.in/check.v1" - "gopkg.in/CiscoM31/mgo.v2" - "gopkg.in/CiscoM31/mgo.v2/bson" + "gopkg.in/mgo.v2" + "gopkg.in/mgo.v2/bson" ) var fast = flag.Bool("fast", false, "Skip slow tests") diff --git a/txn/debug.go b/txn/debug.go index 72634a51d..8224bb313 100644 --- a/txn/debug.go +++ b/txn/debug.go @@ -6,7 +6,7 @@ import ( "sort" "sync/atomic" - "gopkg.in/CiscoM31/mgo.v2/bson" + "gopkg.in/mgo.v2/bson" ) var ( diff --git a/txn/flusher.go b/txn/flusher.go index 1b7d9b681..f640a4380 100644 --- a/txn/flusher.go +++ b/txn/flusher.go @@ -3,8 +3,8 @@ package txn import ( "fmt" - "gopkg.in/CiscoM31/mgo.v2" - "gopkg.in/CiscoM31/mgo.v2/bson" + "gopkg.in/mgo.v2" + "gopkg.in/mgo.v2/bson" ) func flush(r *Runner, t *transaction) error { diff --git a/txn/sim_test.go b/txn/sim_test.go index 8a39fe0d8..a369ded7c 100644 --- a/txn/sim_test.go +++ b/txn/sim_test.go @@ -2,10 +2,10 @@ package txn_test import ( "flag" - "gopkg.in/CiscoM31/mgo.v2" - "gopkg.in/CiscoM31/mgo.v2/bson" - "gopkg.in/CiscoM31/mgo.v2/dbtest" - "gopkg.in/CiscoM31/mgo.v2/txn" + "gopkg.in/mgo.v2" + "gopkg.in/mgo.v2/bson" + "gopkg.in/mgo.v2/dbtest" + "gopkg.in/mgo.v2/txn" . "gopkg.in/check.v1" "math/rand" "time" diff --git a/txn/tarjan.go b/txn/tarjan.go index 3e7f8e0a4..e56541c9b 100644 --- a/txn/tarjan.go +++ b/txn/tarjan.go @@ -1,7 +1,7 @@ package txn import ( - "gopkg.in/CiscoM31/mgo.v2/bson" + "gopkg.in/mgo.v2/bson" "sort" ) diff --git a/txn/tarjan_test.go b/txn/tarjan_test.go index 31fe5f584..79745c39b 100644 --- a/txn/tarjan_test.go +++ b/txn/tarjan_test.go @@ -2,7 +2,7 @@ package txn import ( "fmt" - "gopkg.in/CiscoM31/mgo.v2/bson" + "gopkg.in/mgo.v2/bson" . "gopkg.in/check.v1" ) diff --git a/txn/txn.go b/txn/txn.go index ea2fa4199..204b3cf1d 100644 --- a/txn/txn.go +++ b/txn/txn.go @@ -14,8 +14,8 @@ import ( "strings" "sync" - "gopkg.in/CiscoM31/mgo.v2" - "gopkg.in/CiscoM31/mgo.v2/bson" + "gopkg.in/mgo.v2" + "gopkg.in/mgo.v2/bson" crand "crypto/rand" mrand "math/rand" diff --git a/txn/txn_test.go b/txn/txn_test.go index f6b4a63c9..12923ca12 100644 --- a/txn/txn_test.go +++ b/txn/txn_test.go @@ -8,10 +8,10 @@ import ( "time" . "gopkg.in/check.v1" - "gopkg.in/CiscoM31/mgo.v2" - "gopkg.in/CiscoM31/mgo.v2/bson" - "gopkg.in/CiscoM31/mgo.v2/dbtest" - "gopkg.in/CiscoM31/mgo.v2/txn" + "gopkg.in/mgo.v2" + "gopkg.in/mgo.v2/bson" + "gopkg.in/mgo.v2/dbtest" + "gopkg.in/mgo.v2/txn" ) func TestAll(t *testing.T) { From c427782963fd8e75a60b3124159ef41d874467d0 Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Sat, 18 Mar 2017 08:30:22 -0700 Subject: [PATCH 07/69] specify mongo version in UT, verify running version matches requested version --- dbtest/dbserver_test.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/dbtest/dbserver_test.go b/dbtest/dbserver_test.go index 5b7f290d2..029e875cf 100644 --- a/dbtest/dbserver_test.go +++ b/dbtest/dbserver_test.go @@ -35,11 +35,16 @@ func (s *S) TearDownTest(c *C) { func (s *S) TestRunAsDocker(c *C) { var server dbtest.DBServer server.SetPath(c.MkDir()) - server.SetExecType(dbtest.LocalProcess) + server.SetVersion("3.4") + server.SetExecType(dbtest.Docker) defer server.Stop() session := server.Session() err := session.DB("mydb").C("mycoll").Insert(M{"a": 1}) + buildInfo, err := session.BuildInfo() + c.Assert(err, IsNil) + c.Assert(buildInfo.VersionAtLeast(3, 4), Equals, true) + session.Close() c.Assert(err, IsNil) } From 0ed34a291a38f2610163e2d02caae67fd420a2ab Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Fri, 24 Mar 2017 07:35:53 -0700 Subject: [PATCH 08/69] Invoke 'docker pull' before 'docker run' to avoid dial problems --- dbtest/dbserver.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index 1df83ae5e..a553399c3 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -65,7 +65,19 @@ func (dbs *DBServer) execContainer(port int) (*exec.Cmd) { if dbs.version == "" { dbs.version = "latest" } + // It may take a long time to download the mongo image if the docker image is not installed. + // Execute 'docker pull' now to pull the image before executing it. Otherwise Dial() may fail + // with a timeout after 10 seconds. args := []string{ + "pull", + fmt.Sprintf("mongo:%s", dbs.version), + } + cmd := exec.Command("docker", args...) + err := cmd.Run() + if err != nil { + panic(err) + } + args = []string{ "run", "-p", fmt.Sprintf("%d:%d", port, 27017), @@ -181,8 +193,10 @@ func (dbs *DBServer) Session() *mgo.Session { if dbs.session == nil { mgo.ResetStats() var err error - dbs.session, err = mgo.Dial(dbs.host + "/test") + dbs.session, err = mgo.DialWithTimeout(dbs.host + "/test", 10 * time.Second) if err != nil { + fmt.Fprintf(os.Stderr, "---- Unable to dial mongod:\n") + fmt.Fprintf(os.Stderr, "%s", dbs.output.Bytes()) panic(err) } } From 039a97817c11ae080dc9f261f105e39b859715fd Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Fri, 24 Mar 2017 09:00:45 -0700 Subject: [PATCH 09/69] Add debug statements --- dbtest/dbserver.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index a553399c3..31646be14 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -37,7 +37,8 @@ type DBServer struct { dbpath string host string version string // The request MongoDB version, when running within a container - eType int + eType int // Specify whether mongo should run as a container or regular process + debug bool // Log debug statements tomb tomb.Tomb } @@ -54,6 +55,10 @@ func (dbs *DBServer) SetVersion(version string) { dbs.version = version } +func (dbs *DBServer) SetDebug(enableDebug bool) { + dbs.debug = enableDebug +} + // SetExecType specifies if the DB instance should run locally or as a container. func (dbs *DBServer) SetExecType(execType int) { dbs.eType = execType @@ -73,10 +78,12 @@ func (dbs *DBServer) execContainer(port int) (*exec.Cmd) { fmt.Sprintf("mongo:%s", dbs.version), } cmd := exec.Command("docker", args...) + if dbs.debug { fmt.Printf("Pulling Mongo docker image\n") } err := cmd.Run() if err != nil { panic(err) } + if dbs.debug { fmt.Printf("Pulled Mongo docker image\n") } args = []string{ "run", "-p", @@ -128,6 +135,7 @@ func (dbs *DBServer) start() { } dbs.server.Stdout = &dbs.output dbs.server.Stderr = &dbs.output + if dbs.debug { fmt.Printf("Starting Mongo instance: %v\n", dbs.server.Args) } err = dbs.server.Start() if err != nil { panic(err) From 2eb66053ccef85ba4941267304381ebabdd1d5c7 Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Fri, 24 Mar 2017 12:26:41 -0700 Subject: [PATCH 10/69] Allocate pseudo tty to address chown permission denied on some platforms --- dbtest/dbserver.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index 31646be14..14719e71b 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -84,8 +84,11 @@ func (dbs *DBServer) execContainer(port int) (*exec.Cmd) { panic(err) } if dbs.debug { fmt.Printf("Pulled Mongo docker image\n") } + // On some platforms, we get "chown: changing ownership of '/proc/1/fd/1': Permission denied" unless + // we allocate a pseudo tty (-t option) args = []string{ "run", + "-t", "-p", fmt.Sprintf("%d:%d", port, 27017), "--rm", // Automatically remove the container when it exits From c757253923017f2b8639a933d1c8284640102254 Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Thu, 30 Mar 2017 19:32:01 -0700 Subject: [PATCH 11/69] Invoke 'docker stop' to stop container when test has completed --- dbtest/dbserver.go | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index 14719e71b..e6c7ccbe1 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -2,7 +2,9 @@ package dbtest import ( "bytes" + "encoding/hex" "fmt" + "math/rand" "net" "os" "os/exec" @@ -39,6 +41,7 @@ type DBServer struct { version string // The request MongoDB version, when running within a container eType int // Specify whether mongo should run as a container or regular process debug bool // Log debug statements + containerName string // The container name, when running mgo within a container tomb tomb.Tomb } @@ -84,6 +87,16 @@ func (dbs *DBServer) execContainer(port int) (*exec.Cmd) { panic(err) } if dbs.debug { fmt.Printf("Pulled Mongo docker image\n") } + + // Generate a name for the container. This will help to inspect the container + // and get the Mongo PID. + u := make([]byte, 8) + _, err = rand.Read(u) + if err != nil { + panic(err) + } + dbs.containerName = fmt.Sprintf("%s", hex.EncodeToString(u)) + // On some platforms, we get "chown: changing ownership of '/proc/1/fd/1': Permission denied" unless // we allocate a pseudo tty (-t option) args = []string{ @@ -92,11 +105,30 @@ func (dbs *DBServer) execContainer(port int) (*exec.Cmd) { "-p", fmt.Sprintf("%d:%d", port, 27017), "--rm", // Automatically remove the container when it exits + "--name", + dbs.containerName, fmt.Sprintf("mongo:%s", dbs.version), + "--nssize", "1", + "--noprealloc", + "--smallfiles", + "--nojournal", } return exec.Command("docker", args...) } +// Stop the docker container running Mongo. +func (dbs *DBServer) stopContainer() { + args := []string{ + "stop", + dbs.containerName, + } + cmd := exec.Command("docker", args...) + err := cmd.Run() + if err != nil { + panic(err) + } +} + // Start Mongo DB as process on host. It assumes Mongo is already installed func (dbs *DBServer) execLocal(port int) (*exec.Cmd) { args := []string{ @@ -182,6 +214,10 @@ func (dbs *DBServer) Stop() { } } if dbs.server != nil { + if dbs.eType == Docker { + // Invoke 'docker stop' + dbs.stopContainer() + } dbs.tomb.Kill(nil) dbs.server.Process.Signal(os.Interrupt) select { From 99f3ecbea0898e8696c0d724f06beb2088de2b09 Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Thu, 30 Mar 2017 19:41:27 -0700 Subject: [PATCH 12/69] Invoke 'docker rm' to delete the container when test has completed --- dbtest/dbserver.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index e6c7ccbe1..c60542281 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -127,6 +127,15 @@ func (dbs *DBServer) stopContainer() { if err != nil { panic(err) } + // Remove the container and its unamed volume. + // In some cases the "docker run --rm" option does not remove the container. + args = []string{ + "rm", + "-v", + dbs.containerName, + } + cmd = exec.Command("docker", args...) + err = cmd.Run() } // Start Mongo DB as process on host. It assumes Mongo is already installed From 6d2e8d9814073ad10b8994a9b0ced6b0c4baf9d5 Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Thu, 30 Mar 2017 23:49:01 -0700 Subject: [PATCH 13/69] Set random seed to prevent container name collisions --- dbtest/dbserver.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index c60542281..fd71ef921 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -91,7 +91,10 @@ func (dbs *DBServer) execContainer(port int) (*exec.Cmd) { // Generate a name for the container. This will help to inspect the container // and get the Mongo PID. u := make([]byte, 8) - _, err = rand.Read(u) + // The default number generator is deterministic. + s := rand.NewSource(time.Now().UnixNano()) + r := rand.New(s) + _, err = r.Read(u) if err != nil { panic(err) } From 8de861e9e05e704bf7e0125de553fb48d57af662 Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Fri, 31 Mar 2017 09:11:43 -0700 Subject: [PATCH 14/69] Run gofmt --- dbtest/dbserver.go | 106 +++++++++++++++++++++------------------- dbtest/dbserver_test.go | 6 +-- 2 files changed, 59 insertions(+), 53 deletions(-) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index fd71ef921..fd6cec69f 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -2,9 +2,9 @@ package dbtest import ( "bytes" - "encoding/hex" + "encoding/hex" "fmt" - "math/rand" + "math/rand" "net" "os" "os/exec" @@ -18,9 +18,9 @@ import ( // Constants to define how the DB test instance should be executed const ( // Run MongoDB as local process - LocalProcess = 0 + LocalProcess = 0 // Run MongoDB within a docker container - Docker = 1 + Docker = 1 ) // DBServer controls a MongoDB server process to be used within test suites. @@ -33,16 +33,16 @@ const ( // Before the DBServer is used the SetPath method must be called to define // the location for the database files to be stored. type DBServer struct { - session *mgo.Session - output bytes.Buffer - server *exec.Cmd - dbpath string - host string - version string // The request MongoDB version, when running within a container - eType int // Specify whether mongo should run as a container or regular process - debug bool // Log debug statements - containerName string // The container name, when running mgo within a container - tomb tomb.Tomb + session *mgo.Session + output bytes.Buffer + server *exec.Cmd + dbpath string + host string + version string // The request MongoDB version, when running within a container + eType int // Specify whether mongo should run as a container or regular process + debug bool // Log debug statements + containerName string // The container name, when running mgo within a container + tomb tomb.Tomb } // SetPath defines the path to the directory where the database files will be @@ -55,7 +55,7 @@ func (dbs *DBServer) SetPath(dbpath string) { // SetVersion defines the desired MongoDB version to run within a container. // The attribute is ignored when running MongoDB outside a container. func (dbs *DBServer) SetVersion(version string) { - dbs.version = version + dbs.version = version } func (dbs *DBServer) SetDebug(enableDebug bool) { @@ -69,7 +69,7 @@ func (dbs *DBServer) SetExecType(execType int) { // Start Mongo DB within Docker container on host. // It assumes Docker is already installed -func (dbs *DBServer) execContainer(port int) (*exec.Cmd) { +func (dbs *DBServer) execContainer(port int) *exec.Cmd { if dbs.version == "" { dbs.version = "latest" } @@ -81,24 +81,28 @@ func (dbs *DBServer) execContainer(port int) (*exec.Cmd) { fmt.Sprintf("mongo:%s", dbs.version), } cmd := exec.Command("docker", args...) - if dbs.debug { fmt.Printf("Pulling Mongo docker image\n") } + if dbs.debug { + fmt.Printf("Pulling Mongo docker image\n") + } err := cmd.Run() - if err != nil { - panic(err) - } - if dbs.debug { fmt.Printf("Pulled Mongo docker image\n") } + if err != nil { + panic(err) + } + if dbs.debug { + fmt.Printf("Pulled Mongo docker image\n") + } - // Generate a name for the container. This will help to inspect the container - // and get the Mongo PID. - u := make([]byte, 8) - // The default number generator is deterministic. - s := rand.NewSource(time.Now().UnixNano()) - r := rand.New(s) - _, err = r.Read(u) - if err != nil { - panic(err) - } - dbs.containerName = fmt.Sprintf("%s", hex.EncodeToString(u)) + // Generate a name for the container. This will help to inspect the container + // and get the Mongo PID. + u := make([]byte, 8) + // The default number generator is deterministic. + s := rand.NewSource(time.Now().UnixNano()) + r := rand.New(s) + _, err = r.Read(u) + if err != nil { + panic(err) + } + dbs.containerName = fmt.Sprintf("%s", hex.EncodeToString(u)) // On some platforms, we get "chown: changing ownership of '/proc/1/fd/1': Permission denied" unless // we allocate a pseudo tty (-t option) @@ -108,8 +112,8 @@ func (dbs *DBServer) execContainer(port int) (*exec.Cmd) { "-p", fmt.Sprintf("%d:%d", port, 27017), "--rm", // Automatically remove the container when it exits - "--name", - dbs.containerName, + "--name", + dbs.containerName, fmt.Sprintf("mongo:%s", dbs.version), "--nssize", "1", "--noprealloc", @@ -123,26 +127,26 @@ func (dbs *DBServer) execContainer(port int) (*exec.Cmd) { func (dbs *DBServer) stopContainer() { args := []string{ "stop", - dbs.containerName, + dbs.containerName, } cmd := exec.Command("docker", args...) err := cmd.Run() - if err != nil { - panic(err) - } - // Remove the container and its unamed volume. - // In some cases the "docker run --rm" option does not remove the container. + if err != nil { + panic(err) + } + // Remove the container and its unamed volume. + // In some cases the "docker run --rm" option does not remove the container. args = []string{ "rm", - "-v", - dbs.containerName, + "-v", + dbs.containerName, } cmd = exec.Command("docker", args...) err = cmd.Run() } // Start Mongo DB as process on host. It assumes Mongo is already installed -func (dbs *DBServer) execLocal(port int) (*exec.Cmd) { +func (dbs *DBServer) execLocal(port int) *exec.Cmd { args := []string{ "--dbpath", dbs.dbpath, "--bind_ip", "127.0.0.1", @@ -170,7 +174,7 @@ func (dbs *DBServer) start() { addr := l.Addr().(*net.TCPAddr) l.Close() dbs.host = addr.String() - + dbs.tomb = tomb.Tomb{} switch dbs.eType { case LocalProcess: @@ -182,7 +186,9 @@ func (dbs *DBServer) start() { } dbs.server.Stdout = &dbs.output dbs.server.Stderr = &dbs.output - if dbs.debug { fmt.Printf("Starting Mongo instance: %v\n", dbs.server.Args) } + if dbs.debug { + fmt.Printf("Starting Mongo instance: %v\n", dbs.server.Args) + } err = dbs.server.Start() if err != nil { panic(err) @@ -226,10 +232,10 @@ func (dbs *DBServer) Stop() { } } if dbs.server != nil { - if dbs.eType == Docker { - // Invoke 'docker stop' - dbs.stopContainer() - } + if dbs.eType == Docker { + // Invoke 'docker stop' + dbs.stopContainer() + } dbs.tomb.Kill(nil) dbs.server.Process.Signal(os.Interrupt) select { @@ -252,7 +258,7 @@ func (dbs *DBServer) Session() *mgo.Session { if dbs.session == nil { mgo.ResetStats() var err error - dbs.session, err = mgo.DialWithTimeout(dbs.host + "/test", 10 * time.Second) + dbs.session, err = mgo.DialWithTimeout(dbs.host+"/test", 10*time.Second) if err != nil { fmt.Fprintf(os.Stderr, "---- Unable to dial mongod:\n") fmt.Fprintf(os.Stderr, "%s", dbs.output.Bytes()) diff --git a/dbtest/dbserver_test.go b/dbtest/dbserver_test.go index 029e875cf..67d4b186f 100644 --- a/dbtest/dbserver_test.go +++ b/dbtest/dbserver_test.go @@ -41,9 +41,9 @@ func (s *S) TestRunAsDocker(c *C) { session := server.Session() err := session.DB("mydb").C("mycoll").Insert(M{"a": 1}) - buildInfo, err := session.BuildInfo() - c.Assert(err, IsNil) - c.Assert(buildInfo.VersionAtLeast(3, 4), Equals, true) + buildInfo, err := session.BuildInfo() + c.Assert(err, IsNil) + c.Assert(buildInfo.VersionAtLeast(3, 4), Equals, true) session.Close() c.Assert(err, IsNil) From 45272df24a84e8e1b7570767c022e7b84578d502 Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Mon, 10 Apr 2017 10:14:55 -0700 Subject: [PATCH 15/69] Fix race condition on shutdown --- dbtest/dbserver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index fd6cec69f..3acee825f 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -232,11 +232,11 @@ func (dbs *DBServer) Stop() { } } if dbs.server != nil { + dbs.tomb.Kill(nil) if dbs.eType == Docker { // Invoke 'docker stop' dbs.stopContainer() } - dbs.tomb.Kill(nil) dbs.server.Process.Signal(os.Interrupt) select { case <-dbs.tomb.Dead(): From 0b386697c4089c9ab8434ba9fd083d73e680b7aa Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Thu, 13 Apr 2017 10:33:43 -0700 Subject: [PATCH 16/69] Add helper function to return the host name of the DB test instance. When the test instance runs as a container, returns the container name --- dbtest/dbserver.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index 3acee825f..9ab28711f 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -123,6 +123,21 @@ func (dbs *DBServer) execContainer(port int) *exec.Cmd { return exec.Command("docker", args...) } +// Returns the host name of the Mongo test instance. +// If the test instance runs as a container, it returns the container name. +// If the test instance runs in the host, returns the host name. +func (dbs *DBServer) getHostName() { + if dbs.eType == Docker { + return dbs.containerName + } else { + if hostname, err := os.Hostname(); err != nil { + return hostname + } else { + return "127.0.0.1" + } + } +} + // Stop the docker container running Mongo. func (dbs *DBServer) stopContainer() { args := []string{ From 6439679dd40a0ee8feb708927d8036e954ec09fb Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Thu, 13 Apr 2017 10:38:04 -0700 Subject: [PATCH 17/69] Add helper function to return the host name of the DB test instance. When the test instance runs as a container, returns the container name --- dbtest/dbserver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index 9ab28711f..fb79828af 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -126,7 +126,7 @@ func (dbs *DBServer) execContainer(port int) *exec.Cmd { // Returns the host name of the Mongo test instance. // If the test instance runs as a container, it returns the container name. // If the test instance runs in the host, returns the host name. -func (dbs *DBServer) getHostName() { +func (dbs *DBServer) GetHostName() { if dbs.eType == Docker { return dbs.containerName } else { From c67d9adce26aa39455f29435bb830d5678ed210b Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Thu, 13 Apr 2017 10:41:57 -0700 Subject: [PATCH 18/69] Add helper function to return the host name of the DB test instance. When the test instance runs as a container, returns the container name --- dbtest/dbserver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index fb79828af..c9bb558ff 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -126,7 +126,7 @@ func (dbs *DBServer) execContainer(port int) *exec.Cmd { // Returns the host name of the Mongo test instance. // If the test instance runs as a container, it returns the container name. // If the test instance runs in the host, returns the host name. -func (dbs *DBServer) GetHostName() { +func (dbs *DBServer) GetHostName() string { if dbs.eType == Docker { return dbs.containerName } else { From 163faccc8570d3ed45e6f2949ef19314f32152ab Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Wed, 19 Apr 2017 18:58:07 -0700 Subject: [PATCH 19/69] Print MGO stats before panic --- dbtest/dbserver.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index c9bb558ff..d7711f601 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -299,7 +299,9 @@ func (dbs *DBServer) checkSessions() { } time.Sleep(100 * time.Millisecond) } - panic("There are mgo sessions still alive.") + stats := mgo.GetStats() + panic(fmt.Sprintf("There are mgo sessions still alive. SocketCount=%d InUseCount=%d", + stats.SocketsAlive, stats.SocketsInUse)) } // Wipe drops all created databases and their data. From 4f5053df387a5023d73d999c61a18fe6996f2aec Mon Sep 17 00:00:00 2001 From: Kondal Boreddy Date: Tue, 20 Jun 2017 12:04:57 -0700 Subject: [PATCH 20/69] Split bulk update and remove operations to stay under limit. --- bulk_test.go | 45 +++++++++++++++++++++++++++++++++++++++++++++ session.go | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+) diff --git a/bulk_test.go b/bulk_test.go index cb280bbfa..7a5dbecc0 100644 --- a/bulk_test.go +++ b/bulk_test.go @@ -317,6 +317,28 @@ func (s *S) TestBulkUpdate(c *C) { c.Assert(res, DeepEquals, []doc{{10}, {20}, {30}}) } +func (s *S) TestBulkUpdateOver1000(c *C) { + session, err := mgo.Dial("localhost:40001") + c.Assert(err, IsNil) + defer session.Close() + + coll := session.DB("mydb").C("mycoll") + + bulk := coll.Bulk() + for i := 0; i < 1010; i++ { + bulk.Insert(M{"n": i}) + } + _, err = bulk.Run() + c.Assert(err, IsNil) + bulk = coll.Bulk() + for i := 0; i < 1010; i++ { + bulk.Update(M{"n": i}, M{"$set": M{"m": i}}) + } + // if not handle well, mongo will return error here + _, err = bulk.Run() + c.Assert(err, IsNil) +} + func (s *S) TestBulkUpdateError(c *C) { session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) @@ -502,3 +524,26 @@ func (s *S) TestBulkRemoveAll(c *C) { c.Assert(err, IsNil) c.Assert(res, DeepEquals, []doc{{3}}) } + +func (s *S) TestBulkDeleteOver1000(c *C) { + session, err := mgo.Dial("localhost:40001") + c.Assert(err, IsNil) + defer session.Close() + + coll := session.DB("mydb").C("mycoll") + + bulk := coll.Bulk() + for i := 0; i < 1010; i++ { + bulk.Insert(M{"n": i}) + } + _, err = bulk.Run() + c.Assert(err, IsNil) + bulk = coll.Bulk() + for i := 0; i < 1010; i++ { + bulk.Remove(M{"n": i}) + } + // if not handle well, mongo will return error here + _, err = bulk.Run() + c.Assert(err, IsNil) +} + diff --git a/session.go b/session.go index 3dccf364e..3bd5c1f62 100644 --- a/session.go +++ b/session.go @@ -4613,6 +4613,7 @@ func (c *Collection) writeOp(op interface{}, ordered bool) (lerr *LastError, err if socket.ServerInfo().MaxWireVersion >= 2 { // Servers with a more recent write protocol benefit from write commands. + inputOp := op if op, ok := op.(*insertOp); ok && len(op.documents) > 1000 { var lerr LastError @@ -4641,6 +4642,54 @@ func (c *Collection) writeOp(op interface{}, ordered bool) (lerr *LastError, err return &lerr, lerr.ecases[0].Err } return &lerr, nil + } else if updateOps, ok := inputOp.(bulkUpdateOp); ok && len(updateOps) > 1000 { + var lerr LastError + // Maximum batch size is 1000. Must split out in separate operations for compatibility. + all := updateOps + for i := 0; i < len(all); i += 1000 { + l := i + 1000 + if l > len(all) { + l = len(all) + } + updateOps = all[i:l] + oplerr, err := c.writeOpCommand(socket, safeOp, updateOps, ordered, bypassValidation) + lerr.N += oplerr.N + lerr.modified += oplerr.modified + if err != nil { + for ei := range oplerr.ecases { + oplerr.ecases[ei].Index += i + } + lerr.ecases = append(lerr.ecases, oplerr.ecases...) + } + } + if len(lerr.ecases) != 0 { + return &lerr, lerr.ecases[0].Err + } + return &lerr, nil + } else if deleteOps, ok := inputOp.(bulkDeleteOp); ok && len(deleteOps) > 1000 { + var lerr LastError + // Maximum batch size is 1000. Must split out in separate operations for compatibility. + all := deleteOps + for i := 0; i < len(all); i += 1000 { + l := i + 1000 + if l > len(all) { + l = len(all) + } + deleteOps = all[i:l] + oplerr, err := c.writeOpCommand(socket, safeOp, deleteOps, ordered, bypassValidation) + lerr.N += oplerr.N + lerr.modified += oplerr.modified + if err != nil { + for ei := range oplerr.ecases { + oplerr.ecases[ei].Index += i + } + lerr.ecases = append(lerr.ecases, oplerr.ecases...) + } + } + if len(lerr.ecases) != 0 { + return &lerr, lerr.ecases[0].Err + } + return &lerr, nil } return c.writeOpCommand(socket, safeOp, op, ordered, bypassValidation) } else if updateOps, ok := op.(bulkUpdateOp); ok { From 70c8959816ee69d4eab758cfb85a68e34d9bf943 Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Fri, 23 Jun 2017 16:39:45 -0700 Subject: [PATCH 21/69] CSCve54898 Add more debug info for tests --- dbtest/dbserver.go | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index d7711f601..415e8fb54 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -206,24 +206,28 @@ func (dbs *DBServer) start() { } err = dbs.server.Start() if err != nil { - panic(err) + panic("Failed to start test Mongo instance", err) } dbs.tomb.Go(dbs.monitor) dbs.Wipe() } +func (dbs DBServer) printMongoDebugInfo() { + fmt.Fprintf(os.Stderr, "---- mongod processes running right now:\n") + cmd := exec.Command("/bin/sh", "-c", "ps auxw | grep mongod") + cmd.Stdout = os.Stderr + cmd.Stderr = os.Stderr + cmd.Run() + fmt.Fprintf(os.Stderr, "----------------------------------------\n") +} + func (dbs *DBServer) monitor() error { dbs.server.Process.Wait() if dbs.tomb.Alive() { // Present some debugging information. fmt.Fprintf(os.Stderr, "---- mongod process died unexpectedly:\n") fmt.Fprintf(os.Stderr, "%s", dbs.output.Bytes()) - fmt.Fprintf(os.Stderr, "---- mongod processes running right now:\n") - cmd := exec.Command("/bin/sh", "-c", "ps auxw | grep mongod") - cmd.Stdout = os.Stderr - cmd.Stderr = os.Stderr - cmd.Run() - fmt.Fprintf(os.Stderr, "----------------------------------------\n") + dbs.printMongoDebugInfo() panic("mongod process died unexpectedly") } @@ -277,6 +281,7 @@ func (dbs *DBServer) Session() *mgo.Session { if err != nil { fmt.Fprintf(os.Stderr, "---- Unable to dial mongod:\n") fmt.Fprintf(os.Stderr, "%s", dbs.output.Bytes()) + dbs.printMongoDebugInfo() panic(err) } } From b1aa086624d81206197488cc0d6ec3acdbf6ae01 Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Fri, 23 Jun 2017 16:57:39 -0700 Subject: [PATCH 22/69] CSCve54898 Add more debug info for tests --- dbtest/dbserver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index 415e8fb54..75875e2c4 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -206,7 +206,7 @@ func (dbs *DBServer) start() { } err = dbs.server.Start() if err != nil { - panic("Failed to start test Mongo instance", err) + panic("Failed to start Mongo instance: " + err.Error()) } dbs.tomb.Go(dbs.monitor) dbs.Wipe() From 6f135e7fff6ab032dd3d49d888bb2e3d94c49ce7 Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Sun, 25 Jun 2017 06:55:50 -0700 Subject: [PATCH 23/69] CSCve54898 Add configurable timeout --- dbtest/dbserver.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index 75875e2c4..b4ef1141c 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -270,14 +270,14 @@ func (dbs *DBServer) Stop() { // must be closed after the test is done with it. // // The first Session obtained from a DBServer will start it. -func (dbs *DBServer) Session() *mgo.Session { +func (dbs *DBServer) SessionWithTimeout(timeout time.Duration) *mgo.Session { if dbs.server == nil { dbs.start() } if dbs.session == nil { mgo.ResetStats() var err error - dbs.session, err = mgo.DialWithTimeout(dbs.host+"/test", 10*time.Second) + dbs.session, err = mgo.DialWithTimeout(dbs.host+"/test", timeout) if err != nil { fmt.Fprintf(os.Stderr, "---- Unable to dial mongod:\n") fmt.Fprintf(os.Stderr, "%s", dbs.output.Bytes()) @@ -288,6 +288,14 @@ func (dbs *DBServer) Session() *mgo.Session { return dbs.session.Copy() } +// Session returns a new session to the server. The returned session +// must be closed after the test is done with it. +// +// The first Session obtained from a DBServer will start it. +func (dbs *DBServer) Session() *mgo.Session { + return dbs.SessionWithTimeout(10*time.Second) +} + // checkSessions ensures all mgo sessions opened were properly closed. // For slightly faster tests, it may be disabled setting the // environmnet variable CHECK_SESSIONS to 0. From 3bc492872dc1118d1d145f2b14e174c82938c3e6 Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Sun, 25 Jun 2017 08:39:00 -0700 Subject: [PATCH 24/69] CSCve54898 Add troubleshooting when mongo runs in a container --- dbtest/dbserver.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index b4ef1141c..ff978c2a9 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -218,6 +218,26 @@ func (dbs DBServer) printMongoDebugInfo() { cmd.Stdout = os.Stderr cmd.Stderr = os.Stderr cmd.Run() + if dbs.eType == Docker { + args := []string{ + "inspect", + dbs.containerName, + } + cmd = exec.Command("docker", args...) + cmd.Stdout = os.Stderr + cmd.Stderr = os.Stderr + fmt.Fprintf(os.Stderr, "---- Container inspect:\n") + cmd.Run() + args = []string{ + "logs", + dbs.containerName, + } + cmd = exec.Command("docker", args...) + cmd.Stdout = os.Stderr + cmd.Stderr = os.Stderr + fmt.Fprintf(os.Stderr, "---- Container logs:\n") + cmd.Run() + } fmt.Fprintf(os.Stderr, "----------------------------------------\n") } From 1e59200486a5c122996db46a8be2f48648478636 Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Mon, 26 Jun 2017 14:38:55 -0700 Subject: [PATCH 25/69] Add log timestamp to help troubleshooting --- dbtest/dbserver.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index ff978c2a9..55aba97b5 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -213,7 +213,7 @@ func (dbs *DBServer) start() { } func (dbs DBServer) printMongoDebugInfo() { - fmt.Fprintf(os.Stderr, "---- mongod processes running right now:\n") + fmt.Fprintf(os.Stderr, "---- %s mongod processes running right now:\n", time.Now().String()) cmd := exec.Command("/bin/sh", "-c", "ps auxw | grep mongod") cmd.Stdout = os.Stderr cmd.Stderr = os.Stderr @@ -226,7 +226,7 @@ func (dbs DBServer) printMongoDebugInfo() { cmd = exec.Command("docker", args...) cmd.Stdout = os.Stderr cmd.Stderr = os.Stderr - fmt.Fprintf(os.Stderr, "---- Container inspect:\n") + fmt.Fprintf(os.Stderr, "---- %s Container inspect:\n", time.Now().String()) cmd.Run() args = []string{ "logs", @@ -235,7 +235,7 @@ func (dbs DBServer) printMongoDebugInfo() { cmd = exec.Command("docker", args...) cmd.Stdout = os.Stderr cmd.Stderr = os.Stderr - fmt.Fprintf(os.Stderr, "---- Container logs:\n") + fmt.Fprintf(os.Stderr, "---- %s Container logs:\n", time.Now().String()) cmd.Run() } fmt.Fprintf(os.Stderr, "----------------------------------------\n") From 41d7481700a8c2e642db25c08aa5ffd71520a4aa Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Mon, 26 Jun 2017 17:45:30 -0700 Subject: [PATCH 26/69] Add more debug for troubleshooting --- dbtest/dbserver.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index 55aba97b5..872185f8b 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -202,18 +202,21 @@ func (dbs *DBServer) start() { dbs.server.Stdout = &dbs.output dbs.server.Stderr = &dbs.output if dbs.debug { - fmt.Printf("Starting Mongo instance: %v\n", dbs.server.Args) + fmt.Printf("[%s] Starting Mongo instance: %v\n", time.Now().String(), dbs.server.Args) } err = dbs.server.Start() if err != nil { panic("Failed to start Mongo instance: " + err.Error()) } + if dbs.debug { + fmt.Printf("[%s] Mongo instance started\n", time.Now().String()) + } dbs.tomb.Go(dbs.monitor) dbs.Wipe() } func (dbs DBServer) printMongoDebugInfo() { - fmt.Fprintf(os.Stderr, "---- %s mongod processes running right now:\n", time.Now().String()) + fmt.Fprintf(os.Stderr, "[%s] mongod processes running right now:\n", time.Now().String()) cmd := exec.Command("/bin/sh", "-c", "ps auxw | grep mongod") cmd.Stdout = os.Stderr cmd.Stderr = os.Stderr @@ -226,7 +229,7 @@ func (dbs DBServer) printMongoDebugInfo() { cmd = exec.Command("docker", args...) cmd.Stdout = os.Stderr cmd.Stderr = os.Stderr - fmt.Fprintf(os.Stderr, "---- %s Container inspect:\n", time.Now().String()) + fmt.Fprintf(os.Stderr, "[%s] Container inspect:\n", time.Now().String()) cmd.Run() args = []string{ "logs", @@ -235,7 +238,7 @@ func (dbs DBServer) printMongoDebugInfo() { cmd = exec.Command("docker", args...) cmd.Stdout = os.Stderr cmd.Stderr = os.Stderr - fmt.Fprintf(os.Stderr, "---- %s Container logs:\n", time.Now().String()) + fmt.Fprintf(os.Stderr, "[%s] Container logs:\n", time.Now().String()) cmd.Run() } fmt.Fprintf(os.Stderr, "----------------------------------------\n") @@ -299,7 +302,7 @@ func (dbs *DBServer) SessionWithTimeout(timeout time.Duration) *mgo.Session { var err error dbs.session, err = mgo.DialWithTimeout(dbs.host+"/test", timeout) if err != nil { - fmt.Fprintf(os.Stderr, "---- Unable to dial mongod:\n") + fmt.Fprintf(os.Stderr, "[%s] Unable to dial mongod. Timeout=%v, Error: %s\n", time.Now().String(), timeout, err.Error()) fmt.Fprintf(os.Stderr, "%s", dbs.output.Bytes()) dbs.printMongoDebugInfo() panic(err) From f2b7ebf5615f7a4708c4b74376155b871a98082e Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Thu, 10 Aug 2017 08:02:15 -0700 Subject: [PATCH 27/69] Send 'docker' command output to stderr/stdout if debug is enabled --- dbtest/dbserver.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index 872185f8b..ac5e1ed04 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -83,6 +83,8 @@ func (dbs *DBServer) execContainer(port int) *exec.Cmd { cmd := exec.Command("docker", args...) if dbs.debug { fmt.Printf("Pulling Mongo docker image\n") + cmd.Stdout = os.Stderr + cmd.Stderr = os.Stderr } err := cmd.Run() if err != nil { @@ -145,6 +147,10 @@ func (dbs *DBServer) stopContainer() { dbs.containerName, } cmd := exec.Command("docker", args...) + if dbs.debug { + cmd.Stdout = os.Stderr + cmd.Stderr = os.Stderr + } err := cmd.Run() if err != nil { panic(err) @@ -157,6 +163,10 @@ func (dbs *DBServer) stopContainer() { dbs.containerName, } cmd = exec.Command("docker", args...) + if dbs.debug { + cmd.Stdout = os.Stderr + cmd.Stderr = os.Stderr + } err = cmd.Run() } From 66ca93ceed771d8c474d4091d2cbb238f1b3fc28 Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Fri, 11 Aug 2017 10:07:58 -0700 Subject: [PATCH 28/69] In case of failure, print list of containers running mongo --- dbtest/dbserver.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index ac5e1ed04..fd18db662 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -232,6 +232,11 @@ func (dbs DBServer) printMongoDebugInfo() { cmd.Stderr = os.Stderr cmd.Run() if dbs.eType == Docker { + fmt.Fprintf(os.Stderr, "[%s] mongod containers running right now:\n", time.Now().String()) + cmd := exec.Command("/bin/sh", "-c", "docker ps -a |grep mongo") + cmd.Stdout = os.Stderr + cmd.Stderr = os.Stderr + cmd.Run() args := []string{ "inspect", dbs.containerName, From aa8ae4403eeb1942261d70c0dc7ebd3936d5e588 Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Tue, 15 Aug 2017 12:12:40 -0700 Subject: [PATCH 29/69] Let docker assign host port --- dbtest/dbserver.go | 145 ++++++++++++++++++++++++++++++--------------- 1 file changed, 97 insertions(+), 48 deletions(-) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index fd18db662..5de270f7c 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -83,8 +83,8 @@ func (dbs *DBServer) execContainer(port int) *exec.Cmd { cmd := exec.Command("docker", args...) if dbs.debug { fmt.Printf("Pulling Mongo docker image\n") - cmd.Stdout = os.Stderr - cmd.Stderr = os.Stderr + cmd.Stdout = os.Stderr + cmd.Stderr = os.Stderr } err := cmd.Run() if err != nil { @@ -104,15 +104,23 @@ func (dbs *DBServer) execContainer(port int) *exec.Cmd { if err != nil { panic(err) } - dbs.containerName = fmt.Sprintf("%s", hex.EncodeToString(u)) + dbs.containerName = fmt.Sprintf("mongo-%s", hex.EncodeToString(u)) // On some platforms, we get "chown: changing ownership of '/proc/1/fd/1': Permission denied" unless // we allocate a pseudo tty (-t option) + var portArg string + if port > 0 { + portArg = fmt.Sprintf("%d:%d", port, 27017) + } else { + // Let docker allocate the port. + // This is useful when multiple tests are running on the same host + portArg = fmt.Sprintf("%d", 27017) + } args = []string{ "run", "-t", "-p", - fmt.Sprintf("%d:%d", port, 27017), + portArg, "--rm", // Automatically remove the container when it exits "--name", dbs.containerName, @@ -140,6 +148,41 @@ func (dbs *DBServer) GetHostName() string { } } +// GetContainerHostPort returns the Host port for the test Mongo instance +func (dbs *DBServer) GetContainerHostPort() (int, error) { + start := time.Now() + var err error + var stderr bytes.Buffer + for time.Since(start) < 30*time.Second { + stderr.Reset() + args := []string{"port", dbs.containerName, fmt.Sprintf("%d", 27017)} + cmd := exec.Command("docker", args...) // #nosec + var out bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &stderr + err = cmd.Run() + if err != nil { + // This could be because the container has not started yet. Retry later + fmt.Printf("Failed to get container host port number. Will retry later...") + time.Sleep(3 * time.Second) + continue + } + s := strings.TrimSpace(out.String()) + o := strings.Split(s, ":") + if len(o) < 2 { + fmt.Printf("Unable to get container host port number: %s", s) + return -1, errors.New(fmt.Sprintf("Unable to get container host port number. %s", s)) + } + i, err2 := strconv.Atoi(o[1]) + if err2 != nil { + fmt.Printf("Unable to parse port number: error=%s, out=%s", err2.Error(), o[1]) + } + return i, err2 + } + fmt.Printf("Failed to run command. error=%s, stderr=%s", err.Error(), stderr.String()) + return -1, err +} + // Stop the docker container running Mongo. func (dbs *DBServer) stopContainer() { args := []string{ @@ -148,9 +191,9 @@ func (dbs *DBServer) stopContainer() { } cmd := exec.Command("docker", args...) if dbs.debug { - cmd.Stdout = os.Stderr - cmd.Stderr = os.Stderr - } + cmd.Stdout = os.Stderr + cmd.Stderr = os.Stderr + } err := cmd.Run() if err != nil { panic(err) @@ -164,9 +207,9 @@ func (dbs *DBServer) stopContainer() { } cmd = exec.Command("docker", args...) if dbs.debug { - cmd.Stdout = os.Stderr - cmd.Stderr = os.Stderr - } + cmd.Stdout = os.Stderr + cmd.Stderr = os.Stderr + } err = cmd.Run() } @@ -205,7 +248,7 @@ func (dbs *DBServer) start() { case LocalProcess: dbs.server = dbs.execLocal(addr.Port) case Docker: - dbs.server = dbs.execContainer(addr.Port) + dbs.server = dbs.execContainer(0) default: panic(fmt.Sprintf("unsupported exec type: %d", dbs.eType)) } @@ -226,37 +269,43 @@ func (dbs *DBServer) start() { } func (dbs DBServer) printMongoDebugInfo() { - fmt.Fprintf(os.Stderr, "[%s] mongod processes running right now:\n", time.Now().String()) - cmd := exec.Command("/bin/sh", "-c", "ps auxw | grep mongod") - cmd.Stdout = os.Stderr - cmd.Stderr = os.Stderr - cmd.Run() - if dbs.eType == Docker { - fmt.Fprintf(os.Stderr, "[%s] mongod containers running right now:\n", time.Now().String()) - cmd := exec.Command("/bin/sh", "-c", "docker ps -a |grep mongo") - cmd.Stdout = os.Stderr - cmd.Stderr = os.Stderr - cmd.Run() - args := []string{ - "inspect", - dbs.containerName, - } - cmd = exec.Command("docker", args...) - cmd.Stdout = os.Stderr - cmd.Stderr = os.Stderr - fmt.Fprintf(os.Stderr, "[%s] Container inspect:\n", time.Now().String()) - cmd.Run() - args = []string{ - "logs", - dbs.containerName, - } - cmd = exec.Command("docker", args...) - cmd.Stdout = os.Stderr - cmd.Stderr = os.Stderr - fmt.Fprintf(os.Stderr, "[%s] Container logs:\n", time.Now().String()) - cmd.Run() - } - fmt.Fprintf(os.Stderr, "----------------------------------------\n") + fmt.Fprintf(os.Stderr, "[%s] mongod processes running right now:\n", time.Now().String()) + cmd := exec.Command("/bin/sh", "-c", "ps auxw | grep mongod") + cmd.Stdout = os.Stderr + cmd.Stderr = os.Stderr + cmd.Run() + if dbs.eType == Docker { + fmt.Fprintf(os.Stderr, "[%s] mongod containers running right now:\n", time.Now().String()) + cmd := exec.Command("/bin/sh", "-c", "docker ps -a |grep mongo") + cmd.Stdout = os.Stderr + cmd.Stderr = os.Stderr + cmd.Run() + + cmd := exec.Command("/bin/sh", "-c", "") + cmd.Stdout = os.Stderr + cmd.Stderr = os.Stderr + cmd.Run() + + args := []string{ + "inspect", + dbs.containerName, + } + cmd = exec.Command("docker", args...) + cmd.Stdout = os.Stderr + cmd.Stderr = os.Stderr + fmt.Fprintf(os.Stderr, "[%s] Container inspect:\n", time.Now().String()) + cmd.Run() + args = []string{ + "logs", + dbs.containerName, + } + cmd = exec.Command("docker", args...) + cmd.Stdout = os.Stderr + cmd.Stderr = os.Stderr + fmt.Fprintf(os.Stderr, "[%s] Container logs:\n", time.Now().String()) + cmd.Run() + } + fmt.Fprintf(os.Stderr, "----------------------------------------\n") } func (dbs *DBServer) monitor() error { @@ -265,7 +314,7 @@ func (dbs *DBServer) monitor() error { // Present some debugging information. fmt.Fprintf(os.Stderr, "---- mongod process died unexpectedly:\n") fmt.Fprintf(os.Stderr, "%s", dbs.output.Bytes()) - dbs.printMongoDebugInfo() + dbs.printMongoDebugInfo() panic("mongod process died unexpectedly") } @@ -319,7 +368,7 @@ func (dbs *DBServer) SessionWithTimeout(timeout time.Duration) *mgo.Session { if err != nil { fmt.Fprintf(os.Stderr, "[%s] Unable to dial mongod. Timeout=%v, Error: %s\n", time.Now().String(), timeout, err.Error()) fmt.Fprintf(os.Stderr, "%s", dbs.output.Bytes()) - dbs.printMongoDebugInfo() + dbs.printMongoDebugInfo() panic(err) } } @@ -331,7 +380,7 @@ func (dbs *DBServer) SessionWithTimeout(timeout time.Duration) *mgo.Session { // // The first Session obtained from a DBServer will start it. func (dbs *DBServer) Session() *mgo.Session { - return dbs.SessionWithTimeout(10*time.Second) + return dbs.SessionWithTimeout(10 * time.Second) } // checkSessions ensures all mgo sessions opened were properly closed. @@ -350,9 +399,9 @@ func (dbs *DBServer) checkSessions() { } time.Sleep(100 * time.Millisecond) } - stats := mgo.GetStats() - panic(fmt.Sprintf("There are mgo sessions still alive. SocketCount=%d InUseCount=%d", - stats.SocketsAlive, stats.SocketsInUse)) + stats := mgo.GetStats() + panic(fmt.Sprintf("There are mgo sessions still alive. SocketCount=%d InUseCount=%d", + stats.SocketsAlive, stats.SocketsInUse)) } // Wipe drops all created databases and their data. From 3986c5bf2bc7b58375901e3cba1a98c4f459dab0 Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Tue, 15 Aug 2017 12:18:33 -0700 Subject: [PATCH 30/69] Let docker assign host port --- dbtest/dbserver.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index 5de270f7c..5578c67bd 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -3,12 +3,14 @@ package dbtest import ( "bytes" "encoding/hex" + "errors" "fmt" "math/rand" "net" "os" "os/exec" "strconv" + "strings" "time" "gopkg.in/mgo.v2" From 1a4c45e1c46ffd5b47e7bec83d50878f7e3d16ce Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Tue, 15 Aug 2017 12:28:18 -0700 Subject: [PATCH 31/69] Let docker assign host port --- dbtest/dbserver.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index 5578c67bd..406e7bff8 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -283,11 +283,6 @@ func (dbs DBServer) printMongoDebugInfo() { cmd.Stderr = os.Stderr cmd.Run() - cmd := exec.Command("/bin/sh", "-c", "") - cmd.Stdout = os.Stderr - cmd.Stderr = os.Stderr - cmd.Run() - args := []string{ "inspect", dbs.containerName, From 34d4ed08acf876a656b6b1a6a9d42a22a966f230 Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Tue, 15 Aug 2017 13:02:54 -0700 Subject: [PATCH 32/69] Let docker assign host port --- dbtest/dbserver.go | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index 406e7bff8..13a6a44d5 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -3,14 +3,14 @@ package dbtest import ( "bytes" "encoding/hex" - "errors" + "errors" "fmt" "math/rand" "net" "os" "os/exec" "strconv" - "strings" + "strings" "time" "gopkg.in/mgo.v2" @@ -165,7 +165,7 @@ func (dbs *DBServer) GetContainerHostPort() (int, error) { err = cmd.Run() if err != nil { // This could be because the container has not started yet. Retry later - fmt.Printf("Failed to get container host port number. Will retry later...") + fmt.Printf("Failed to get container host port number. Will retry later...\n") time.Sleep(3 * time.Second) continue } @@ -177,11 +177,11 @@ func (dbs *DBServer) GetContainerHostPort() (int, error) { } i, err2 := strconv.Atoi(o[1]) if err2 != nil { - fmt.Printf("Unable to parse port number: error=%s, out=%s", err2.Error(), o[1]) + fmt.Printf("Unable to parse port number: error=%s, out=%s\n", err2.Error(), o[1]) } return i, err2 } - fmt.Printf("Failed to run command. error=%s, stderr=%s", err.Error(), stderr.String()) + fmt.Printf("Failed to run command. error=%s, stderr=%s\n", err.Error(), stderr.String()) return -1, err } @@ -243,11 +243,11 @@ func (dbs *DBServer) start() { } addr := l.Addr().(*net.TCPAddr) l.Close() - dbs.host = addr.String() dbs.tomb = tomb.Tomb{} switch dbs.eType { case LocalProcess: + dbs.host = addr.String() dbs.server = dbs.execLocal(addr.Port) case Docker: dbs.server = dbs.execContainer(0) @@ -257,7 +257,7 @@ func (dbs *DBServer) start() { dbs.server.Stdout = &dbs.output dbs.server.Stderr = &dbs.output if dbs.debug { - fmt.Printf("[%s] Starting Mongo instance: %v\n", time.Now().String(), dbs.server.Args) + fmt.Printf("[%s] Starting Mongo instance: %v. Address: %s\n", time.Now().String(), dbs.server.Args, dbs.host) } err = dbs.server.Start() if err != nil { @@ -266,6 +266,13 @@ func (dbs *DBServer) start() { if dbs.debug { fmt.Printf("[%s] Mongo instance started\n", time.Now().String()) } + if dbs.eType == Docker { + p, err2 := dbs.GetContainerHostPort() + if err2 != nil { + panic(err2) + } + dbs.host = fmt.Sprintf("127.0.0.1:%d", p) + } dbs.tomb.Go(dbs.monitor) dbs.Wipe() } From eb71381b279b0209a5837ab6b6ed9aeb4d2c6242 Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Tue, 15 Aug 2017 22:13:11 -0700 Subject: [PATCH 33/69] Let docker assign host port --- dbtest/dbserver.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index 13a6a44d5..322e70397 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -155,7 +155,7 @@ func (dbs *DBServer) GetContainerHostPort() (int, error) { start := time.Now() var err error var stderr bytes.Buffer - for time.Since(start) < 30*time.Second { + for time.Since(start) < 60*time.Second { stderr.Reset() args := []string{"port", dbs.containerName, fmt.Sprintf("%d", 27017)} cmd := exec.Command("docker", args...) // #nosec @@ -173,10 +173,12 @@ func (dbs *DBServer) GetContainerHostPort() (int, error) { o := strings.Split(s, ":") if len(o) < 2 { fmt.Printf("Unable to get container host port number: %s", s) + dbs.printMongoDebugInfo() return -1, errors.New(fmt.Sprintf("Unable to get container host port number. %s", s)) } i, err2 := strconv.Atoi(o[1]) if err2 != nil { + dbs.printMongoDebugInfo() fmt.Printf("Unable to parse port number: error=%s, out=%s\n", err2.Error(), o[1]) } return i, err2 From 9bbc6232d2349855796abda824d24cb75035ea12 Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Tue, 19 Sep 2017 07:51:19 -0700 Subject: [PATCH 34/69] Add retry loop when pulling mongo image --- dbtest/dbserver.go | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index 322e70397..ba38b075b 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -82,16 +82,27 @@ func (dbs *DBServer) execContainer(port int) *exec.Cmd { "pull", fmt.Sprintf("mongo:%s", dbs.version), } - cmd := exec.Command("docker", args...) - if dbs.debug { - fmt.Printf("Pulling Mongo docker image\n") - cmd.Stdout = os.Stderr - cmd.Stderr = os.Stderr - } - err := cmd.Run() - if err != nil { - panic(err) - } + start := time.Now() + var err error + // Seeing intermittent issues such as: + // Error response from daemon: Get https://registry-1.docker.io/v2/: net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers) + for time.Since(start) < 60*time.Second { + cmd := exec.Command("docker", args...) + if dbs.debug { + fmt.Printf("Pulling Mongo docker image\n") + cmd.Stdout = os.Stderr + cmd.Stderr = os.Stderr + } + err = cmd.Run() + if err == nil { + break + } else { + time.Sleep(5 * time.Second) + } + } + if err != nil { + panic(err) + } if dbs.debug { fmt.Printf("Pulled Mongo docker image\n") } From a90f58c4c4fc61c77e7cdd256f9b1dd2b4de1d33 Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Tue, 19 Sep 2017 09:22:06 -0700 Subject: [PATCH 35/69] Add print statement for debugging --- dbtest/dbserver.go | 1 + 1 file changed, 1 insertion(+) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index ba38b075b..9ec82b7d6 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -97,6 +97,7 @@ func (dbs *DBServer) execContainer(port int) *exec.Cmd { if err == nil { break } else { + fmt.Printf("Failed to pull Mongo container image. err=%s", err.String()) time.Sleep(5 * time.Second) } } From 9ba021d48b8aaace32734223a1cb4dea85c2418b Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Tue, 19 Sep 2017 10:25:01 -0700 Subject: [PATCH 36/69] Add print statement for debugging --- dbtest/dbserver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index 9ec82b7d6..759d23a55 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -97,7 +97,7 @@ func (dbs *DBServer) execContainer(port int) *exec.Cmd { if err == nil { break } else { - fmt.Printf("Failed to pull Mongo container image. err=%s", err.String()) + fmt.Printf("Failed to pull Mongo container image. err=%s", err.Error()) time.Sleep(5 * time.Second) } } From 9af84a5331bd9cd8a318d19665412bbd988c32b2 Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Tue, 26 Sep 2017 16:57:26 -0700 Subject: [PATCH 37/69] Print log statement when skipping Wipe() --- dbtest/dbserver.go | 1 + 1 file changed, 1 insertion(+) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index 759d23a55..9ab698783 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -432,6 +432,7 @@ func (dbs *DBServer) checkSessions() { // there is a session leak. func (dbs *DBServer) Wipe() { if dbs.server == nil || dbs.session == nil { + fmt.Printf("Skip Wipe()") return } dbs.checkSessions() From 26872c508e16dd7c472c2857c9b5725133e576dd Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Tue, 26 Sep 2017 17:00:42 -0700 Subject: [PATCH 38/69] Print log statement when skipping Wipe() --- dbtest/dbserver.go | 1 + 1 file changed, 1 insertion(+) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index 9ab698783..ddf20934d 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -451,6 +451,7 @@ func (dbs *DBServer) Wipe() { switch name { case "admin", "local", "config": default: + fmt.Printf("Drop database '%s'", name) err = session.DB(name).DropDatabase() if err != nil { panic(err) From 95d62cb9995d74fc647dc6d95e340d910b6c2fb8 Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Tue, 26 Sep 2017 17:15:39 -0700 Subject: [PATCH 39/69] Print log statement when skipping Wipe() --- dbtest/dbserver.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index ddf20934d..95fd8d0ec 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -432,7 +432,7 @@ func (dbs *DBServer) checkSessions() { // there is a session leak. func (dbs *DBServer) Wipe() { if dbs.server == nil || dbs.session == nil { - fmt.Printf("Skip Wipe()") + fmt.Printf("Skip Wipe()\n") return } dbs.checkSessions() @@ -451,7 +451,7 @@ func (dbs *DBServer) Wipe() { switch name { case "admin", "local", "config": default: - fmt.Printf("Drop database '%s'", name) + fmt.Printf("Drop database '%s'\n", name) err = session.DB(name).DropDatabase() if err != nil { panic(err) From c1b4e885ad0bbb65426ddea00c50991fca78e317 Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Thu, 7 Dec 2017 23:24:19 -0800 Subject: [PATCH 40/69] Add more debugging --- bson/decode.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bson/decode.go b/bson/decode.go index 7c2d8416a..927b54ce9 100644 --- a/bson/decode.go +++ b/bson/decode.go @@ -549,7 +549,9 @@ func (d *decoder) readElemTo(out reflect.Value, kind byte) (good bool) { case 0xFF: // Min key in = MinKey default: - panic(fmt.Sprintf("Unknown element kind (0x%02X)", kind)) + var st []byte = make([]byte, 4096) + w := runtime.Stack(st, false) + panic(fmt.Sprintf("Unknown element kind (0x%02X) BT: %s", kind, string(st[:w]))) } outt := out.Type() From cf480964648ba2de1066e92c0657db8a3880fa60 Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Thu, 7 Dec 2017 23:38:57 -0800 Subject: [PATCH 41/69] Add more debugging --- bson/decode.go | 1 + 1 file changed, 1 insertion(+) diff --git a/bson/decode.go b/bson/decode.go index 927b54ce9..505eb48ef 100644 --- a/bson/decode.go +++ b/bson/decode.go @@ -34,6 +34,7 @@ import ( "reflect" "strconv" "sync" + "runtime" "time" ) From 3fcf468856f3c6a21737f5d157db2e6641ed4e33 Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Sat, 11 May 2019 15:32:30 -0700 Subject: [PATCH 42/69] Improve logging for unit tests --- dbtest/dbserver.go | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index 95fd8d0ec..5694396df 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -89,7 +89,7 @@ func (dbs *DBServer) execContainer(port int) *exec.Cmd { for time.Since(start) < 60*time.Second { cmd := exec.Command("docker", args...) if dbs.debug { - fmt.Printf("Pulling Mongo docker image\n") + fmt.Printf("[%s] Pulling Mongo docker image\n", time.Now().String()) cmd.Stdout = os.Stderr cmd.Stderr = os.Stderr } @@ -97,7 +97,7 @@ func (dbs *DBServer) execContainer(port int) *exec.Cmd { if err == nil { break } else { - fmt.Printf("Failed to pull Mongo container image. err=%s", err.Error()) + fmt.Printf("[%s] Failed to pull Mongo container image. err=%s", time.Now().String(), err.Error()) time.Sleep(5 * time.Second) } } @@ -105,7 +105,7 @@ func (dbs *DBServer) execContainer(port int) *exec.Cmd { panic(err) } if dbs.debug { - fmt.Printf("Pulled Mongo docker image\n") + fmt.Printf("[%s] Pulled Mongo docker image\n", time.Now().String()) } // Generate a name for the container. This will help to inspect the container @@ -177,7 +177,7 @@ func (dbs *DBServer) GetContainerHostPort() (int, error) { err = cmd.Run() if err != nil { // This could be because the container has not started yet. Retry later - fmt.Printf("Failed to get container host port number. Will retry later...\n") + fmt.Printf("[%s] Failed to get container host port number. Will retry later...\n", time.Now().String()) time.Sleep(3 * time.Second) continue } @@ -191,11 +191,13 @@ func (dbs *DBServer) GetContainerHostPort() (int, error) { i, err2 := strconv.Atoi(o[1]) if err2 != nil { dbs.printMongoDebugInfo() - fmt.Printf("Unable to parse port number: error=%s, out=%s\n", err2.Error(), o[1]) - } + fmt.Printf("[%s] Unable to parse port number: error=%s, out=%s\n", time.Now().String(), err2.Error(), o[1]) + } else { + fmt.Printf("[%s] MongoDB Container host port number: %d\n", time.Now().String(), i) + } return i, err2 } - fmt.Printf("Failed to run command. error=%s, stderr=%s\n", err.Error(), stderr.String()) + fmt.Printf("[%s] Failed to run command. error=%s, stderr=%s\n", time.Now().String(), err.Error(), stderr.String()) return -1, err } @@ -384,7 +386,7 @@ func (dbs *DBServer) SessionWithTimeout(timeout time.Duration) *mgo.Session { var err error dbs.session, err = mgo.DialWithTimeout(dbs.host+"/test", timeout) if err != nil { - fmt.Fprintf(os.Stderr, "[%s] Unable to dial mongod. Timeout=%v, Error: %s\n", time.Now().String(), timeout, err.Error()) + fmt.Fprintf(os.Stderr, "[%s] Unable to dial mongod located at '%s'. Timeout=%v, Error: %s\n", time.Now().String(), dbs.host, timeout, err.Error()) fmt.Fprintf(os.Stderr, "%s", dbs.output.Bytes()) dbs.printMongoDebugInfo() panic(err) @@ -432,7 +434,7 @@ func (dbs *DBServer) checkSessions() { // there is a session leak. func (dbs *DBServer) Wipe() { if dbs.server == nil || dbs.session == nil { - fmt.Printf("Skip Wipe()\n") + fmt.Printf("[%s] Skip Wipe()\n", time.Now().String()) return } dbs.checkSessions() From fd583cf1f32fe058432cd5d4f154eec8f47df1fe Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Sat, 11 May 2019 16:01:54 -0700 Subject: [PATCH 43/69] Add option to attach UT container to a specific docker network --- dbtest/dbserver.go | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index 5694396df..b52559fb0 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -42,6 +42,7 @@ type DBServer struct { host string version string // The request MongoDB version, when running within a container eType int // Specify whether mongo should run as a container or regular process + network string // The name of the docker network to which the UT container should be attached debug bool // Log debug statements containerName string // The container name, when running mgo within a container tomb tomb.Tomb @@ -69,9 +70,14 @@ func (dbs *DBServer) SetExecType(execType int) { dbs.eType = execType } +// SetNetwork sets the name of the docker network to which the UT container should be attached. +func (dbs *DBServer) SetNetwork(network string) { + dbs.network = network +} + // Start Mongo DB within Docker container on host. // It assumes Docker is already installed -func (dbs *DBServer) execContainer(port int) *exec.Cmd { +func (dbs *DBServer) execContainer(port int, network string) *exec.Cmd { if dbs.version == "" { dbs.version = "latest" } @@ -136,6 +142,14 @@ func (dbs *DBServer) execContainer(port int) *exec.Cmd { "-p", portArg, "--rm", // Automatically remove the container when it exits + } + if network != "" { + args = append(args, []string{ + "--net", + network, + }...) + } + args = append(args, []string{ "--name", dbs.containerName, fmt.Sprintf("mongo:%s", dbs.version), @@ -143,7 +157,7 @@ func (dbs *DBServer) execContainer(port int) *exec.Cmd { "--noprealloc", "--smallfiles", "--nojournal", - } + }...) return exec.Command("docker", args...) } @@ -266,14 +280,14 @@ func (dbs *DBServer) start() { dbs.host = addr.String() dbs.server = dbs.execLocal(addr.Port) case Docker: - dbs.server = dbs.execContainer(0) + dbs.server = dbs.execContainer(0, dbs.network) default: panic(fmt.Sprintf("unsupported exec type: %d", dbs.eType)) } dbs.server.Stdout = &dbs.output dbs.server.Stderr = &dbs.output if dbs.debug { - fmt.Printf("[%s] Starting Mongo instance: %v. Address: %s\n", time.Now().String(), dbs.server.Args, dbs.host) + fmt.Printf("[%s] Starting Mongo instance: %v. Address: %s. Network: '%s'\n", time.Now().String(), dbs.server.Args, dbs.host, dbs.network) } err = dbs.server.Start() if err != nil { From b58a6c5fd8d0c9c43cea0c68244d5b2ce5382902 Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Sat, 11 May 2019 16:05:04 -0700 Subject: [PATCH 44/69] Add option to attach UT container to a specific docker network. Run gomft --- dbtest/dbserver.go | 68 +++++++++++++++++++++++----------------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index b52559fb0..9ece58975 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -42,7 +42,7 @@ type DBServer struct { host string version string // The request MongoDB version, when running within a container eType int // Specify whether mongo should run as a container or regular process - network string // The name of the docker network to which the UT container should be attached + network string // The name of the docker network to which the UT container should be attached debug bool // Log debug statements containerName string // The container name, when running mgo within a container tomb tomb.Tomb @@ -88,28 +88,28 @@ func (dbs *DBServer) execContainer(port int, network string) *exec.Cmd { "pull", fmt.Sprintf("mongo:%s", dbs.version), } - start := time.Now() - var err error - // Seeing intermittent issues such as: - // Error response from daemon: Get https://registry-1.docker.io/v2/: net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers) - for time.Since(start) < 60*time.Second { - cmd := exec.Command("docker", args...) - if dbs.debug { - fmt.Printf("[%s] Pulling Mongo docker image\n", time.Now().String()) - cmd.Stdout = os.Stderr - cmd.Stderr = os.Stderr - } - err = cmd.Run() - if err == nil { - break - } else { - fmt.Printf("[%s] Failed to pull Mongo container image. err=%s", time.Now().String(), err.Error()) - time.Sleep(5 * time.Second) - } - } - if err != nil { - panic(err) - } + start := time.Now() + var err error + // Seeing intermittent issues such as: + // Error response from daemon: Get https://registry-1.docker.io/v2/: net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers) + for time.Since(start) < 60*time.Second { + cmd := exec.Command("docker", args...) + if dbs.debug { + fmt.Printf("[%s] Pulling Mongo docker image\n", time.Now().String()) + cmd.Stdout = os.Stderr + cmd.Stderr = os.Stderr + } + err = cmd.Run() + if err == nil { + break + } else { + fmt.Printf("[%s] Failed to pull Mongo container image. err=%s", time.Now().String(), err.Error()) + time.Sleep(5 * time.Second) + } + } + if err != nil { + panic(err) + } if dbs.debug { fmt.Printf("[%s] Pulled Mongo docker image\n", time.Now().String()) } @@ -142,14 +142,14 @@ func (dbs *DBServer) execContainer(port int, network string) *exec.Cmd { "-p", portArg, "--rm", // Automatically remove the container when it exits - } - if network != "" { - args = append(args, []string{ - "--net", - network, - }...) - } - args = append(args, []string{ + } + if network != "" { + args = append(args, []string{ + "--net", + network, + }...) + } + args = append(args, []string{ "--name", dbs.containerName, fmt.Sprintf("mongo:%s", dbs.version), @@ -208,7 +208,7 @@ func (dbs *DBServer) GetContainerHostPort() (int, error) { fmt.Printf("[%s] Unable to parse port number: error=%s, out=%s\n", time.Now().String(), err2.Error(), o[1]) } else { fmt.Printf("[%s] MongoDB Container host port number: %d\n", time.Now().String(), i) - } + } return i, err2 } fmt.Printf("[%s] Failed to run command. error=%s, stderr=%s\n", time.Now().String(), err.Error(), stderr.String()) @@ -448,7 +448,7 @@ func (dbs *DBServer) checkSessions() { // there is a session leak. func (dbs *DBServer) Wipe() { if dbs.server == nil || dbs.session == nil { - fmt.Printf("[%s] Skip Wipe()\n", time.Now().String()) + fmt.Printf("[%s] Skip Wipe()\n", time.Now().String()) return } dbs.checkSessions() @@ -467,7 +467,7 @@ func (dbs *DBServer) Wipe() { switch name { case "admin", "local", "config": default: - fmt.Printf("Drop database '%s'\n", name) + fmt.Printf("Drop database '%s'\n", name) err = session.DB(name).DropDatabase() if err != nil { panic(err) From bb1322126a389fda9ab22bd50c69d7bffa8e83e9 Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Sat, 11 May 2019 17:19:23 -0700 Subject: [PATCH 45/69] Add option to attach UT container to a specific docker network. Discover IP address of UT container --- dbtest/dbserver.go | 38 +++++++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index 9ece58975..5e857a46c 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -39,7 +39,7 @@ type DBServer struct { output bytes.Buffer server *exec.Cmd dbpath string - host string + host string // The IP address or host name of the mgo instance. version string // The request MongoDB version, when running within a container eType int // Specify whether mongo should run as a container or regular process network string // The name of the docker network to which the UT container should be attached @@ -166,7 +166,7 @@ func (dbs *DBServer) execContainer(port int, network string) *exec.Cmd { // If the test instance runs in the host, returns the host name. func (dbs *DBServer) GetHostName() string { if dbs.eType == Docker { - return dbs.containerName + return dbs.host } else { if hostname, err := os.Hostname(); err != nil { return hostname @@ -176,19 +176,35 @@ func (dbs *DBServer) GetHostName() string { } } -// GetContainerHostPort returns the Host port for the test Mongo instance -func (dbs *DBServer) GetContainerHostPort() (int, error) { +// GetContainerHostPort returns the IP address and port for the test Mongo instance +func (dbs *DBServer) GetContainerHostPort() (string, int, error) { start := time.Now() var err error var stderr bytes.Buffer for time.Since(start) < 60*time.Second { stderr.Reset() - args := []string{"port", dbs.containerName, fmt.Sprintf("%d", 27017)} + args := []string{"inspect", "-f", "'{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}'", dbs.containerName} cmd := exec.Command("docker", args...) // #nosec var out bytes.Buffer cmd.Stdout = &out cmd.Stderr = &stderr err = cmd.Run() + if err != nil { + // This could be because the container has not started yet. Retry later + fmt.Printf("[%s] Failed to get container IP address. Will retry later...\n", time.Now().String()) + time.Sleep(3 * time.Second) + continue + } + ipAddr := strings.TrimSpace(out.String()) + dbs.host = ipAddr + + // Get port + out.Reset() + args = []string{"port", dbs.containerName, fmt.Sprintf("%d", 27017)} + cmd = exec.Command("docker", args...) // #nosec + cmd.Stdout = &out + cmd.Stderr = &stderr + err = cmd.Run() if err != nil { // This could be because the container has not started yet. Retry later fmt.Printf("[%s] Failed to get container host port number. Will retry later...\n", time.Now().String()) @@ -200,19 +216,19 @@ func (dbs *DBServer) GetContainerHostPort() (int, error) { if len(o) < 2 { fmt.Printf("Unable to get container host port number: %s", s) dbs.printMongoDebugInfo() - return -1, errors.New(fmt.Sprintf("Unable to get container host port number. %s", s)) + return "", -1, errors.New(fmt.Sprintf("Unable to get container host port number. %s", s)) } i, err2 := strconv.Atoi(o[1]) if err2 != nil { dbs.printMongoDebugInfo() fmt.Printf("[%s] Unable to parse port number: error=%s, out=%s\n", time.Now().String(), err2.Error(), o[1]) } else { - fmt.Printf("[%s] MongoDB Container host port number: %d\n", time.Now().String(), i) + fmt.Printf("[%s] MongoDB available at: %s:%d\n", time.Now().String(), ipAddr, i) } - return i, err2 + return ipAddr, i, err2 } fmt.Printf("[%s] Failed to run command. error=%s, stderr=%s\n", time.Now().String(), err.Error(), stderr.String()) - return -1, err + return "", -1, err } // Stop the docker container running Mongo. @@ -297,11 +313,11 @@ func (dbs *DBServer) start() { fmt.Printf("[%s] Mongo instance started\n", time.Now().String()) } if dbs.eType == Docker { - p, err2 := dbs.GetContainerHostPort() + ipAddr, p, err2 := dbs.GetContainerHostPort() if err2 != nil { panic(err2) } - dbs.host = fmt.Sprintf("127.0.0.1:%d", p) + dbs.host = fmt.Sprintf("%s:%d", ipAddr, p) } dbs.tomb.Go(dbs.monitor) dbs.Wipe() From c83ef0198ac3604e51c7eff907959b8dc2961d1d Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Sat, 11 May 2019 17:34:04 -0700 Subject: [PATCH 46/69] Add option to attach UT container to a specific docker network. Add code comments --- dbtest/dbserver.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index 5e857a46c..347448e7b 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -177,6 +177,10 @@ func (dbs *DBServer) GetHostName() string { } // GetContainerHostPort returns the IP address and port for the test Mongo instance +// The Host port is the port which is exposed to the host OS. This is useful if the client tries +// to connect to the Mongo instance from the host. +// If the client connects directly on the docker bridge network (such as when the client is also +// running in a container), then client should connect to port 27017. func (dbs *DBServer) GetContainerHostPort() (string, int, error) { start := time.Now() var err error From b8a31fb85845c6bd7b38e609cfb170d1186274df Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Sat, 11 May 2019 17:51:27 -0700 Subject: [PATCH 47/69] Add option to attach UT container to a specific docker network. --- dbtest/dbserver.go | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index 347448e7b..04c628b3a 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -39,7 +39,8 @@ type DBServer struct { output bytes.Buffer server *exec.Cmd dbpath string - host string // The IP address or host name of the mgo instance. + hostPort string // The IP address and port number of the mgo instance. + hostname string // The IP address or hostname of the container. version string // The request MongoDB version, when running within a container eType int // Specify whether mongo should run as a container or regular process network string // The name of the docker network to which the UT container should be attached @@ -162,11 +163,11 @@ func (dbs *DBServer) execContainer(port int, network string) *exec.Cmd { } // Returns the host name of the Mongo test instance. -// If the test instance runs as a container, it returns the container name. +// If the test instance runs as a container, it returns the IP address of the container. // If the test instance runs in the host, returns the host name. func (dbs *DBServer) GetHostName() string { if dbs.eType == Docker { - return dbs.host + return dbs.hostname } else { if hostname, err := os.Hostname(); err != nil { return hostname @@ -199,8 +200,8 @@ func (dbs *DBServer) GetContainerHostPort() (string, int, error) { time.Sleep(3 * time.Second) continue } - ipAddr := strings.TrimSpace(out.String()) - dbs.host = ipAddr + ipAddr := strings.Trim(strings.TrimSpace(out.String()), "'") + dbs.hostname = ipAddr // Get port out.Reset() @@ -227,7 +228,7 @@ func (dbs *DBServer) GetContainerHostPort() (string, int, error) { dbs.printMongoDebugInfo() fmt.Printf("[%s] Unable to parse port number: error=%s, out=%s\n", time.Now().String(), err2.Error(), o[1]) } else { - fmt.Printf("[%s] MongoDB available at: %s:%d\n", time.Now().String(), ipAddr, i) + fmt.Printf("[%s] MongoDB available on bridge network at: %s. Host port: %d. Container port: %d\n", time.Now().String(), ipAddr, i, 27017) } return ipAddr, i, err2 } @@ -297,7 +298,7 @@ func (dbs *DBServer) start() { dbs.tomb = tomb.Tomb{} switch dbs.eType { case LocalProcess: - dbs.host = addr.String() + dbs.hostPort = addr.String() dbs.server = dbs.execLocal(addr.Port) case Docker: dbs.server = dbs.execContainer(0, dbs.network) @@ -307,7 +308,7 @@ func (dbs *DBServer) start() { dbs.server.Stdout = &dbs.output dbs.server.Stderr = &dbs.output if dbs.debug { - fmt.Printf("[%s] Starting Mongo instance: %v. Address: %s. Network: '%s'\n", time.Now().String(), dbs.server.Args, dbs.host, dbs.network) + fmt.Printf("[%s] Starting Mongo instance: %v. Address: %s. Network: '%s'\n", time.Now().String(), dbs.server.Args, dbs.hostPort, dbs.network) } err = dbs.server.Start() if err != nil { @@ -317,11 +318,11 @@ func (dbs *DBServer) start() { fmt.Printf("[%s] Mongo instance started\n", time.Now().String()) } if dbs.eType == Docker { - ipAddr, p, err2 := dbs.GetContainerHostPort() + ipAddr, _, err2 := dbs.GetContainerHostPort() if err2 != nil { panic(err2) } - dbs.host = fmt.Sprintf("%s:%d", ipAddr, p) + dbs.hostPort = fmt.Sprintf("%s:%d", ipAddr, 27017) } dbs.tomb.Go(dbs.monitor) dbs.Wipe() @@ -418,9 +419,10 @@ func (dbs *DBServer) SessionWithTimeout(timeout time.Duration) *mgo.Session { if dbs.session == nil { mgo.ResetStats() var err error - dbs.session, err = mgo.DialWithTimeout(dbs.host+"/test", timeout) + fmt.Printf("[%s] Dialing mongod located at '%s'\n", time.Now().String(), dbs.hostPort) + dbs.session, err = mgo.DialWithTimeout(dbs.hostPort+"/test", timeout) if err != nil { - fmt.Fprintf(os.Stderr, "[%s] Unable to dial mongod located at '%s'. Timeout=%v, Error: %s\n", time.Now().String(), dbs.host, timeout, err.Error()) + fmt.Fprintf(os.Stderr, "[%s] Unable to dial mongod located at '%s'. Timeout=%v, Error: %s\n", time.Now().String(), dbs.hostPort, timeout, err.Error()) fmt.Fprintf(os.Stderr, "%s", dbs.output.Bytes()) dbs.printMongoDebugInfo() panic(err) From 498ec5238e160ee47b4830b06f8045f25b34d6d3 Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Sun, 12 May 2019 11:08:56 -0700 Subject: [PATCH 48/69] Add option to attach UT container to a specific docker network. --- dbtest/dbserver.go | 62 +++++++--------------------------------------- 1 file changed, 9 insertions(+), 53 deletions(-) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index 04c628b3a..26469354d 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -3,7 +3,6 @@ package dbtest import ( "bytes" "encoding/hex" - "errors" "fmt" "math/rand" "net" @@ -77,8 +76,8 @@ func (dbs *DBServer) SetNetwork(network string) { } // Start Mongo DB within Docker container on host. -// It assumes Docker is already installed -func (dbs *DBServer) execContainer(port int, network string) *exec.Cmd { +// It assumes Docker is already installed. +func (dbs *DBServer) execContainer(network string) *exec.Cmd { if dbs.version == "" { dbs.version = "latest" } @@ -127,21 +126,9 @@ func (dbs *DBServer) execContainer(port int, network string) *exec.Cmd { } dbs.containerName = fmt.Sprintf("mongo-%s", hex.EncodeToString(u)) - // On some platforms, we get "chown: changing ownership of '/proc/1/fd/1': Permission denied" unless - // we allocate a pseudo tty (-t option) - var portArg string - if port > 0 { - portArg = fmt.Sprintf("%d:%d", port, 27017) - } else { - // Let docker allocate the port. - // This is useful when multiple tests are running on the same host - portArg = fmt.Sprintf("%d", 27017) - } args = []string{ "run", "-t", - "-p", - portArg, "--rm", // Automatically remove the container when it exits } if network != "" { @@ -177,12 +164,10 @@ func (dbs *DBServer) GetHostName() string { } } -// GetContainerHostPort returns the IP address and port for the test Mongo instance -// The Host port is the port which is exposed to the host OS. This is useful if the client tries -// to connect to the Mongo instance from the host. -// If the client connects directly on the docker bridge network (such as when the client is also +// GetContainerIpAddr returns the IP address of the test Mongo instance +// The client should connect directly on the docker bridge network (such as when the client is also // running in a container), then client should connect to port 27017. -func (dbs *DBServer) GetContainerHostPort() (string, int, error) { +func (dbs *DBServer) GetContainerIpAddr() (string, error) { start := time.Now() var err error var stderr bytes.Buffer @@ -202,38 +187,9 @@ func (dbs *DBServer) GetContainerHostPort() (string, int, error) { } ipAddr := strings.Trim(strings.TrimSpace(out.String()), "'") dbs.hostname = ipAddr - - // Get port - out.Reset() - args = []string{"port", dbs.containerName, fmt.Sprintf("%d", 27017)} - cmd = exec.Command("docker", args...) // #nosec - cmd.Stdout = &out - cmd.Stderr = &stderr - err = cmd.Run() - if err != nil { - // This could be because the container has not started yet. Retry later - fmt.Printf("[%s] Failed to get container host port number. Will retry later...\n", time.Now().String()) - time.Sleep(3 * time.Second) - continue - } - s := strings.TrimSpace(out.String()) - o := strings.Split(s, ":") - if len(o) < 2 { - fmt.Printf("Unable to get container host port number: %s", s) - dbs.printMongoDebugInfo() - return "", -1, errors.New(fmt.Sprintf("Unable to get container host port number. %s", s)) - } - i, err2 := strconv.Atoi(o[1]) - if err2 != nil { - dbs.printMongoDebugInfo() - fmt.Printf("[%s] Unable to parse port number: error=%s, out=%s\n", time.Now().String(), err2.Error(), o[1]) - } else { - fmt.Printf("[%s] MongoDB available on bridge network at: %s. Host port: %d. Container port: %d\n", time.Now().String(), ipAddr, i, 27017) - } - return ipAddr, i, err2 + return dbs.hostname, err } - fmt.Printf("[%s] Failed to run command. error=%s, stderr=%s\n", time.Now().String(), err.Error(), stderr.String()) - return "", -1, err + return "", fmt.Errorf("[%s] Failed to run command. error=%s, stderr=%s\n", time.Now().String(), err.Error(), stderr.String()) } // Stop the docker container running Mongo. @@ -301,7 +257,7 @@ func (dbs *DBServer) start() { dbs.hostPort = addr.String() dbs.server = dbs.execLocal(addr.Port) case Docker: - dbs.server = dbs.execContainer(0, dbs.network) + dbs.server = dbs.execContainer(dbs.network) default: panic(fmt.Sprintf("unsupported exec type: %d", dbs.eType)) } @@ -318,7 +274,7 @@ func (dbs *DBServer) start() { fmt.Printf("[%s] Mongo instance started\n", time.Now().String()) } if dbs.eType == Docker { - ipAddr, _, err2 := dbs.GetContainerHostPort() + ipAddr, err2 := dbs.GetContainerIpAddr() if err2 != nil { panic(err2) } From 0f9e67fe5b9d430d81dca2a254bc2cae3118dc23 Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Mon, 13 May 2019 06:17:26 -0700 Subject: [PATCH 49/69] Add option to attach UT container to a specific docker network. --- dbtest/dbserver.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index 26469354d..8f3927c7c 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -164,6 +164,11 @@ func (dbs *DBServer) GetHostName() string { } } +// GetContainerName returns the name of the container, when running the Mongo UT instance in a container. +func (dbs *DBServer) GetContainerName() string { + return dbs.containerName +} + // GetContainerIpAddr returns the IP address of the test Mongo instance // The client should connect directly on the docker bridge network (such as when the client is also // running in a container), then client should connect to port 27017. From e1f5322de264a8d1dd7b86485583f2ba2b6e400c Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Mon, 3 Jun 2019 14:38:14 -0700 Subject: [PATCH 50/69] Add knob to expose container port to host --- dbtest/dbserver.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index 8f3927c7c..3efbd00d2 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -43,6 +43,7 @@ type DBServer struct { version string // The request MongoDB version, when running within a container eType int // Specify whether mongo should run as a container or regular process network string // The name of the docker network to which the UT container should be attached + exposePort bool // Specify whether container port should be exposed to the host OS. debug bool // Log debug statements containerName string // The container name, when running mgo within a container tomb tomb.Tomb @@ -75,9 +76,14 @@ func (dbs *DBServer) SetNetwork(network string) { dbs.network = network } +// SetExposePort sets whether the container port should be exposed to the host OS. +func (dbs *DBServer) SetExposePort(exposePort bool) { + dbs.exposePort = exposePort +} + // Start Mongo DB within Docker container on host. // It assumes Docker is already installed. -func (dbs *DBServer) execContainer(network string) *exec.Cmd { +func (dbs *DBServer) execContainer(network string, exposePort bool) *exec.Cmd { if dbs.version == "" { dbs.version = "latest" } @@ -137,6 +143,9 @@ func (dbs *DBServer) execContainer(network string) *exec.Cmd { network, }...) } + if exposePort { + args = append(args, []string{"-p", fmt.Sprintf("%d:%d", 27017, 27017)}...) + } args = append(args, []string{ "--name", dbs.containerName, @@ -262,7 +271,7 @@ func (dbs *DBServer) start() { dbs.hostPort = addr.String() dbs.server = dbs.execLocal(addr.Port) case Docker: - dbs.server = dbs.execContainer(dbs.network) + dbs.server = dbs.execContainer(dbs.network, dbs.exposePort) default: panic(fmt.Sprintf("unsupported exec type: %d", dbs.eType)) } From f1918824adbca85ea43ef3f003fd0c65ecbf4a06 Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Mon, 3 Jun 2019 15:39:56 -0700 Subject: [PATCH 51/69] Add knob to expose container port to host --- dbtest/dbserver.go | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index 3efbd00d2..68485fc61 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -39,11 +39,11 @@ type DBServer struct { server *exec.Cmd dbpath string hostPort string // The IP address and port number of the mgo instance. - hostname string // The IP address or hostname of the container. + hostname string // The IP address or hostname of the container. version string // The request MongoDB version, when running within a container eType int // Specify whether mongo should run as a container or regular process network string // The name of the docker network to which the UT container should be attached - exposePort bool // Specify whether container port should be exposed to the host OS. + exposePort bool // Specify whether container port should be exposed to the host OS. debug bool // Log debug statements containerName string // The container name, when running mgo within a container tomb tomb.Tomb @@ -143,9 +143,9 @@ func (dbs *DBServer) execContainer(network string, exposePort bool) *exec.Cmd { network, }...) } - if exposePort { + if exposePort { args = append(args, []string{"-p", fmt.Sprintf("%d:%d", 27017, 27017)}...) - } + } args = append(args, []string{ "--name", dbs.containerName, @@ -175,13 +175,16 @@ func (dbs *DBServer) GetHostName() string { // GetContainerName returns the name of the container, when running the Mongo UT instance in a container. func (dbs *DBServer) GetContainerName() string { - return dbs.containerName + return dbs.containerName } // GetContainerIpAddr returns the IP address of the test Mongo instance -// The client should connect directly on the docker bridge network (such as when the client is also +// The client should connect directly on the docker bridge network (such as when the client is also // running in a container), then client should connect to port 27017. func (dbs *DBServer) GetContainerIpAddr() (string, error) { + if dbs.network == "" { + return "127.0.0.1", nil + } start := time.Now() var err error var stderr bytes.Buffer @@ -389,7 +392,7 @@ func (dbs *DBServer) SessionWithTimeout(timeout time.Duration) *mgo.Session { if dbs.session == nil { mgo.ResetStats() var err error - fmt.Printf("[%s] Dialing mongod located at '%s'\n", time.Now().String(), dbs.hostPort) + fmt.Printf("[%s] Dialing mongod located at '%s'\n", time.Now().String(), dbs.hostPort) dbs.session, err = mgo.DialWithTimeout(dbs.hostPort+"/test", timeout) if err != nil { fmt.Fprintf(os.Stderr, "[%s] Unable to dial mongod located at '%s'. Timeout=%v, Error: %s\n", time.Now().String(), dbs.hostPort, timeout, err.Error()) From 90de89a8c8bcfb203234fa59fb99b3dfe4703b67 Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Mon, 3 Jun 2019 16:22:12 -0700 Subject: [PATCH 52/69] Add knob to expose container port to host --- dbtest/dbserver.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index 68485fc61..374ec9b03 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -182,9 +182,6 @@ func (dbs *DBServer) GetContainerName() string { // The client should connect directly on the docker bridge network (such as when the client is also // running in a container), then client should connect to port 27017. func (dbs *DBServer) GetContainerIpAddr() (string, error) { - if dbs.network == "" { - return "127.0.0.1", nil - } start := time.Now() var err error var stderr bytes.Buffer @@ -204,7 +201,11 @@ func (dbs *DBServer) GetContainerIpAddr() (string, error) { } ipAddr := strings.Trim(strings.TrimSpace(out.String()), "'") dbs.hostname = ipAddr - return dbs.hostname, err + if dbs.network == "" { + return "127.0.0.1", nil + } else { + return dbs.hostname, err + } } return "", fmt.Errorf("[%s] Failed to run command. error=%s, stderr=%s\n", time.Now().String(), err.Error(), stderr.String()) } From 3151693cb57e328f5057e8ef49f7d03cb1306b6b Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Mon, 3 Jun 2019 17:05:17 -0700 Subject: [PATCH 53/69] Add knob to expose container port to host --- dbtest/dbserver.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index 374ec9b03..661b837ce 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -186,6 +186,10 @@ func (dbs *DBServer) GetContainerIpAddr() (string, error) { var err error var stderr bytes.Buffer for time.Since(start) < 60*time.Second { + if dbs.server.ProcessState != nil { + // The process has exited + return "", fmt.Errorf("Process has exited\n%s", dbs.output.String()) + } stderr.Reset() args := []string{"inspect", "-f", "'{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}'", dbs.containerName} cmd := exec.Command("docker", args...) // #nosec @@ -195,7 +199,7 @@ func (dbs *DBServer) GetContainerIpAddr() (string, error) { err = cmd.Run() if err != nil { // This could be because the container has not started yet. Retry later - fmt.Printf("[%s] Failed to get container IP address. Will retry later...\n", time.Now().String()) + fmt.Printf("[%s] Failed to get container IP address. Will retry later.\n", time.Now().String()) time.Sleep(3 * time.Second) continue } @@ -291,6 +295,13 @@ func (dbs *DBServer) start() { if dbs.debug { fmt.Printf("[%s] Mongo instance started\n", time.Now().String()) } + go func() { + // Call Wait() so cmd.ProcessState is set after command has completed. + err = dbs.server.Wait() + if err != nil { + fmt.Printf("[%s] Command exited\n", time.Now().String()) + } + }() if dbs.eType == Docker { ipAddr, err2 := dbs.GetContainerIpAddr() if err2 != nil { From 8057a007910817e3587aaf0a3d76a3f358b93b98 Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Tue, 4 Jun 2019 09:53:29 -0700 Subject: [PATCH 54/69] Add knob to expose container port to host --- dbtest/dbserver.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index 661b837ce..9ea5bdc5d 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -188,7 +188,8 @@ func (dbs *DBServer) GetContainerIpAddr() (string, error) { for time.Since(start) < 60*time.Second { if dbs.server.ProcessState != nil { // The process has exited - return "", fmt.Errorf("Process has exited\n%s", dbs.output.String()) + fmt.Printf("[%s] Mongo container has exited unexpectedly. Output:\n%s\n", time.Now().String(), dbs.output.String()) + return "", fmt.Errorf("Process has exited") } stderr.Reset() args := []string{"inspect", "-f", "'{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}'", dbs.containerName} @@ -299,7 +300,7 @@ func (dbs *DBServer) start() { // Call Wait() so cmd.ProcessState is set after command has completed. err = dbs.server.Wait() if err != nil { - fmt.Printf("[%s] Command exited\n", time.Now().String()) + fmt.Printf("[%s] Command exited. Output:\n%s\n", time.Now().String(), dbs.output.String()) } }() if dbs.eType == Docker { From 3a88debd0fddfc979303ff2e0c8ce0d069bc1ade Mon Sep 17 00:00:00 2001 From: "Paul Fraley (pfraley)" Date: Wed, 3 Jul 2019 15:04:50 -0700 Subject: [PATCH 55/69] bson: add WithContext versions of API functions - Marshal - if GetBSON() returned the same object it was passed, encode.go would turn around and call add*() function on that same object causing an infinite loop. Using NewContextWithSkipCustom() avoids infinite loop (avoids calling GetBSON() again). Returning the same object is useful in the case where GetBSON(): - obtains bson.Raw from fields hidden (annotations) from the standard marshaler - sets bson.Raw fields visible (annotations) to the standard marshaler - returns the same object such that standard marshalling can occur - context.Context can be used to control how fields are marshaled - Unmarshal - calling Unmarshal on the same object from SetBSON() would cause infinite loop - fix: pass bson.NewContextWithSkipCustom(ctx, obj) to UnmarshalWithContext() - context.Context can be used to control how fields are unmarshaled --- bson/bson.go | 35 +++++++++--- bson/context.go | 121 ++++++++++++++++++++++++++++++++++++++++ bson/decode.go | 144 ++++++++++++++++++++++++++++++++---------------- bson/encode.go | 119 +++++++++++++++++++++++++++------------ 4 files changed, 330 insertions(+), 89 deletions(-) create mode 100644 bson/context.go diff --git a/bson/bson.go b/bson/bson.go index 7fb7f8cae..733e94a65 100644 --- a/bson/bson.go +++ b/bson/bson.go @@ -34,6 +34,7 @@ package bson import ( "bytes" + "context" "crypto/md5" "crypto/rand" "encoding/binary" @@ -64,6 +65,10 @@ type Getter interface { GetBSON() (interface{}, error) } +type GetterCtx interface { + GetBSONWithContext(context.Context) (interface{}, error) +} + // A value implementing the bson.Setter interface will receive the BSON // value via the SetBSON method during unmarshaling, and the object // itself will not be changed as usual. @@ -95,6 +100,10 @@ type Setter interface { SetBSON(raw Raw) error } +type SetterCtx interface { + SetBSONWithContext(ctx context.Context, raw Raw) error +} + // SetZero may be returned from a SetBSON method to have the value set to // its respective zero value. When used in pointer values, this will set the // field to nil rather than to the pre-allocated value. @@ -279,7 +288,7 @@ var nullBytes = []byte("null") func (id *ObjectId) UnmarshalJSON(data []byte) error { if len(data) > 0 && (data[0] == '{' || data[0] == 'O') { var v struct { - Id json.RawMessage `json:"$oid"` + Id json.RawMessage `json:"$oid"` Func struct { Id json.RawMessage } `json:"$oidFunc"` @@ -505,13 +514,17 @@ func handleErr(err *error) { // F int64 "myf,omitempty,minsize" // } // -func Marshal(in interface{}) (out []byte, err error) { +func MarshalWithContext(ctx context.Context, in interface{}) (out []byte, err error) { defer handleErr(&err) e := &encoder{make([]byte, 0, initialBufferSize)} - e.addDoc(reflect.ValueOf(in)) + e.addDoc(ctx, reflect.ValueOf(in)) return e.out, nil } +func Marshal(in interface{}) (out []byte, err error) { + return MarshalWithContext(context.TODO(), in) +} + // Unmarshal deserializes data from in into the out value. The out value // must be a map, a pointer to a struct, or a pointer to a bson.D value. // In the case of struct values, only exported fields will be deserialized. @@ -547,7 +560,7 @@ func Marshal(in interface{}) (out []byte, err error) { // silently skipped. // // Pointer values are initialized when necessary. -func Unmarshal(in []byte, out interface{}) (err error) { +func UnmarshalWithContext(ctx context.Context, in []byte, out interface{}) (err error) { if raw, ok := out.(*Raw); ok { raw.Kind = 3 raw.Data = in @@ -560,7 +573,7 @@ func Unmarshal(in []byte, out interface{}) (err error) { fallthrough case reflect.Map: d := newDecoder(in) - d.readDocTo(v) + d.readDocTo(ctx, v) case reflect.Struct: return errors.New("Unmarshal can't deal with struct values. Use a pointer.") default: @@ -569,12 +582,16 @@ func Unmarshal(in []byte, out interface{}) (err error) { return nil } +func Unmarshal(in []byte, out interface{}) (err error) { + return UnmarshalWithContext(context.TODO(), in, out) +} + // Unmarshal deserializes raw into the out value. If the out value type // is not compatible with raw, a *bson.TypeError is returned. // // See the Unmarshal function documentation for more details on the // unmarshalling process. -func (raw Raw) Unmarshal(out interface{}) (err error) { +func (raw Raw) UnmarshalWithContext(ctx context.Context, out interface{}) (err error) { defer handleErr(&err) v := reflect.ValueOf(out) switch v.Kind() { @@ -583,7 +600,7 @@ func (raw Raw) Unmarshal(out interface{}) (err error) { fallthrough case reflect.Map: d := newDecoder(raw.Data) - good := d.readElemTo(v, raw.Kind) + good := d.readElemTo(ctx, v, raw.Kind) if !good { return &TypeError{v.Type(), raw.Kind} } @@ -595,6 +612,10 @@ func (raw Raw) Unmarshal(out interface{}) (err error) { return nil } +func (raw Raw) Unmarshal(out interface{}) (err error) { + return raw.UnmarshalWithContext(context.TODO(), out) +} + type TypeError struct { Type reflect.Type Kind byte diff --git a/bson/context.go b/bson/context.go new file mode 100644 index 000000000..70a062603 --- /dev/null +++ b/bson/context.go @@ -0,0 +1,121 @@ +// BSON library for Go +// +// Copyright (c) 2010-2012 - Gustavo Niemeyer +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// gobson - BSON library for Go. +// + +package bson + +import ( + "context" + "fmt" + "reflect" + "strings" +) + +type bsonOptions struct { + // skipCustom is used by + // - decode.go to skip looking for a custom SetBSON() or SetBSONWithContext() + // - encode.go to skip looking for a custom GetBSON() or GetBSONWithContext() + // for any type with the base type name specified by 'skipCustom'. + // This is useful to avoid infinite loop caused by: + // - calling Unmarshal from custom SetBSON function (decode.go) + // - encode.go calling custom GetBSON after just calling custom GetBSON for a given type + skipCustom string +} + +type key int + +var bsonKey key = 0 + +// Returns the topmost bsonOptions value stored in ctx, if any. +func fromContext(ctx context.Context) (*bsonOptions, bool) { + if ctx == nil { + return nil, false + } + opts, ok := ctx.Value(bsonKey).(*bsonOptions) + return opts, ok +} + +// Returns the base type name (type name without a prefix that contains any combination of * or []). +func baseTypeName(typ reflect.Type) string { + return strings.Trim(fmt.Sprintf("%v", typ), "*[]") +} + +// Creates a new context with a value for skipCustom based on base type name of valu. +func NewContextWithSkipCustom(ctx context.Context, valu interface{}) context.Context { + if ctx == nil { + ctx = context.Background() + } + //Debug(0, fmt.Sprintf("NewContextWithSkipCustom: %s", baseTypeName(reflect.TypeOf(valu)))) + return context.WithValue(ctx, bsonKey, &bsonOptions{skipCustom: baseTypeName(reflect.TypeOf(valu))}) +} + +// IsSkipCustom is useful to avoid infinite loop caused by: +// - calling Unmarshal from custom SetBSON function (decode.go) +// - encode.go calling custom GetBSON after just calling custom GetBSON for a given type +// +// Returns true if base type name of typ is the same as skipCustom. +// +// This method is used to skip all custom SetBSON/GetBSON functions of all types with the same base type. +// Note: if goal is to skip the custom functions of certain variations of a base type, +// skipCustom will not work (it will skip all variants). +func IsSkipCustom(ctx context.Context, typ reflect.Type) bool { + if opts, _ := fromContext(ctx); opts != nil { + //Debug(0, fmt.Sprintf("IsSkipCustom(%s) opts(%s) base(%s) %v", typ, opts.skipCustom, + // baseTypeName(typ), opts.skipCustom == baseTypeName(typ))) + return opts.skipCustom == baseTypeName(typ) + } + //Debug(0, fmt.Sprintf("IsSkipCustom(%s) false", typ)) + return false +} + +// These were useful in debugging infinite loop issues: +// - calling Unmarshal from custom SetBSON function (decode.go) +// - encode.go calling custom GetBSON after just calling custom GetBSON for a given type +// uncomment them (along with the invocations of Debug()) to aid debugging. +//func valueType(valu reflect.Value) string { +// if !valu.IsValid() { +// return "Invalid" +// } +// return fmt.Sprintf("%s", valu.Type()) +//} +// +//var currentIndentLevel int = 0 +//var infinteLoopCounter int = 10000 +// +//func Debug(delta int, msg string) { +// infinteLoopCounter-- +// if infinteLoopCounter < 0 { +// panic(fmt.Errorf("YOU ARE IN INFINITE LOOP")) +// } +// if delta > 0 { +// currentIndentLevel += delta +// } +// fmt.Printf("%s%s\n", strings.Repeat(" ", currentIndentLevel), msg) +// if delta < 0 { +// currentIndentLevel += delta +// } +//} diff --git a/bson/decode.go b/bson/decode.go index 505eb48ef..a8324897a 100644 --- a/bson/decode.go +++ b/bson/decode.go @@ -28,13 +28,14 @@ package bson import ( + "context" "fmt" "math" "net/url" "reflect" + "runtime" "strconv" "sync" - "runtime" "time" ) @@ -72,26 +73,38 @@ const ( setterNone setterType setterAddr + setterCtxType + setterCtxAddr ) var setterStyles map[reflect.Type]int var setterIface reflect.Type +var setterCtxIface reflect.Type var setterMutex sync.RWMutex func init() { var iface Setter + var ifaceCtx SetterCtx setterIface = reflect.TypeOf(&iface).Elem() + setterCtxIface = reflect.TypeOf(&ifaceCtx).Elem() setterStyles = make(map[reflect.Type]int) } -func setterStyle(outt reflect.Type) int { +func setterStyle(ctx context.Context, outt reflect.Type) int { + if IsSkipCustom(ctx, outt) { + return setterNone + } setterMutex.RLock() style := setterStyles[outt] setterMutex.RUnlock() if style == setterUnknown { setterMutex.Lock() defer setterMutex.Unlock() - if outt.Implements(setterIface) { + if outt.Implements(setterCtxIface) { + setterStyles[outt] = setterCtxType + } else if reflect.PtrTo(outt).Implements(setterCtxIface) { + setterStyles[outt] = setterCtxAddr + } else if outt.Implements(setterIface) { setterStyles[outt] = setterType } else if reflect.PtrTo(outt).Implements(setterIface) { setterStyles[outt] = setterAddr @@ -103,9 +116,9 @@ func setterStyle(outt reflect.Type) int { return style } -func getSetter(outt reflect.Type, out reflect.Value) Setter { - style := setterStyle(outt) - if style == setterNone { +func getSetter(ctx context.Context, outt reflect.Type, out reflect.Value) Setter { + style := setterStyle(ctx, outt) + if style != setterType && style != setterAddr { return nil } if style == setterAddr { @@ -119,6 +132,22 @@ func getSetter(outt reflect.Type, out reflect.Value) Setter { return out.Interface().(Setter) } +func getSetterCtx(ctx context.Context, outt reflect.Type, out reflect.Value) SetterCtx { + style := setterStyle(ctx, outt) + if style != setterCtxType && style != setterCtxAddr { + return nil + } + if style == setterCtxAddr { + if !out.CanAddr() { + return nil + } + out = out.Addr() + } else if outt.Kind() == reflect.Ptr && out.IsNil() { + out.Set(reflect.New(outt.Elem())) + } + return out.Interface().(SetterCtx) +} + func clearMap(m reflect.Value) { var none reflect.Value for _, k := range m.MapKeys() { @@ -126,7 +155,7 @@ func clearMap(m reflect.Value) { } } -func (d *decoder) readDocTo(out reflect.Value) { +func (d *decoder) readDocTo(ctx context.Context, out reflect.Value) { var elemType reflect.Type outt := out.Type() outk := outt.Kind() @@ -135,9 +164,18 @@ func (d *decoder) readDocTo(out reflect.Value) { if outk == reflect.Ptr && out.IsNil() { out.Set(reflect.New(outt.Elem())) } - if setter := getSetter(outt, out); setter != nil { + if setterCtx := getSetterCtx(ctx, outt, out); setterCtx != nil { + var raw Raw + d.readDocTo(ctx, reflect.ValueOf(&raw)) + err := setterCtx.SetBSONWithContext(ctx, raw) + if _, ok := err.(*TypeError); err != nil && !ok { + panic(err) + } + return + } + if setter := getSetter(ctx, outt, out); setter != nil { var raw Raw - d.readDocTo(reflect.ValueOf(&raw)) + d.readDocTo(ctx, reflect.ValueOf(&raw)) err := setter.SetBSON(raw) if _, ok := err.(*TypeError); err != nil && !ok { panic(err) @@ -215,10 +253,10 @@ func (d *decoder) readDocTo(out reflect.Value) { case reflect.Slice: switch outt.Elem() { case typeDocElem: - origout.Set(d.readDocElems(outt)) + origout.Set(d.readDocElems(ctx, outt)) return case typeRawDocElem: - origout.Set(d.readRawDocElems(outt)) + origout.Set(d.readRawDocElems(ctx, outt)) return } fallthrough @@ -241,7 +279,7 @@ func (d *decoder) readDocTo(out reflect.Value) { switch outk { case reflect.Map: e := reflect.New(elemType).Elem() - if d.readElemTo(e, kind) { + if d.readElemTo(ctx, e, kind) { k := reflect.ValueOf(name) if convertKey { k = k.Convert(keyType) @@ -250,24 +288,24 @@ func (d *decoder) readDocTo(out reflect.Value) { } case reflect.Struct: if outt == typeRaw { - d.dropElem(kind) + d.dropElem(ctx, kind) } else { if info, ok := fieldsMap[name]; ok { if info.Inline == nil { - d.readElemTo(out.Field(info.Num), kind) + d.readElemTo(ctx, out.Field(info.Num), kind) } else { - d.readElemTo(out.FieldByIndex(info.Inline), kind) + d.readElemTo(ctx, out.FieldByIndex(info.Inline), kind) } } else if inlineMap.IsValid() { if inlineMap.IsNil() { inlineMap.Set(reflect.MakeMap(inlineMap.Type())) } e := reflect.New(elemType).Elem() - if d.readElemTo(e, kind) { + if d.readElemTo(ctx, e, kind) { inlineMap.SetMapIndex(reflect.ValueOf(name), e) } } else { - d.dropElem(kind) + d.dropElem(ctx, kind) } } case reflect.Slice: @@ -288,7 +326,7 @@ func (d *decoder) readDocTo(out reflect.Value) { } } -func (d *decoder) readArrayDocTo(out reflect.Value) { +func (d *decoder) readArrayDocTo(ctx context.Context, out reflect.Value) { end := int(d.readInt32()) end += d.i - 4 if end <= d.i || end > len(d.in) || d.in[end-1] != '\x00' { @@ -308,7 +346,7 @@ func (d *decoder) readArrayDocTo(out reflect.Value) { corrupted() } d.i++ - d.readElemTo(out.Index(i), kind) + d.readElemTo(ctx, out.Index(i), kind) if d.i >= end { corrupted() } @@ -323,11 +361,11 @@ func (d *decoder) readArrayDocTo(out reflect.Value) { } } -func (d *decoder) readSliceDoc(t reflect.Type) interface{} { +func (d *decoder) readSliceDoc(ctx context.Context, t reflect.Type) interface{} { tmp := make([]reflect.Value, 0, 8) elemType := t.Elem() if elemType == typeRawDocElem { - d.dropElem(0x04) + d.dropElem(ctx, 0x04) return reflect.Zero(t).Interface() } @@ -346,7 +384,7 @@ func (d *decoder) readSliceDoc(t reflect.Type) interface{} { } d.i++ e := reflect.New(elemType).Elem() - if d.readElemTo(e, kind) { + if d.readElemTo(ctx, e, kind) { tmp = append(tmp, e) } if d.i >= end { @@ -369,14 +407,14 @@ func (d *decoder) readSliceDoc(t reflect.Type) interface{} { var typeSlice = reflect.TypeOf([]interface{}{}) var typeIface = typeSlice.Elem() -func (d *decoder) readDocElems(typ reflect.Type) reflect.Value { +func (d *decoder) readDocElems(ctx context.Context, typ reflect.Type) reflect.Value { docType := d.docType d.docType = typ slice := make([]DocElem, 0, 8) d.readDocWith(func(kind byte, name string) { e := DocElem{Name: name} v := reflect.ValueOf(&e.Value) - if d.readElemTo(v.Elem(), kind) { + if d.readElemTo(ctx, v.Elem(), kind) { slice = append(slice, e) } }) @@ -386,14 +424,14 @@ func (d *decoder) readDocElems(typ reflect.Type) reflect.Value { return slicev } -func (d *decoder) readRawDocElems(typ reflect.Type) reflect.Value { +func (d *decoder) readRawDocElems(ctx context.Context, typ reflect.Type) reflect.Value { docType := d.docType d.docType = typ slice := make([]RawDocElem, 0, 8) d.readDocWith(func(kind byte, name string) { e := RawDocElem{Name: name} v := reflect.ValueOf(&e.Value) - if d.readElemTo(v.Elem(), kind) { + if d.readElemTo(ctx, v.Elem(), kind) { slice = append(slice, e) } }) @@ -431,14 +469,14 @@ func (d *decoder) readDocWith(f func(kind byte, name string)) { var blackHole = settableValueOf(struct{}{}) -func (d *decoder) dropElem(kind byte) { - d.readElemTo(blackHole, kind) +func (d *decoder) dropElem(ctx context.Context, kind byte) { + d.readElemTo(ctx, blackHole, kind) } // Attempt to decode an element from the document and put it into out. // If the types are not compatible, the returned ok value will be // false and out will be unchanged. -func (d *decoder) readElemTo(out reflect.Value, kind byte) (good bool) { +func (d *decoder) readElemTo(ctx context.Context, out reflect.Value, kind byte) (good bool) { start := d.i @@ -448,25 +486,25 @@ func (d *decoder) readElemTo(out reflect.Value, kind byte) (good bool) { outk := out.Kind() switch outk { case reflect.Interface, reflect.Ptr, reflect.Struct, reflect.Map: - d.readDocTo(out) + d.readDocTo(ctx, out) return true } - if setterStyle(outt) != setterNone { - d.readDocTo(out) + if setterStyle(ctx, outt) != setterNone { + d.readDocTo(ctx, out) return true } if outk == reflect.Slice { switch outt.Elem() { case typeDocElem: - out.Set(d.readDocElems(outt)) + out.Set(d.readDocElems(ctx, outt)) case typeRawDocElem: - out.Set(d.readRawDocElems(outt)) + out.Set(d.readRawDocElems(ctx, outt)) default: - d.readDocTo(blackHole) + d.readDocTo(ctx, blackHole) } return true } - d.readDocTo(blackHole) + d.readDocTo(ctx, blackHole) return true } @@ -481,9 +519,9 @@ func (d *decoder) readElemTo(out reflect.Value, kind byte) (good bool) { panic("Can't happen. Handled above.") case 0x04: // Array outt := out.Type() - if setterStyle(outt) != setterNone { + if setterStyle(ctx, outt) != setterNone { // Skip the value so its data is handed to the setter below. - d.dropElem(kind) + d.dropElem(ctx, kind) break } for outt.Kind() == reflect.Ptr { @@ -491,12 +529,12 @@ func (d *decoder) readElemTo(out reflect.Value, kind byte) (good bool) { } switch outt.Kind() { case reflect.Array: - d.readArrayDocTo(out) + d.readArrayDocTo(ctx, out) return true case reflect.Slice: - in = d.readSliceDoc(outt) + in = d.readSliceDoc(ctx, outt) default: - in = d.readSliceDoc(typeSlice) + in = d.readSliceDoc(ctx, typeSlice) } case 0x05: // Binary b := d.readBinary() @@ -532,7 +570,7 @@ func (d *decoder) readElemTo(out reflect.Value, kind byte) (good bool) { case 0x0F: // JavaScript with scope d.i += 4 // Skip length js := JavaScript{d.readStr(), make(M)} - d.readDocTo(reflect.ValueOf(js.Scope)) + d.readDocTo(ctx, reflect.ValueOf(js.Scope)) in = js case 0x10: // Int32 in = int(d.readInt32()) @@ -550,9 +588,9 @@ func (d *decoder) readElemTo(out reflect.Value, kind byte) (good bool) { case 0xFF: // Min key in = MinKey default: - var st []byte = make([]byte, 4096) - w := runtime.Stack(st, false) - panic(fmt.Sprintf("Unknown element kind (0x%02X) BT: %s", kind, string(st[:w]))) + var st []byte = make([]byte, 4096) + w := runtime.Stack(st, false) + panic(fmt.Sprintf("Unknown element kind (0x%02X) BT: %s", kind, string(st[:w]))) } outt := out.Type() @@ -562,7 +600,21 @@ func (d *decoder) readElemTo(out reflect.Value, kind byte) (good bool) { return true } - if setter := getSetter(outt, out); setter != nil { + if setterCtx := getSetterCtx(ctx, outt, out); setterCtx != nil { + err := setterCtx.SetBSONWithContext(ctx, Raw{kind, d.in[start:d.i]}) + if err == SetZero { + out.Set(reflect.Zero(outt)) + return true + } + if err == nil { + return true + } + if _, ok := err.(*TypeError); !ok { + panic(err) + } + return false + } + if setter := getSetter(ctx, outt, out); setter != nil { err := setter.SetBSON(Raw{kind, d.in[start:d.i]}) if err == SetZero { out.Set(reflect.Zero(outt)) diff --git a/bson/encode.go b/bson/encode.go index add39e865..ed93b940f 100644 --- a/bson/encode.go +++ b/bson/encode.go @@ -28,6 +28,7 @@ package bson import ( + "context" "encoding/json" "fmt" "math" @@ -81,15 +82,35 @@ type encoder struct { out []byte } -func (e *encoder) addDoc(v reflect.Value) { +func (e *encoder) addDoc(ctx context.Context, v reflect.Value) { + //cnt := 0 + //Debug(4, fmt.Sprintf("addDoc(%v)-A", valueType(v))) + //defer func() { + // Debug(-4, fmt.Sprintf("addDoc(%v)-Z", valueType(v))) + //}() for { - if vi, ok := v.Interface().(Getter); ok { - getv, err := vi.GetBSON() - if err != nil { - panic(err) + //cnt += 1 + if !IsSkipCustom(ctx, v.Type()) { + if vi, ok := v.Interface().(GetterCtx); ok { + //Debug(0, fmt.Sprintf("addDoc(%02d)-GetterCtx(%v)", cnt, valueType(v))) + getv, err := vi.GetBSONWithContext(ctx) + if err != nil { + panic(err) + } + v = reflect.ValueOf(getv) + ctx = NewContextWithSkipCustom(ctx, getv) + continue + } + if vi, ok := v.Interface().(Getter); ok { + //Debug(0, fmt.Sprintf("addDoc(%02d)-Getter(%v)", cnt, valueType(v))) + getv, err := vi.GetBSON() + if err != nil { + panic(err) + } + v = reflect.ValueOf(getv) + ctx = NewContextWithSkipCustom(ctx, getv) + continue } - v = reflect.ValueOf(getv) - continue } if v.Kind() == reflect.Ptr { v = v.Elem() @@ -97,6 +118,7 @@ func (e *encoder) addDoc(v reflect.Value) { } break } + //Debug(0, fmt.Sprintf("addDoc(%v)-B", valueType(v))) if v.Type() == typeRaw { raw := v.Interface().(Raw) @@ -110,15 +132,16 @@ func (e *encoder) addDoc(v reflect.Value) { return } + //Debug(0, fmt.Sprintf("addDoc(%v)-C", valueType(v))) start := e.reserveInt32() switch v.Kind() { case reflect.Map: - e.addMap(v) + e.addMap(ctx, v) case reflect.Struct: - e.addStruct(v) + e.addStruct(ctx, v) case reflect.Array, reflect.Slice: - e.addSlice(v) + e.addSlice(ctx, v) default: panic("Can't marshal " + v.Type().String() + " as a BSON document") } @@ -127,13 +150,13 @@ func (e *encoder) addDoc(v reflect.Value) { e.setInt32(start, int32(len(e.out)-start)) } -func (e *encoder) addMap(v reflect.Value) { +func (e *encoder) addMap(ctx context.Context, v reflect.Value) { for _, k := range v.MapKeys() { - e.addElem(k.String(), v.MapIndex(k), false) + e.addElem(ctx, k.String(), v.MapIndex(k), false) } } -func (e *encoder) addStruct(v reflect.Value) { +func (e *encoder) addStruct(ctx context.Context, v reflect.Value) { sinfo, err := getStructInfo(v.Type()) if err != nil { panic(err) @@ -147,7 +170,7 @@ func (e *encoder) addStruct(v reflect.Value) { if _, found := sinfo.FieldsMap[ks]; found { panic(fmt.Sprintf("Can't have key %q in inlined map; conflicts with struct field", ks)) } - e.addElem(ks, m.MapIndex(k), false) + e.addElem(ctx, ks, m.MapIndex(k), false) } } } @@ -160,7 +183,7 @@ func (e *encoder) addStruct(v reflect.Value) { if info.OmitEmpty && isZero(value) { continue } - e.addElem(info.Key, value, info.MinSize) + e.addElem(ctx, info.Key, value, info.MinSize) } } @@ -200,17 +223,21 @@ func isZero(v reflect.Value) bool { return false } -func (e *encoder) addSlice(v reflect.Value) { +func (e *encoder) addSlice(ctx context.Context, v reflect.Value) { + //Debug(4, fmt.Sprintf("addSlice(%v)-A", valueType(v))) + //defer func() { + // Debug(-4, fmt.Sprintf("addSlice(%v)-Z", valueType(v))) + //}() vi := v.Interface() if d, ok := vi.(D); ok { for _, elem := range d { - e.addElem(elem.Name, reflect.ValueOf(elem.Value), false) + e.addElem(ctx, elem.Name, reflect.ValueOf(elem.Value), false) } return } if d, ok := vi.(RawD); ok { for _, elem := range d { - e.addElem(elem.Name, reflect.ValueOf(elem.Value), false) + e.addElem(ctx, elem.Name, reflect.ValueOf(elem.Value), false) } return } @@ -219,19 +246,19 @@ func (e *encoder) addSlice(v reflect.Value) { if et == typeDocElem { for i := 0; i < l; i++ { elem := v.Index(i).Interface().(DocElem) - e.addElem(elem.Name, reflect.ValueOf(elem.Value), false) + e.addElem(ctx, elem.Name, reflect.ValueOf(elem.Value), false) } return } if et == typeRawDocElem { for i := 0; i < l; i++ { elem := v.Index(i).Interface().(RawDocElem) - e.addElem(elem.Name, reflect.ValueOf(elem.Value), false) + e.addElem(ctx, elem.Name, reflect.ValueOf(elem.Value), false) } return } for i := 0; i < l; i++ { - e.addElem(itoa(i), v.Index(i), false) + e.addElem(ctx, itoa(i), v.Index(i), false) } } @@ -244,29 +271,49 @@ func (e *encoder) addElemName(kind byte, name string) { e.addBytes(0) } -func (e *encoder) addElem(name string, v reflect.Value, minSize bool) { +func (e *encoder) addElem(ctx context.Context, name string, v reflect.Value, minSize bool) { + //Debug(4, fmt.Sprintf("addElem(%v)[%v]-A", valueType(v), name)) + //defer func() { + // Debug(-4, fmt.Sprintf("addElem(%v)[%v]-Z", valueType(v), name)) + //}() if !v.IsValid() { e.addElemName(0x0A, name) return } - if getter, ok := v.Interface().(Getter); ok { - getv, err := getter.GetBSON() - if err != nil { - panic(err) + //Debug(0, fmt.Sprintf("addElem(%v)-B", valueType(v))) + if !IsSkipCustom(ctx, v.Type()) { + if getter, ok := v.Interface().(GetterCtx); ok { + //Debug(0, fmt.Sprintf("addElem-GetterCtx(%v)", valueType(v))) + getv, err := getter.GetBSONWithContext(ctx) + if err != nil { + panic(err) + } + v = reflect.ValueOf(getv) + e.addElem(NewContextWithSkipCustom(ctx, getv), name, v, minSize) + return + } + if getter, ok := v.Interface().(Getter); ok { + //Debug(0, fmt.Sprintf("addElem-Getter(%v)", valueType(v))) + getv, err := getter.GetBSON() + if err != nil { + panic(err) + } + v = reflect.ValueOf(getv) + e.addElem(NewContextWithSkipCustom(ctx, getv), name, v, minSize) + return } - e.addElem(name, reflect.ValueOf(getv), minSize) - return } + //Debug(0, fmt.Sprintf("addElem(%v)-C", valueType(v))) switch v.Kind() { case reflect.Interface: - e.addElem(name, v.Elem(), minSize) + e.addElem(ctx, name, v.Elem(), minSize) case reflect.Ptr: - e.addElem(name, v.Elem(), minSize) + e.addElem(ctx, name, v.Elem(), minSize) case reflect.String: s := v.String() @@ -348,7 +395,7 @@ func (e *encoder) addElem(name string, v reflect.Value, minSize bool) { case reflect.Map: e.addElemName(0x03, name) - e.addDoc(v) + e.addDoc(ctx, v) case reflect.Slice: vt := v.Type() @@ -358,10 +405,10 @@ func (e *encoder) addElem(name string, v reflect.Value, minSize bool) { e.addBinary(0x00, v.Bytes()) } else if et == typeDocElem || et == typeRawDocElem { e.addElemName(0x03, name) - e.addDoc(v) + e.addDoc(ctx, v) } else { e.addElemName(0x04, name) - e.addDoc(v) + e.addDoc(ctx, v) } case reflect.Array: @@ -381,7 +428,7 @@ func (e *encoder) addElem(name string, v reflect.Value, minSize bool) { } } else { e.addElemName(0x04, name) - e.addDoc(v) + e.addDoc(ctx, v) } case reflect.Struct: @@ -429,7 +476,7 @@ func (e *encoder) addElem(name string, v reflect.Value, minSize bool) { e.addElemName(0x0F, name) start := e.reserveInt32() e.addStr(s.Code) - e.addDoc(reflect.ValueOf(s.Scope)) + e.addDoc(ctx, reflect.ValueOf(s.Scope)) e.setInt32(start, int32(len(e.out)-start)) } @@ -447,7 +494,7 @@ func (e *encoder) addElem(name string, v reflect.Value, minSize bool) { default: e.addElemName(0x03, name) - e.addDoc(v) + e.addDoc(ctx, v) } default: From e7448e45427043c238de7975afc3261c6a717d5d Mon Sep 17 00:00:00 2001 From: Phil Feairheller Date: Tue, 27 Sep 2016 08:44:14 -0400 Subject: [PATCH 56/69] Added "PartialFilterExpression" support to Indexes. If specified, the index only references documents that match the filter expression. --- session.go | 33 ++++++++++++++++++--------------- session_test.go | 15 +++++++++++++++ 2 files changed, 33 insertions(+), 15 deletions(-) diff --git a/session.go b/session.go index 3bd5c1f62..92a87389a 100644 --- a/session.go +++ b/session.go @@ -1011,7 +1011,7 @@ type indexSpec struct { DefaultLanguage string "default_language,omitempty" LanguageOverride string "language_override,omitempty" TextIndexVersion int "textIndexVersion,omitempty" - + PartialFilterExpression bson.M "partialFilterExpression,omitempty" Collation *Collation "collation,omitempty" } @@ -1021,6 +1021,7 @@ type Index struct { DropDups bool // Drop documents with the same index key as a previously indexed one Background bool // Build index in background and return immediately Sparse bool // Only index documents containing the Key fields + PartialFilterExpression bson.M //If specified, the index only references documents that match the filter expression // If ExpireAfter is defined the server will periodically delete // documents with indexed time.Time older than the provided delta. @@ -1284,6 +1285,7 @@ func (c *Collection) EnsureIndex(index Index) error { DropDups: index.DropDups, Background: index.Background, Sparse: index.Sparse, + PartialFilterExpression: index.PartialFilterExpression, Bits: index.Bits, Min: index.Minf, Max: index.Maxf, @@ -1494,20 +1496,21 @@ func (c *Collection) Indexes() (indexes []Index, err error) { func indexFromSpec(spec indexSpec) Index { index := Index{ - Name: spec.Name, - Key: simpleIndexKey(spec.Key), - Unique: spec.Unique, - DropDups: spec.DropDups, - Background: spec.Background, - Sparse: spec.Sparse, - Minf: spec.Min, - Maxf: spec.Max, - Bits: spec.Bits, - BucketSize: spec.BucketSize, - DefaultLanguage: spec.DefaultLanguage, - LanguageOverride: spec.LanguageOverride, - ExpireAfter: time.Duration(spec.ExpireAfter) * time.Second, - Collation: spec.Collation, + Name: spec.Name, + Key: simpleIndexKey(spec.Key), + Unique: spec.Unique, + DropDups: spec.DropDups, + Background: spec.Background, + Sparse: spec.Sparse, + Minf: spec.Min, + Maxf: spec.Max, + Bits: spec.Bits, + BucketSize: spec.BucketSize, + DefaultLanguage: spec.DefaultLanguage, + LanguageOverride: spec.LanguageOverride, + ExpireAfter: time.Duration(spec.ExpireAfter) * time.Second, + Collation: spec.Collation, + PartialFilterExpression: spec.PartialFilterExpression, } if float64(int(spec.Min)) == spec.Min && float64(int(spec.Max)) == spec.Max { index.Min = int(spec.Min) diff --git a/session_test.go b/session_test.go index 492f21078..9c34a59b9 100644 --- a/session_test.go +++ b/session_test.go @@ -2975,6 +2975,21 @@ var indexTests = []struct { "key": M{"cn": 1}, "ns": "mydb.mycoll", }, +}, { + mgo.Index{ + Key: []string{"p"}, + Unique: true, + PartialFilterExpression: bson.M{ + "p": bson.M{"$exists": true}, + }, + }, + M{ + "name": "p_1", + "key": M{"p": 1}, + "ns": "mydb.mycoll", + "unique": true, + "partialFilterExpression": M{"p" : M{"$exists" : true}}, + }, }} func (s *S) TestEnsureIndex(c *C) { From c056e846476fb9e88ff39e41c3fbf6457d6447e1 Mon Sep 17 00:00:00 2001 From: "Paul Fraley (pfraley)" Date: Tue, 9 Jul 2019 12:40:44 -0700 Subject: [PATCH 57/69] bson: remove Debug() --- bson/context.go | 32 -------------------------------- bson/encode.go | 22 ---------------------- 2 files changed, 54 deletions(-) diff --git a/bson/context.go b/bson/context.go index 70a062603..c81118e8e 100644 --- a/bson/context.go +++ b/bson/context.go @@ -69,7 +69,6 @@ func NewContextWithSkipCustom(ctx context.Context, valu interface{}) context.Con if ctx == nil { ctx = context.Background() } - //Debug(0, fmt.Sprintf("NewContextWithSkipCustom: %s", baseTypeName(reflect.TypeOf(valu)))) return context.WithValue(ctx, bsonKey, &bsonOptions{skipCustom: baseTypeName(reflect.TypeOf(valu))}) } @@ -84,38 +83,7 @@ func NewContextWithSkipCustom(ctx context.Context, valu interface{}) context.Con // skipCustom will not work (it will skip all variants). func IsSkipCustom(ctx context.Context, typ reflect.Type) bool { if opts, _ := fromContext(ctx); opts != nil { - //Debug(0, fmt.Sprintf("IsSkipCustom(%s) opts(%s) base(%s) %v", typ, opts.skipCustom, - // baseTypeName(typ), opts.skipCustom == baseTypeName(typ))) return opts.skipCustom == baseTypeName(typ) } - //Debug(0, fmt.Sprintf("IsSkipCustom(%s) false", typ)) return false } - -// These were useful in debugging infinite loop issues: -// - calling Unmarshal from custom SetBSON function (decode.go) -// - encode.go calling custom GetBSON after just calling custom GetBSON for a given type -// uncomment them (along with the invocations of Debug()) to aid debugging. -//func valueType(valu reflect.Value) string { -// if !valu.IsValid() { -// return "Invalid" -// } -// return fmt.Sprintf("%s", valu.Type()) -//} -// -//var currentIndentLevel int = 0 -//var infinteLoopCounter int = 10000 -// -//func Debug(delta int, msg string) { -// infinteLoopCounter-- -// if infinteLoopCounter < 0 { -// panic(fmt.Errorf("YOU ARE IN INFINITE LOOP")) -// } -// if delta > 0 { -// currentIndentLevel += delta -// } -// fmt.Printf("%s%s\n", strings.Repeat(" ", currentIndentLevel), msg) -// if delta < 0 { -// currentIndentLevel += delta -// } -//} diff --git a/bson/encode.go b/bson/encode.go index ed93b940f..29966018d 100644 --- a/bson/encode.go +++ b/bson/encode.go @@ -83,16 +83,10 @@ type encoder struct { } func (e *encoder) addDoc(ctx context.Context, v reflect.Value) { - //cnt := 0 - //Debug(4, fmt.Sprintf("addDoc(%v)-A", valueType(v))) - //defer func() { - // Debug(-4, fmt.Sprintf("addDoc(%v)-Z", valueType(v))) - //}() for { //cnt += 1 if !IsSkipCustom(ctx, v.Type()) { if vi, ok := v.Interface().(GetterCtx); ok { - //Debug(0, fmt.Sprintf("addDoc(%02d)-GetterCtx(%v)", cnt, valueType(v))) getv, err := vi.GetBSONWithContext(ctx) if err != nil { panic(err) @@ -102,7 +96,6 @@ func (e *encoder) addDoc(ctx context.Context, v reflect.Value) { continue } if vi, ok := v.Interface().(Getter); ok { - //Debug(0, fmt.Sprintf("addDoc(%02d)-Getter(%v)", cnt, valueType(v))) getv, err := vi.GetBSON() if err != nil { panic(err) @@ -118,7 +111,6 @@ func (e *encoder) addDoc(ctx context.Context, v reflect.Value) { } break } - //Debug(0, fmt.Sprintf("addDoc(%v)-B", valueType(v))) if v.Type() == typeRaw { raw := v.Interface().(Raw) @@ -132,7 +124,6 @@ func (e *encoder) addDoc(ctx context.Context, v reflect.Value) { return } - //Debug(0, fmt.Sprintf("addDoc(%v)-C", valueType(v))) start := e.reserveInt32() switch v.Kind() { @@ -224,10 +215,6 @@ func isZero(v reflect.Value) bool { } func (e *encoder) addSlice(ctx context.Context, v reflect.Value) { - //Debug(4, fmt.Sprintf("addSlice(%v)-A", valueType(v))) - //defer func() { - // Debug(-4, fmt.Sprintf("addSlice(%v)-Z", valueType(v))) - //}() vi := v.Interface() if d, ok := vi.(D); ok { for _, elem := range d { @@ -272,20 +259,13 @@ func (e *encoder) addElemName(kind byte, name string) { } func (e *encoder) addElem(ctx context.Context, name string, v reflect.Value, minSize bool) { - //Debug(4, fmt.Sprintf("addElem(%v)[%v]-A", valueType(v), name)) - //defer func() { - // Debug(-4, fmt.Sprintf("addElem(%v)[%v]-Z", valueType(v), name)) - //}() - if !v.IsValid() { e.addElemName(0x0A, name) return } - //Debug(0, fmt.Sprintf("addElem(%v)-B", valueType(v))) if !IsSkipCustom(ctx, v.Type()) { if getter, ok := v.Interface().(GetterCtx); ok { - //Debug(0, fmt.Sprintf("addElem-GetterCtx(%v)", valueType(v))) getv, err := getter.GetBSONWithContext(ctx) if err != nil { panic(err) @@ -295,7 +275,6 @@ func (e *encoder) addElem(ctx context.Context, name string, v reflect.Value, min return } if getter, ok := v.Interface().(Getter); ok { - //Debug(0, fmt.Sprintf("addElem-Getter(%v)", valueType(v))) getv, err := getter.GetBSON() if err != nil { panic(err) @@ -306,7 +285,6 @@ func (e *encoder) addElem(ctx context.Context, name string, v reflect.Value, min } } - //Debug(0, fmt.Sprintf("addElem(%v)-C", valueType(v))) switch v.Kind() { case reflect.Interface: From 5c2d36850f89e7e33ca0c1eab2a7b7957ba79689 Mon Sep 17 00:00:00 2001 From: Parvathi Nair Date: Thu, 5 Dec 2019 14:24:08 -0800 Subject: [PATCH 58/69] removes the deprecated MMAPv1 storage engine and the MMAPv1-specific configuration options --- dbtest/dbserver.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index 9ea5bdc5d..04bc22070 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -150,10 +150,6 @@ func (dbs *DBServer) execContainer(network string, exposePort bool) *exec.Cmd { "--name", dbs.containerName, fmt.Sprintf("mongo:%s", dbs.version), - "--nssize", "1", - "--noprealloc", - "--smallfiles", - "--nojournal", }...) return exec.Command("docker", args...) } From a4fc8954662991ee3aad54686fd643d928bed284 Mon Sep 17 00:00:00 2001 From: Parvathi Nair Date: Fri, 6 Dec 2019 15:33:43 -0800 Subject: [PATCH 59/69] Adding support for starting mongo test server in replicaset formation --- dbtest/dbserver.go | 222 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 217 insertions(+), 5 deletions(-) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index 04bc22070..479962ea4 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -13,7 +13,7 @@ import ( "time" "gopkg.in/mgo.v2" - "gopkg.in/tomb.v2" + "gopkg.in/mgo.v2/bson" ) // Constants to define how the DB test instance should be executed @@ -47,6 +47,7 @@ type DBServer struct { debug bool // Log debug statements containerName string // The container name, when running mgo within a container tomb tomb.Tomb + rsName string // ReplicaSet Name. If not empty- the mongod will be started as a Replica Set Server } // SetPath defines the path to the directory where the database files will be @@ -81,6 +82,11 @@ func (dbs *DBServer) SetExposePort(exposePort bool) { dbs.exposePort = exposePort } +// SetReplicaSetName sets whether the mongod is started as a replica set +func (dbs *DBServer) SetReplicaSetName(rsName string) { + dbs.rsName = rsName +} + // Start Mongo DB within Docker container on host. // It assumes Docker is already installed. func (dbs *DBServer) execContainer(network string, exposePort bool) *exec.Cmd { @@ -151,6 +157,15 @@ func (dbs *DBServer) execContainer(network string, exposePort bool) *exec.Cmd { dbs.containerName, fmt.Sprintf("mongo:%s", dbs.version), }...) + + if dbs.rsName != "" { + args = append(args, []string{ + "mongod", + "--replSet", + dbs.rsName, + }...) + } + fmt.Println("DB start up arguments are: ", args) return exec.Command("docker", args...) } @@ -247,10 +262,13 @@ func (dbs *DBServer) execLocal(port int) *exec.Cmd { "--dbpath", dbs.dbpath, "--bind_ip", "127.0.0.1", "--port", strconv.Itoa(port), - "--nssize", "1", - "--noprealloc", - "--smallfiles", - "--nojournal", + } + + if dbs.rsName != "" { + args = append(args, []string{ + "--replSet", + dbs.rsName, + }...) } return exec.Command("mongod", args...) } @@ -402,6 +420,26 @@ func (dbs *DBServer) SessionWithTimeout(timeout time.Duration) *mgo.Session { mgo.ResetStats() var err error fmt.Printf("[%s] Dialing mongod located at '%s'\n", time.Now().String(), dbs.hostPort) + // If The MongoDB Driver is Configured with ReplicaSet - (Then configure Replica Set first!) + // Directly Connect First - and then configure Replica Set! + if dbs.rsName != "" { + dbs.session, err = mgo.DialWithTimeout(dbs.hostPort+"/test?connect=direct", timeout) + if err != nil { + fmt.Fprintf(os.Stderr, "[%s] Unable to dial mongod located at '%s'. Timeout=%v, Error: %s\n", time.Now().String(), dbs.hostPort, timeout, err.Error()) + fmt.Fprintf(os.Stderr, "%s", dbs.output.Bytes()) + dbs.printMongoDebugInfo() + panic(err) + } + err = dbs.Initiate() + if err != nil { + fmt.Fprintf(os.Stderr, "Unable to Configure Replica Set '%s'. Timeout=%v, Error: %s\n", dbs.rsName, dbs.hostPort, timeout, err.Error()) + fmt.Fprintf(os.Stderr, "%s", dbs.output.Bytes()) + dbs.printMongoDebugInfo() + panic(err) + } + dbs.session.Close() // Create a new One Below without Direct=true... + + } dbs.session, err = mgo.DialWithTimeout(dbs.hostPort+"/test", timeout) if err != nil { fmt.Fprintf(os.Stderr, "[%s] Unable to dial mongod located at '%s'. Timeout=%v, Error: %s\n", time.Now().String(), dbs.hostPort, timeout, err.Error()) @@ -479,3 +517,177 @@ func (dbs *DBServer) Wipe() { } } } + +// Code From https://github.com/juju/replicaset To Start MongoDB in replica Set Configuration +// Config is the document stored in mongodb that defines the servers in the +// replica set +type Config struct { + Name string `bson:"_id"` + Version int `bson:"version"` + ProtocolVersion int `bson:"protocolVersion,omitempty"` + Members []Member `bson:"members"` +} + +// Member holds configuration information for a replica set member. +// +// See http://docs.mongodb.org/manual/reference/replica-configuration/ +// for more details +type Member struct { + // Id is a unique id for a member in a set. + Id int `bson:"_id"` + + // Address holds the network address of the member, + // in the form hostname:port. + Address string `bson:"host"` +} + +func (dbs *DBServer) Initiate() error { + monotonicSession := dbs.session.Clone() + defer monotonicSession.Close() + monotonicSession.SetMode(mgo.Monotonic, true) + protocolVersion := 1 + var err error + // We don't know mongod's ability to use a correct IPv6 addr format + // until the server is started, but we need to know before we can start + // it. Try the older, incorrect format, if the correct format fails. + cfg := []Config{ + { + Name: dbs.rsName, + Version: 1, + ProtocolVersion: protocolVersion, + Members: []Member{{ + Id: 0, + Address: dbs.hostPort, + }}, + }, + } + + // Attempt replSetInitiate, with potential retries. + for i := 0; i < 5; i++ { + monotonicSession.Refresh() + if err = doAttemptInitiate(monotonicSession, cfg); err != nil { + time.Sleep(100 * time.Millisecond) + continue + } + break + } + + // Wait for replSetInitiate to complete. Even if err != nil, + // it may be that replSetInitiate is still in progress, so + // attempt CurrentStatus. + for i := 0; i < 10; i++ { + monotonicSession.Refresh() + var status *Status + status, err = getCurrentStatus(monotonicSession) + if err != nil { + fmt.Println("Initiate: fetching replication status failed: %v", err) + } + if err != nil || len(status.Members) == 0 { + time.Sleep(500 * time.Millisecond) + continue + } + break + } + return err +} + +// CurrentStatus returns the status of the replica set for the given session. +func getCurrentStatus(session *mgo.Session) (*Status, error) { + status := &Status{} + err := session.Run("replSetGetStatus", status) + if err != nil { + return nil, fmt.Errorf("cannot get replica set status: %v", err) + } + + for index, member := range status.Members { + status.Members[index].Address = member.Address + } + return status, nil +} + +// Status holds data about the status of members of the replica set returned +// from replSetGetStatus +// +// See http://docs.mongodb.org/manual/reference/command/replSetGetStatus/#dbcmd.replSetGetStatus +type Status struct { + Name string `bson:"set"` + Members []MemberStatus `bson:"members"` +} + +// Status holds the status of a replica set member returned from +// replSetGetStatus. +type MemberStatus struct { + // Id holds the replica set id of the member that the status is describing. + Id int `bson:"_id"` + + // Address holds address of the member that the status is describing. + Address string `bson:"name"` + + // Self holds whether this is the status for the member that + // the session is connected to. + Self bool `bson:"self"` + + // ErrMsg holds the most recent error or status message received + // from the member. + ErrMsg string `bson:"errmsg"` + + // Healthy reports whether the member is up. It is true for the + // member that the request was made to. + Healthy bool `bson:"health"` + + // State describes the current state of the member. + State MemberState `bson:"state"` + +} +// doAttemptInitiate will attempt to initiate a mongodb replicaset with each of +// the given configs, returning as soon as one config is successful. +func doAttemptInitiate(monotonicSession *mgo.Session, cfg []Config) error { + var err error + for _, c := range cfg { + if err = monotonicSession.Run(bson.D{{"replSetInitiate", c}}, nil); err != nil { + fmt.Println("Unsuccessful attempt to initiate replicaset: %v", err) + continue + } + return nil + } + return err +} + +type MemberState int + +const ( + StartupState = iota + PrimaryState + SecondaryState + RecoveringState + FatalState + Startup2State + UnknownState + ArbiterState + DownState + RollbackState + ShunnedState +) + +var memberStateStrings = []string{ + StartupState: "STARTUP", + PrimaryState: "PRIMARY", + SecondaryState: "SECONDARY", + RecoveringState: "RECOVERING", + FatalState: "FATAL", + Startup2State: "STARTUP2", + UnknownState: "UNKNOWN", + ArbiterState: "ARBITER", + DownState: "DOWN", + RollbackState: "ROLLBACK", + ShunnedState: "SHUNNED", +} + +// String returns a string describing the state. +func (state MemberState) String() string { + if state < 0 || int(state) >= len(memberStateStrings) { + return "INVALID_MEMBER_STATE" + } + return memberStateStrings[state] +} + From 5216163e547644b3f3baf1a4adedf73b4368392c Mon Sep 17 00:00:00 2001 From: Parvathi Nair Date: Mon, 9 Dec 2019 15:16:21 -0800 Subject: [PATCH 60/69] Adding support for starting mongo test server in replicaset formation --- dbtest/dbserver.go | 1 + 1 file changed, 1 insertion(+) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index 479962ea4..3ad1682bd 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -14,6 +14,7 @@ import ( "gopkg.in/mgo.v2" "gopkg.in/mgo.v2/bson" + "gopkg.in/tomb.v2" ) // Constants to define how the DB test instance should be executed From 76f864ac1d6828194b09c9c0b9103c2f477be657 Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Mon, 23 Mar 2020 09:44:27 -0700 Subject: [PATCH 61/69] Add method to set container name --- dbtest/dbserver.go | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index 3ad1682bd..2b2de9077 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -88,6 +88,11 @@ func (dbs *DBServer) SetReplicaSetName(rsName string) { dbs.rsName = rsName } +// SetContainerName sets the name of the docker container when the DB instance is started within a container. +func (dbs *DBServer) SetContainerName(containerName string) { + dbs.containerName = containerName +} + // Start Mongo DB within Docker container on host. // It assumes Docker is already installed. func (dbs *DBServer) execContainer(network string, exposePort bool) *exec.Cmd { @@ -127,18 +132,6 @@ func (dbs *DBServer) execContainer(network string, exposePort bool) *exec.Cmd { fmt.Printf("[%s] Pulled Mongo docker image\n", time.Now().String()) } - // Generate a name for the container. This will help to inspect the container - // and get the Mongo PID. - u := make([]byte, 8) - // The default number generator is deterministic. - s := rand.NewSource(time.Now().UnixNano()) - r := rand.New(s) - _, err = r.Read(u) - if err != nil { - panic(err) - } - dbs.containerName = fmt.Sprintf("mongo-%s", hex.EncodeToString(u)) - args = []string{ "run", "-t", @@ -289,6 +282,20 @@ func (dbs *DBServer) start() { addr := l.Addr().(*net.TCPAddr) l.Close() + if dbs.containerName == "" { + // Generate a name for the container. This will help to inspect the container + // and get the Mongo PID. + u := make([]byte, 8) + // The default number generator is deterministic. + s := rand.NewSource(time.Now().UnixNano()) + r := rand.New(s) + _, err = r.Read(u) + if err != nil { + panic(err) + } + dbs.containerName = fmt.Sprintf("mongo-%s", hex.EncodeToString(u)) + } + dbs.tomb = tomb.Tomb{} switch dbs.eType { case LocalProcess: From ec2e737ccd86d33cdc46bd044ae45ea70ee9fe63 Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Mon, 29 Jun 2020 12:06:51 -0700 Subject: [PATCH 62/69] use log package to log records --- dbtest/dbserver.go | 63 +++++++++++++++++++++++----------------------- 1 file changed, 31 insertions(+), 32 deletions(-) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index 2b2de9077..68f462a8b 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/hex" "fmt" + "log" "math/rand" "net" "os" @@ -48,7 +49,7 @@ type DBServer struct { debug bool // Log debug statements containerName string // The container name, when running mgo within a container tomb tomb.Tomb - rsName string // ReplicaSet Name. If not empty- the mongod will be started as a Replica Set Server + rsName string // ReplicaSet Name. If not empty- the mongod will be started as a Replica Set Server } // SetPath defines the path to the directory where the database files will be @@ -112,8 +113,8 @@ func (dbs *DBServer) execContainer(network string, exposePort bool) *exec.Cmd { // Error response from daemon: Get https://registry-1.docker.io/v2/: net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers) for time.Since(start) < 60*time.Second { cmd := exec.Command("docker", args...) + log.Printf("Pulling Mongo docker image") if dbs.debug { - fmt.Printf("[%s] Pulling Mongo docker image\n", time.Now().String()) cmd.Stdout = os.Stderr cmd.Stderr = os.Stderr } @@ -121,16 +122,14 @@ func (dbs *DBServer) execContainer(network string, exposePort bool) *exec.Cmd { if err == nil { break } else { - fmt.Printf("[%s] Failed to pull Mongo container image. err=%s", time.Now().String(), err.Error()) + log.Printf("Failed to pull Mongo container image. err=%s", err.Error()) time.Sleep(5 * time.Second) } } if err != nil { panic(err) } - if dbs.debug { - fmt.Printf("[%s] Pulled Mongo docker image\n", time.Now().String()) - } + log.Printf("Pulled Mongo docker image") args = []string{ "run", @@ -153,7 +152,7 @@ func (dbs *DBServer) execContainer(network string, exposePort bool) *exec.Cmd { }...) if dbs.rsName != "" { - args = append(args, []string{ + args = append(args, []string{ "mongod", "--replSet", dbs.rsName, @@ -193,7 +192,7 @@ func (dbs *DBServer) GetContainerIpAddr() (string, error) { for time.Since(start) < 60*time.Second { if dbs.server.ProcessState != nil { // The process has exited - fmt.Printf("[%s] Mongo container has exited unexpectedly. Output:\n%s\n", time.Now().String(), dbs.output.String()) + log.Printf("Mongo container has exited unexpectedly. Output:\n%s", dbs.output.String()) return "", fmt.Errorf("Process has exited") } stderr.Reset() @@ -205,7 +204,7 @@ func (dbs *DBServer) GetContainerIpAddr() (string, error) { err = cmd.Run() if err != nil { // This could be because the container has not started yet. Retry later - fmt.Printf("[%s] Failed to get container IP address. Will retry later.\n", time.Now().String()) + log.Printf("Failed to get container IP address. Will retry later.") time.Sleep(3 * time.Second) continue } @@ -259,7 +258,7 @@ func (dbs *DBServer) execLocal(port int) *exec.Cmd { } if dbs.rsName != "" { - args = append(args, []string{ + args = append(args, []string{ "--replSet", dbs.rsName, }...) @@ -268,6 +267,7 @@ func (dbs *DBServer) execLocal(port int) *exec.Cmd { } func (dbs *DBServer) start() { + log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds | log.Llongfile) if dbs.server != nil { panic("DBServer already started") } @@ -282,19 +282,19 @@ func (dbs *DBServer) start() { addr := l.Addr().(*net.TCPAddr) l.Close() - if dbs.containerName == "" { - // Generate a name for the container. This will help to inspect the container - // and get the Mongo PID. - u := make([]byte, 8) - // The default number generator is deterministic. - s := rand.NewSource(time.Now().UnixNano()) - r := rand.New(s) - _, err = r.Read(u) - if err != nil { - panic(err) - } - dbs.containerName = fmt.Sprintf("mongo-%s", hex.EncodeToString(u)) - } + if dbs.containerName == "" { + // Generate a name for the container. This will help to inspect the container + // and get the Mongo PID. + u := make([]byte, 8) + // The default number generator is deterministic. + s := rand.NewSource(time.Now().UnixNano()) + r := rand.New(s) + _, err = r.Read(u) + if err != nil { + panic(err) + } + dbs.containerName = fmt.Sprintf("mongo-%s", hex.EncodeToString(u)) + } dbs.tomb = tomb.Tomb{} switch dbs.eType { @@ -309,20 +309,20 @@ func (dbs *DBServer) start() { dbs.server.Stdout = &dbs.output dbs.server.Stderr = &dbs.output if dbs.debug { - fmt.Printf("[%s] Starting Mongo instance: %v. Address: %s. Network: '%s'\n", time.Now().String(), dbs.server.Args, dbs.hostPort, dbs.network) + log.Printf("Starting Mongo instance: %v. Address: %s. Network: '%s'", dbs.server.Args, dbs.hostPort, dbs.network) } err = dbs.server.Start() if err != nil { panic("Failed to start Mongo instance: " + err.Error()) } if dbs.debug { - fmt.Printf("[%s] Mongo instance started\n", time.Now().String()) + log.Printf("Mongo instance started") } go func() { // Call Wait() so cmd.ProcessState is set after command has completed. err = dbs.server.Wait() if err != nil { - fmt.Printf("[%s] Command exited. Output:\n%s\n", time.Now().String(), dbs.output.String()) + log.Printf("Command exited. Output:\n%s", dbs.output.String()) } }() if dbs.eType == Docker { @@ -427,7 +427,7 @@ func (dbs *DBServer) SessionWithTimeout(timeout time.Duration) *mgo.Session { if dbs.session == nil { mgo.ResetStats() var err error - fmt.Printf("[%s] Dialing mongod located at '%s'\n", time.Now().String(), dbs.hostPort) + log.Printf("Dialing mongod located at '%s'", dbs.hostPort) // If The MongoDB Driver is Configured with ReplicaSet - (Then configure Replica Set first!) // Directly Connect First - and then configure Replica Set! if dbs.rsName != "" { @@ -440,7 +440,7 @@ func (dbs *DBServer) SessionWithTimeout(timeout time.Duration) *mgo.Session { } err = dbs.Initiate() if err != nil { - fmt.Fprintf(os.Stderr, "Unable to Configure Replica Set '%s'. Timeout=%v, Error: %s\n", dbs.rsName, dbs.hostPort, timeout, err.Error()) + fmt.Fprintf(os.Stderr, "Unable to Configure Replica Set '%s'. Timeout=%v, Error: %s\n", dbs.rsName, dbs.hostPort, timeout, err.Error()) fmt.Fprintf(os.Stderr, "%s", dbs.output.Bytes()) dbs.printMongoDebugInfo() panic(err) @@ -498,7 +498,7 @@ func (dbs *DBServer) checkSessions() { // there is a session leak. func (dbs *DBServer) Wipe() { if dbs.server == nil || dbs.session == nil { - fmt.Printf("[%s] Skip Wipe()\n", time.Now().String()) + log.Printf("Skip Wipe()") return } dbs.checkSessions() @@ -517,7 +517,7 @@ func (dbs *DBServer) Wipe() { switch name { case "admin", "local", "config": default: - fmt.Printf("Drop database '%s'\n", name) + log.Printf("Drop database '%s'", name) err = session.DB(name).DropDatabase() if err != nil { panic(err) @@ -645,8 +645,8 @@ type MemberStatus struct { // State describes the current state of the member. State MemberState `bson:"state"` - } + // doAttemptInitiate will attempt to initiate a mongodb replicaset with each of // the given configs, returning as soon as one config is successful. func doAttemptInitiate(monotonicSession *mgo.Session, cfg []Config) error { @@ -698,4 +698,3 @@ func (state MemberState) String() string { } return memberStateStrings[state] } - From 4e59dc077ea384acef8cf7528897750854958b31 Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Mon, 29 Jun 2020 13:27:11 -0700 Subject: [PATCH 63/69] use log package to log records --- dbtest/dbserver.go | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index 68f462a8b..9c0442c01 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -158,7 +158,7 @@ func (dbs *DBServer) execContainer(network string, exposePort bool) *exec.Cmd { dbs.rsName, }...) } - fmt.Println("DB start up arguments are: ", args) + log.Printf("DB start up arguments are: %v", args) return exec.Command("docker", args...) } @@ -204,7 +204,7 @@ func (dbs *DBServer) GetContainerIpAddr() (string, error) { err = cmd.Run() if err != nil { // This could be because the container has not started yet. Retry later - log.Printf("Failed to get container IP address. Will retry later.") + log.Printf("Failed to get container IP address. Will retry later. Err: %v", err) time.Sleep(3 * time.Second) continue } @@ -216,7 +216,8 @@ func (dbs *DBServer) GetContainerIpAddr() (string, error) { return dbs.hostname, err } } - return "", fmt.Errorf("[%s] Failed to run command. error=%s, stderr=%s\n", time.Now().String(), err.Error(), stderr.String()) + log.Printf("Unable to get container IP address: %v", err) + return "", fmt.Errorf("Failed to run command. error=%s, stderr=%s\n", err.Error(), stderr.String()) } // Stop the docker container running Mongo. @@ -308,16 +309,12 @@ func (dbs *DBServer) start() { } dbs.server.Stdout = &dbs.output dbs.server.Stderr = &dbs.output - if dbs.debug { - log.Printf("Starting Mongo instance: %v. Address: %s. Network: '%s'", dbs.server.Args, dbs.hostPort, dbs.network) - } + log.Printf("Starting Mongo instance: %v. Address: %s. Network: '%s'", dbs.server.Args, dbs.hostPort, dbs.network) err = dbs.server.Start() if err != nil { panic("Failed to start Mongo instance: " + err.Error()) } - if dbs.debug { - log.Printf("Mongo instance started") - } + log.Printf("Mongo instance started") go func() { // Call Wait() so cmd.ProcessState is set after command has completed. err = dbs.server.Wait() @@ -427,21 +424,21 @@ func (dbs *DBServer) SessionWithTimeout(timeout time.Duration) *mgo.Session { if dbs.session == nil { mgo.ResetStats() var err error - log.Printf("Dialing mongod located at '%s'", dbs.hostPort) + log.Printf("Dialing mongod located at '%s'. timeout: %v", dbs.hostPort, timeout) // If The MongoDB Driver is Configured with ReplicaSet - (Then configure Replica Set first!) // Directly Connect First - and then configure Replica Set! if dbs.rsName != "" { dbs.session, err = mgo.DialWithTimeout(dbs.hostPort+"/test?connect=direct", timeout) if err != nil { - fmt.Fprintf(os.Stderr, "[%s] Unable to dial mongod located at '%s'. Timeout=%v, Error: %s\n", time.Now().String(), dbs.hostPort, timeout, err.Error()) - fmt.Fprintf(os.Stderr, "%s", dbs.output.Bytes()) + log.Printf("Unable to dial mongod located at '%s'. Timeout=%v, Error: %s", dbs.hostPort, timeout, err.Error()) + log.Printf("%s", dbs.output.Bytes()) dbs.printMongoDebugInfo() panic(err) } err = dbs.Initiate() if err != nil { - fmt.Fprintf(os.Stderr, "Unable to Configure Replica Set '%s'. Timeout=%v, Error: %s\n", dbs.rsName, dbs.hostPort, timeout, err.Error()) - fmt.Fprintf(os.Stderr, "%s", dbs.output.Bytes()) + log.Printf("Unable to Configure Replica Set '%s'. Timeout=%v, Error: %s", dbs.rsName, timeout, err.Error()) + log.Printf("%s", dbs.output.Bytes()) dbs.printMongoDebugInfo() panic(err) } @@ -450,8 +447,8 @@ func (dbs *DBServer) SessionWithTimeout(timeout time.Duration) *mgo.Session { } dbs.session, err = mgo.DialWithTimeout(dbs.hostPort+"/test", timeout) if err != nil { - fmt.Fprintf(os.Stderr, "[%s] Unable to dial mongod located at '%s'. Timeout=%v, Error: %s\n", time.Now().String(), dbs.hostPort, timeout, err.Error()) - fmt.Fprintf(os.Stderr, "%s", dbs.output.Bytes()) + log.Printf("Unable to dial mongod located at '%s'. Timeout=%v, Error: %s", dbs.hostPort, timeout, err.Error()) + log.Printf("%s", dbs.output.Bytes()) dbs.printMongoDebugInfo() panic(err) } @@ -588,7 +585,7 @@ func (dbs *DBServer) Initiate() error { var status *Status status, err = getCurrentStatus(monotonicSession) if err != nil { - fmt.Println("Initiate: fetching replication status failed: %v", err) + log.Printf("Initiate: fetching replication status failed: %v", err) } if err != nil || len(status.Members) == 0 { time.Sleep(500 * time.Millisecond) @@ -653,7 +650,7 @@ func doAttemptInitiate(monotonicSession *mgo.Session, cfg []Config) error { var err error for _, c := range cfg { if err = monotonicSession.Run(bson.D{{"replSetInitiate", c}}, nil); err != nil { - fmt.Println("Unsuccessful attempt to initiate replicaset: %v", err) + log.Printf("Unsuccessful attempt to initiate replicaset: %v", err) continue } return nil From f43e66ba88246174e77e86ceeee0ee1b5fa18d7a Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Mon, 29 Jun 2020 13:40:11 -0700 Subject: [PATCH 64/69] use log package to log records --- dbtest/dbserver.go | 1 + 1 file changed, 1 insertion(+) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index 9c0442c01..f4c9e310e 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -210,6 +210,7 @@ func (dbs *DBServer) GetContainerIpAddr() (string, error) { } ipAddr := strings.Trim(strings.TrimSpace(out.String()), "'") dbs.hostname = ipAddr + log.Printf("Mongo IP address is %v", dbs.hostname) if dbs.network == "" { return "127.0.0.1", nil } else { From 16644d404dc9691646f10d4644c2c9e654f5eab5 Mon Sep 17 00:00:00 2001 From: "Parvathi Nair (panair)" Date: Tue, 28 Jul 2020 18:33:02 -0700 Subject: [PATCH 65/69] Adding Support for Collation in Query --- session.go | 66 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ socket.go | 1 + 2 files changed, 67 insertions(+) diff --git a/session.go b/session.go index 92a87389a..0220debc9 100644 --- a/session.go +++ b/session.go @@ -2175,6 +2175,8 @@ type Pipe struct { pipeline interface{} allowDisk bool batchSize int + maxTimeMS int64 + collation *Collation } type pipeCmd struct { @@ -2183,6 +2185,8 @@ type pipeCmd struct { Cursor *pipeCmdCursor ",omitempty" Explain bool ",omitempty" AllowDisk bool "allowDiskUse,omitempty" + MaxTimeMS int64 `bson:"maxTimeMS,omitempty"` + Collation *Collation `bson:"collation,omitempty"` } type pipeCmdCursor struct { @@ -2236,6 +2240,10 @@ func (p *Pipe) Iter() *Iter { Pipeline: p.pipeline, AllowDisk: p.allowDisk, Cursor: &pipeCmdCursor{p.batchSize}, + Collation: p.collation, + } + if p.maxTimeMS > 0 { + cmd.MaxTimeMS = p.maxTimeMS } err := c.Database.Run(cmd, &result) if e, ok := err.(*QueryError); ok && e.Message == `unrecognized field "cursor` { @@ -2374,6 +2382,30 @@ func (p *Pipe) Batch(n int) *Pipe { return p } +// SetMaxTime sets the maximum amount of time to allow the query to run. +// +func (p *Pipe) SetMaxTime(d time.Duration) *Pipe { + p.maxTimeMS = int64(d / time.Millisecond) + return p +} + + +// Collation allows to specify language-specific rules for string comparison, +// such as rules for lettercase and accent marks. +// When specifying collation, the locale field is mandatory; all other collation +// fields are optional +// +// Relevant documentation: +// +// https://docs.mongodb.com/manual/reference/collation/ +// +func (p *Pipe) Collation(collation *Collation) *Pipe { + if collation != nil { + p.collation = collation + } + return p +} + // mgo.v3: Use a single user-visible error type. type LastError struct { @@ -2861,6 +2893,38 @@ func (q *Query) Sort(fields ...string) *Query { return q } +// Collation allows to specify language-specific rules for string comparison, +// such as rules for lettercase and accent marks. +// When specifying collation, the locale field is mandatory; all other collation +// fields are optional +// +// For example, to perform a case and diacritic insensitive query: +// +// var res []bson.M +// collation := &mgo.Collation{Locale: "en", Strength: 1} +// err = db.C("mycoll").Find(bson.M{"a": "a"}).Collation(collation).All(&res) +// if err != nil { +// return err +// } +// +// This query will match following documents: +// +// {"a": "a"} +// {"a": "A"} +// {"a": "รข"} +// +// Relevant documentation: +// +// https://docs.mongodb.com/manual/reference/collation/ +// +func (q *Query) Collation(collation *Collation) *Query { + q.m.Lock() + q.op.options.Collation = collation + q.op.hasOptions = true + q.m.Unlock() + return q +} + // Explain returns a number of details about how the MongoDB server would // execute the requested query, such as the number of objects examined, // the number of times the read lock was yielded to allow writes to go in, @@ -3158,6 +3222,7 @@ func prepareFindOp(socket *mongoSocket, op *queryOp, limit int32) bool { Sort: op.options.OrderBy, Skip: op.skip, Limit: limit, + Collation: op.options.Collation, MaxTimeMS: op.options.MaxTimeMS, MaxScan: op.options.MaxScan, Hint: op.options.Hint, @@ -3225,6 +3290,7 @@ type findCmd struct { OplogReplay bool `bson:"oplogReplay,omitempty"` NoCursorTimeout bool `bson:"noCursorTimeout,omitempty"` AllowPartialResults bool `bson:"allowPartialResults,omitempty"` + Collation *Collation `bson:"collation,omitempty"` } // getMoreCmd holds the command used for requesting more query results on MongoDB 3.2+. diff --git a/socket.go b/socket.go index 8891dd5d7..13ca361f0 100644 --- a/socket.go +++ b/socket.go @@ -91,6 +91,7 @@ type queryWrapper struct { MaxScan int "$maxScan,omitempty" MaxTimeMS int "$maxTimeMS,omitempty" Comment string "$comment,omitempty" + Collation *Collation `bson:"$collation,omitempty"` } func (op *queryOp) finalQuery(socket *mongoSocket) interface{} { From 293db6db34c7bbfb0aea301b303556749d19fe63 Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Tue, 24 Nov 2020 19:50:36 -0800 Subject: [PATCH 66/69] add check to determine if image exists locally before doing docker pull --- dbtest/dbserver.go | 44 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index f4c9e310e..7df39df66 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -94,35 +94,48 @@ func (dbs *DBServer) SetContainerName(containerName string) { dbs.containerName = containerName } -// Start Mongo DB within Docker container on host. -// It assumes Docker is already installed. -func (dbs *DBServer) execContainer(network string, exposePort bool) *exec.Cmd { - if dbs.version == "" { - dbs.version = "latest" +func (dbs *DBServer) pullDockerImage(dockerImage string) { + // Check if the docker image exists in the local registry. + args := []string{ + "images", + "-q", + dockerImage, } + cmd := exec.Command("docker", args...) + err = cmd.Run() + if err == nil { + // The image is already present locally. + // Do not invoke docker pull because: + // 1. Every network operations counts towards the dockerhub API rate limiting. + // 2. Reduce the chance of intermittent network issues. + return + } + // It may take a long time to download the mongo image if the docker image is not installed. // Execute 'docker pull' now to pull the image before executing it. Otherwise Dial() may fail // with a timeout after 10 seconds. args := []string{ "pull", - fmt.Sprintf("mongo:%s", dbs.version), + dockerImage, } start := time.Now() var err error + var stdout, stderr bytes.Buffer // Seeing intermittent issues such as: // Error response from daemon: Get https://registry-1.docker.io/v2/: net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers) for time.Since(start) < 60*time.Second { cmd := exec.Command("docker", args...) - log.Printf("Pulling Mongo docker image") + log.Printf("Pulling Mongo docker image %s", dockerImage) if dbs.debug { - cmd.Stdout = os.Stderr - cmd.Stderr = os.Stderr + cmd.Stdout = stdout + cmd.Stderr = stderr } err = cmd.Run() if err == nil { break } else { - log.Printf("Failed to pull Mongo container image. err=%s", err.Error()) + log.Printf("Failed to pull Mongo container image. err=%s\n%s\n%s", + err.Error(), stdout.String(), stderr.String()) time.Sleep(5 * time.Second) } } @@ -130,6 +143,17 @@ func (dbs *DBServer) execContainer(network string, exposePort bool) *exec.Cmd { panic(err) } log.Printf("Pulled Mongo docker image") +} + +// Start Mongo DB within Docker container on host. +// It assumes Docker is already installed. +func (dbs *DBServer) execContainer(network string, exposePort bool) *exec.Cmd { + if dbs.version == "" { + dbs.version = "latest" + } + + dockerImage := fmt.Sprintf("mongo:%s", dbs.version) + dbs.pullDockerImage(dockerImage) args = []string{ "run", From 4efb54bbf27a76ef56faab78f06ef6a0485f7b8f Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Tue, 24 Nov 2020 19:57:32 -0800 Subject: [PATCH 67/69] add check to determine if image exists locally before doing docker pull --- dbtest/dbserver.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index 7df39df66..7f12e3fc9 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -102,7 +102,7 @@ func (dbs *DBServer) pullDockerImage(dockerImage string) { dockerImage, } cmd := exec.Command("docker", args...) - err = cmd.Run() + err := cmd.Run() if err == nil { // The image is already present locally. // Do not invoke docker pull because: @@ -114,7 +114,7 @@ func (dbs *DBServer) pullDockerImage(dockerImage string) { // It may take a long time to download the mongo image if the docker image is not installed. // Execute 'docker pull' now to pull the image before executing it. Otherwise Dial() may fail // with a timeout after 10 seconds. - args := []string{ + args = []string{ "pull", dockerImage, } @@ -127,8 +127,8 @@ func (dbs *DBServer) pullDockerImage(dockerImage string) { cmd := exec.Command("docker", args...) log.Printf("Pulling Mongo docker image %s", dockerImage) if dbs.debug { - cmd.Stdout = stdout - cmd.Stderr = stderr + cmd.Stdout = &stdout + cmd.Stderr = &stderr } err = cmd.Run() if err == nil { @@ -155,7 +155,7 @@ func (dbs *DBServer) execContainer(network string, exposePort bool) *exec.Cmd { dockerImage := fmt.Sprintf("mongo:%s", dbs.version) dbs.pullDockerImage(dockerImage) - args = []string{ + args := []string{ "run", "-t", "--rm", // Automatically remove the container when it exits From a696e6a5572788df72d9d477f00fc15042433f98 Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Tue, 24 Nov 2020 20:01:02 -0800 Subject: [PATCH 68/69] add check to determine if image exists locally before doing docker pull --- dbtest/dbserver.go | 1 - 1 file changed, 1 deletion(-) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index 7f12e3fc9..81a4ff2e1 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -119,7 +119,6 @@ func (dbs *DBServer) pullDockerImage(dockerImage string) { dockerImage, } start := time.Now() - var err error var stdout, stderr bytes.Buffer // Seeing intermittent issues such as: // Error response from daemon: Get https://registry-1.docker.io/v2/: net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers) From 8850ab9fd38f333f0157391563ab1f90ca1dbb98 Mon Sep 17 00:00:00 2001 From: "Sebastien Rosset (serosset)" Date: Tue, 24 Nov 2020 20:02:20 -0800 Subject: [PATCH 69/69] add check to determine if image exists locally before doing docker pull --- dbtest/dbserver.go | 1 + 1 file changed, 1 insertion(+) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index 81a4ff2e1..29b93e0a7 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -108,6 +108,7 @@ func (dbs *DBServer) pullDockerImage(dockerImage string) { // Do not invoke docker pull because: // 1. Every network operations counts towards the dockerhub API rate limiting. // 2. Reduce the chance of intermittent network issues. + log.Printf("Docker image '%s' is already present in the local registry", dockerImage) return }