RPC Call

Ping: A simple RPC call to confirm a continuous connection.
CreateCollection: Creates a vector collection. You need to select options such as name, vector dimensions, and quantization settings. The created collection is automatically loaded into memory.
DeleteCollection: Deletes a vector collection, and the deleted collection is automatically removed from memory.
GetCollection: Retrieves a vector collection. This is currently an incomplete method, returning only the collection name.
LoadCollection: Loads only user-specified collections into memory to optimize memory usage.
ReleaseCollection: Unloads collections from memory when they are no longer in use.
Flush: Used to save data changes to disk due to Edge limitations. Without using Flush, data will not be saved to disk if the instance is restarted.
Insert: Adds the desired data.
Update: Modifies existing data.
Delete: Removes data.
VectorSearch: Performs a search using only vectors.
FilterSearch: Searches data matching filter conditions using only indexed fields, without vectors.
HybridSearch: Searches for data that matches conditions using both vectors and indexed fields.

TEST Code

CREATE COLLECTION
CREATE QUANTIZED COLLECTION
CREATE DATA
Search Only Vector
Search Only Filter
Hybrid(Vector+Filter)Search
FullCode

Create Collection

conn, _ := grpc.Dial(":50051", grpc.WithTransportCredentials(insecure.NewCredentials()))

dclient := edgeproto.NewEdgeRpcClient(conn)

res, err := dclient.CreateCollection(context.Background(), &edgeproto.Collection{
		CollectionName: "tcollection",
		Dim:            384,
		Distance:       edgeproto.Distance_Cosine,
		Quantization:   edgeproto.Quantization_None,
	})
--output
collection:{collection_name:"tcollection" dim:384} status:true
<nil>

Create QuantizedCollection

conn, _ := grpc.Dial(":50051", grpc.WithTransportCredentials(insecure.NewCredentials()))

dclient := edgeproto.NewEdgeRpcClient(conn)

res, err := dclient.CreateCollection(context.Background(), &edgeproto.Collection{
		CollectionName: "tcollection",
		Dim:            384,
		Distance:       edgeproto.Distance_Cosine,
		Quantization:   edgeproto.Quantization_F16,
	})

--output
collection:{collection_name:"tcollection" quantization:F16 dim:384} status:true
<nil>

Create Data

embeddings.InitEmbeddings()
	dataString := []string{
		"When I'm hungry, I usually have a sandwich.",
		"I tend to eat porridge whenever I feel hungry.",
		"For a typical meal, I often enjoy a bowl of salad.",
	}

	query := "I'm hungry, what should I eat?"

	addString := []string{
		"Sometimes, I grab a quick snack like nuts or fruit.",
		"On most days, I prefer having pasta for dinner.",
	}
	type InsertRecord struct {
		Metadata   map[string]interface{}
		Id         string
		BucketName string
		Vector     []float32
	}
	var uid string
	firstRecord := make([]InsertRecord, 0)
	uid = uuid.New().String()
	firstRecord = append(firstRecord, InsertRecord{
		Metadata: map[string]interface{}{
			"_id":         uid,
			"description": dataString[0],
			"type":        12,
			"gender":      "boy",
			"name":        "John",
		},
	})
	uid = uuid.New().String()
	firstRecord = append(firstRecord, InsertRecord{
		Metadata: map[string]interface{}{
			"_id":         uid,
			"description": dataString[1],
			"type":        12632,
			"gender":      "girl",
			"name":        "anna",
		},
	})
	uid = uuid.New().String()
	firstRecord = append(firstRecord, InsertRecord{
		Metadata: map[string]interface{}{
			"_id":         uid,
			"description": dataString[2],
			"type":        6302,
			"gender":      "boy",
			"name":        "Mark",
		},
	})
	// create dataset
	originDataset := make([]*edgeproto.ModifyDataset, 0, 3)
	for i, data := range firstRecord {
		vec, err := embeddings.TextEmbedding(dataString[i])
		if err != nil {
			log.Fatal(err)
		}
		m, _ := structpb.NewStruct(data.Metadata)
		originDataset = append(originDataset, &edgeproto.ModifyDataset{
			CollectionName: "tcollection",
			Id:             data.Metadata["_id"].(string),
			Vector:         vec,
			Metadata:       m,
		})
	}
	laterRecord := make([]InsertRecord, 0)
	uid = uuid.New().String()
	laterRecord = append(laterRecord, InsertRecord{
		Metadata: map[string]interface{}{
			"_id":         uid,
			"description": addString[0],
			"type":        142124,
			"gender":      "girl",
			"name":        "Ann",
		},
	})
	uid = uuid.New().String()
	laterRecord = append(laterRecord, InsertRecord{
		Metadata: map[string]interface{}{
			"_id":         uid,
			"description": addString[1],
			"type":        122,
			"gender":      "girl",
			"name":        "swan",
		},
	})

	afterDataset := make([]*edgeproto.ModifyDataset, 0, 3)
	for i, data := range laterRecord {
		vec, err := embeddings.TextEmbedding(addString[i])
		if err != nil {
			log.Fatal(err)
		}
		m, _ := structpb.NewStruct(data.Metadata)
		afterDataset = append(afterDataset, &edgeproto.ModifyDataset{
			CollectionName: "tcollection",
			Id:             data.Metadata["_id"].(string),
			Vector:         vec,
			Metadata:       m,
		})
	}
for _, data := range originDataset {
		res, err := dclient.Insert(context.Background(), data)
		fmt.Println(res.Status, res.Error, err)
}

for _, data := range afterDataset {
		res, err := dclient.Insert(context.Background(), data)
		fmt.Println(res.Status, res.Error, err)
}

VectorSearch

query := "I'm hungry, what should I eat?"
searchVec, _ := embeddings.TextEmbedding(query)

fmt.Println("vector search")
resp, err = dclient.VectorSearch(context.Background(), &edgeproto.SearchReq{
		CollectionName: "tcollection",
		Vector:         searchVec,
		TopK:           5,
})
if err != nil {
		log.Fatal(err)
}
if resp.Status {
		for _, dd := range resp.GetCandidates() {
			fmt.Println(dd.Id, dd.Score, dd.Metadata.AsMap())
		}
} else {
		log.Fatal(resp.Error.ErrorMessage)
}

--output
vector search
f9792fea-5054-41df-ae5a-1a53091e2e03 83.32547 map[_id:f9792fea-5054-41df-ae5a-1a53091e2e03 description:For a typical meal, I often enjoy a bowl of salad. gender:boy name:Mark type:6302]
9bb7a1a7-b0cc-4bf1-abc3-b4e59777ad28 79.38654 map[_id:9bb7a1a7-b0cc-4bf1-abc3-b4e59777ad28 description:On most days, I prefer having pasta for dinner. gender:girl name:swan type:122]
ad39be76-f268-43b8-9768-78ff3dd222e4 74.71306 map[_id:ad39be76-f268-43b8-9768-78ff3dd222e4 description:Sometimes, I grab a quick snack like nuts or fruit. gender:girl name:Ann type:142124]
52dadbb2-c180-4fbf-903f-0baa5c47d17b 71.83776 map[_id:52dadbb2-c180-4fbf-903f-0baa5c47d17b description:I tend to eat porridge whenever I feel hungry. gender:girl name:anna type:12632]
42571fd4-fcbe-4cb3-9321-2efc05c1fd68 67.498146 map[_id:42571fd4-fcbe-4cb3-9321-2efc05c1fd68 description:When I'm hungry, I usually have a sandwich. gender:boy name:John type:12]

FilterSearch

fmt.Println("pure search")
resp, err = dclient.FilterSearch(context.Background(), &edgeproto.SearchReq{
		CollectionName: "tcollection",
		Filter: map[string]string{
			"type": "12",
		},
		TopK: 5,
})
if err != nil {
		log.Fatal(err)
}
if resp.Status {
		for _, dd := range resp.GetCandidates() {
			fmt.Println(dd.Id, dd.Score, dd.Metadata.AsMap())
		}
} else {
		log.Fatal(resp.Error.ErrorMessage)
}

--output
pure search
42571fd4-fcbe-4cb3-9321-2efc05c1fd68 100 map[_id:42571fd4-fcbe-4cb3-9321-2efc05c1fd68 description:When I'm hungry, I usually have a sandwich. gender:boy name:John type:12]

HybridSearch

fmt.Println("hybrid search")
resp, err = dclient.HybridSearch(context.Background(), &edgeproto.SearchReq{
		CollectionName: "tcollection",
		Vector:         searchVec,
		Filter: map[string]string{
			"gender": "girl",
		},
		TopK: 5,
})
if err != nil {
		log.Fatal(err)
}
if resp.Status {
		for _, dd := range resp.GetCandidates() {
			fmt.Println(dd.Id, dd.Score, dd.Metadata.AsMap())
		}
} else {
		log.Fatal(resp.Error.ErrorMessage)
}
--output
9bb7a1a7-b0cc-4bf1-abc3-b4e59777ad28 79.38654 map[_id:9bb7a1a7-b0cc-4bf1-abc3-b4e59777ad28 description:On most days, I prefer having pasta for dinner. gender:girl name:swan type:122]
ad39be76-f268-43b8-9768-78ff3dd222e4 74.71306 map[_id:ad39be76-f268-43b8-9768-78ff3dd222e4 description:Sometimes, I grab a quick snack like nuts or fruit. gender:girl name:Ann type:142124]
52dadbb2-c180-4fbf-903f-0baa5c47d17b 71.83776 map[_id:52dadbb2-c180-4fbf-903f-0baa5c47d17b description:I tend to eat porridge whenever I feel hungry. gender:girl name:anna type:12632]

FullCode(main, main2(quantizedCollection), Search)

main

package main

import (
	"context"
	"fmt"
	"log"
	"testc/edgeproto"
	"testc/embeddings"

	"github.com/google/uuid"

	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials/insecure"
	"google.golang.org/protobuf/types/known/structpb"
)

func main() {
	embeddings.InitEmbeddings()
	dataString := []string{
		"When I'm hungry, I usually have a sandwich.",
		"I tend to eat porridge whenever I feel hungry.",
		"For a typical meal, I often enjoy a bowl of salad.",
	}

	query := "I'm hungry, what should I eat?"

	addString := []string{
		"Sometimes, I grab a quick snack like nuts or fruit.",
		"On most days, I prefer having pasta for dinner.",
	}
	type InsertRecord struct {
		Metadata   map[string]interface{}
		Id         string
		BucketName string
		Vector     []float32
	}
	var uid string
	firstRecord := make([]InsertRecord, 0)
	uid = uuid.New().String()
	firstRecord = append(firstRecord, InsertRecord{
		Metadata: map[string]interface{}{
			"_id":         uid,
			"description": dataString[0],
			"type":        12,
			"gender":      "boy",
			"name":        "John",
		},
	})
	uid = uuid.New().String()
	firstRecord = append(firstRecord, InsertRecord{
		Metadata: map[string]interface{}{
			"_id":         uid,
			"description": dataString[1],
			"type":        12632,
			"gender":      "girl",
			"name":        "anna",
		},
	})
	uid = uuid.New().String()
	firstRecord = append(firstRecord, InsertRecord{
		Metadata: map[string]interface{}{
			"_id":         uid,
			"description": dataString[2],
			"type":        6302,
			"gender":      "boy",
			"name":        "Mark",
		},
	})
	// create dataset
	originDataset := make([]*edgeproto.ModifyDataset, 0, 3)
	for i, data := range firstRecord {
		vec, err := embeddings.TextEmbedding(dataString[i])
		if err != nil {
			log.Fatal(err)
		}
		m, _ := structpb.NewStruct(data.Metadata)
		originDataset = append(originDataset, &edgeproto.ModifyDataset{
			CollectionName: "tcollection",
			Id:             data.Metadata["_id"].(string),
			Vector:         vec,
			Metadata:       m,
		})
	}
	laterRecord := make([]InsertRecord, 0)
	uid = uuid.New().String()
	laterRecord = append(laterRecord, InsertRecord{
		Metadata: map[string]interface{}{
			"_id":         uid,
			"description": addString[0],
			"type":        142124,
			"gender":      "girl",
			"name":        "Ann",
		},
	})
	uid = uuid.New().String()
	laterRecord = append(laterRecord, InsertRecord{
		Metadata: map[string]interface{}{
			"_id":         uid,
			"description": addString[1],
			"type":        122,
			"gender":      "girl",
			"name":        "swan",
		},
	})

	afterDataset := make([]*edgeproto.ModifyDataset, 0, 3)
	for i, data := range laterRecord {
		vec, err := embeddings.TextEmbedding(addString[i])
		if err != nil {
			log.Fatal(err)
		}
		m, _ := structpb.NewStruct(data.Metadata)
		afterDataset = append(afterDataset, &edgeproto.ModifyDataset{
			CollectionName: "tcollection",
			Id:             data.Metadata["_id"].(string),
			Vector:         vec,
			Metadata:       m,
		})
	}
	conn, err := grpc.Dial(":50051", grpc.WithTransportCredentials(insecure.NewCredentials()))
	fmt.Println(err)
	dclient := edgeproto.NewEdgeRpcClient(conn)

	res, err := dclient.CreateCollection(context.Background(), &edgeproto.Collection{
		CollectionName: "tcollection",
		Dim:            384,
		Distance:       edgeproto.Distance_Cosine,
		Quantization:   edgeproto.Quantization_None,
	})
	fmt.Println(2142)
	fmt.Println(res)
	fmt.Println(err)
	fmt.Println("11", err, res.Status)

	for _, data := range originDataset {
		res, err := dclient.Insert(context.Background(), data)
		fmt.Println(res.Status, res.Error, err)
	}
	// first search
	searchVec, _ := embeddings.TextEmbedding(query)
	fmt.Println("old data search")
	resp, err := dclient.VectorSearch(context.Background(), &edgeproto.SearchReq{
		CollectionName: "tcollection",
		Vector:         searchVec,
		TopK:           3,
	})
	if err != nil {
		log.Fatal(err)
	}
	if resp.Status {
		for _, dd := range resp.GetCandidates() {

			fmt.Println(dd.Id, dd.Score, dd.Metadata.AsMap())
		}
	} else {
		log.Fatal(res.Error.ErrorMessage)
	}
	//add after data
	for _, data := range afterDataset {
		res, err := dclient.Insert(context.Background(), data)
		fmt.Println(res.Status, res.Error, err)
	}
	// second search
	fmt.Println("old+new data search")
	resp, err = dclient.VectorSearch(context.Background(), &edgeproto.SearchReq{
		CollectionName: "tcollection",
		Vector:         searchVec,
		TopK:           5,
	})
	if err != nil {
		log.Fatal(err)
	}
	if resp.Status {
		for _, dd := range resp.GetCandidates() {
			fmt.Println(dd.Id, dd.Score, dd.Metadata.AsMap())
		}
	} else {
		log.Fatal(resp.Error.ErrorMessage)
	}
	cc, err := dclient.Flush(context.Background(), &edgeproto.CollectionName{
		CollectionName: "tcollection",
	})
	fmt.Println(cc, err)
}

main2

package main

import (
	"context"
	"fmt"
	"log"
	"testc/edgeproto"
	"testc/embeddings"

	"github.com/google/uuid"

	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials/insecure"
	"google.golang.org/protobuf/types/known/structpb"
)

func main() {
	embeddings.InitEmbeddings()
	dataString := []string{
		"When I'm hungry, I usually have a sandwich.",
		"I tend to eat porridge whenever I feel hungry.",
		"For a typical meal, I often enjoy a bowl of salad.",
	}

	query := "I'm hungry, what should I eat?"

	addString := []string{
		"Sometimes, I grab a quick snack like nuts or fruit.",
		"On most days, I prefer having pasta for dinner.",
	}
	type InsertRecord struct {
		Metadata   map[string]interface{}
		Id         string
		BucketName string
		Vector     []float32
	}
	var uid string
	firstRecord := make([]InsertRecord, 0)
	uid = uuid.New().String()
	firstRecord = append(firstRecord, InsertRecord{
		Metadata: map[string]interface{}{
			"_id":         uid,
			"description": dataString[0],
			"type":        12,
			"gender":      "boy",
			"name":        "John",
		},
	})
	uid = uuid.New().String()
	firstRecord = append(firstRecord, InsertRecord{
		Metadata: map[string]interface{}{
			"_id":         uid,
			"description": dataString[1],
			"type":        12632,
			"gender":      "girl",
			"name":        "anna",
		},
	})
	uid = uuid.New().String()
	firstRecord = append(firstRecord, InsertRecord{
		Metadata: map[string]interface{}{
			"_id":         uid,
			"description": dataString[2],
			"type":        6302,
			"gender":      "boy",
			"name":        "Mark",
		},
	})
	// create dataset
	originDataset := make([]*edgeproto.ModifyDataset, 0, 3)
	for i, data := range firstRecord {
		vec, err := embeddings.TextEmbedding(dataString[i])
		if err != nil {
			log.Fatal(err)
		}
		m, _ := structpb.NewStruct(data.Metadata)
		originDataset = append(originDataset, &edgeproto.ModifyDataset{
			CollectionName: "tcollection",
			Id:             data.Metadata["_id"].(string),
			Vector:         vec,
			Metadata:       m,
		})
	}
	laterRecord := make([]InsertRecord, 0)
	uid = uuid.New().String()
	laterRecord = append(laterRecord, InsertRecord{
		Metadata: map[string]interface{}{
			"_id":         uid,
			"description": addString[0],
			"type":        142124,
			"gender":      "girl",
			"name":        "Ann",
		},
	})
	uid = uuid.New().String()
	laterRecord = append(laterRecord, InsertRecord{
		Metadata: map[string]interface{}{
			"_id":         uid,
			"description": addString[1],
			"type":        122,
			"gender":      "girl",
			"name":        "swan",
		},
	})

	afterDataset := make([]*edgeproto.ModifyDataset, 0, 3)
	for i, data := range laterRecord {
		vec, err := embeddings.TextEmbedding(addString[i])
		if err != nil {
			log.Fatal(err)
		}
		m, _ := structpb.NewStruct(data.Metadata)
		afterDataset = append(afterDataset, &edgeproto.ModifyDataset{
			CollectionName: "tcollection",
			Id:             data.Metadata["_id"].(string),
			Vector:         vec,
			Metadata:       m,
		})
	}
	conn, err := grpc.Dial(":50051", grpc.WithTransportCredentials(insecure.NewCredentials()))
	fmt.Println(err)
	dclient := edgeproto.NewEdgeRpcClient(conn)

	res, err := dclient.CreateCollection(context.Background(), &edgeproto.Collection{
		CollectionName: "tcollection",
		Dim:            384,
		Distance:       edgeproto.Distance_Cosine,
		Quantization:   edgeproto.Quantization_F16,
	})
	fmt.Println(2142)
	fmt.Println(res)
	fmt.Println(err)
	fmt.Println("11", err, res.Status)

	for _, data := range originDataset {
		res, err := dclient.Insert(context.Background(), data)
		fmt.Println(res.Status, res.Error, err)
	}
	// first search
	searchVec, _ := embeddings.TextEmbedding(query)
	fmt.Println("old data search")
	resp, err := dclient.VectorSearch(context.Background(), &edgeproto.SearchReq{
		CollectionName: "tcollection",
		Vector:         searchVec,
		TopK:           3,
	})
	if err != nil {
		log.Fatal(err)
	}
	if resp.Status {
		for _, dd := range resp.GetCandidates() {

			fmt.Println(dd.Id, dd.Score, dd.Metadata.AsMap())
		}
	} else {
		log.Fatal(res.Error.ErrorMessage)
	}
	//add after data
	for _, data := range afterDataset {
		res, err := dclient.Insert(context.Background(), data)
		fmt.Println(res.Status, res.Error, err)
	}
	// second search
	fmt.Println("old+new data search")
	resp, err = dclient.VectorSearch(context.Background(), &edgeproto.SearchReq{
		CollectionName: "tcollection",
		Vector:         searchVec,
		TopK:           5,
	})
	if err != nil {
		log.Fatal(err)
	}
	if resp.Status {
		for _, dd := range resp.GetCandidates() {
			fmt.Println(dd.Id, dd.Score, dd.Metadata.AsMap())
		}
	} else {
		log.Fatal(resp.Error.ErrorMessage)
	}
	cc, err := dclient.Flush(context.Background(), &edgeproto.CollectionName{
		CollectionName: "tcollection",
	})
	fmt.Println(cc, err)
}

search

package main

import (
	"context"
	"fmt"
	"log"
	"testc/edgeproto"
	"testc/embeddings"

	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials/insecure"
)

func main() {
	embeddings.InitEmbeddings()
	query := "I'm hungry, what should I eat?"
	conn, err := grpc.Dial(":50051", grpc.WithTransportCredentials(insecure.NewCredentials()))
	fmt.Println(err)

	dclient := edgeproto.NewEdgeRpcClient(conn)
	searchVec, _ := embeddings.TextEmbedding(query)

	fmt.Println("vector search- not load collection")
	resp, err := dclient.VectorSearch(context.Background(), &edgeproto.SearchReq{
		CollectionName: "tcollection",
		Vector:         searchVec,
		TopK:           5,
	})
	fmt.Println(resp, err)
	cc, err := dclient.LoadCollection(context.Background(), &edgeproto.CollectionName{
		CollectionName: "tcollection",
	})
	fmt.Println(cc, err)
	fmt.Println("vector search")
	resp, err = dclient.VectorSearch(context.Background(), &edgeproto.SearchReq{
		CollectionName: "tcollection",
		Vector:         searchVec,
		TopK:           5,
	})
	if err != nil {
		log.Fatal(err)
	}
	if resp.Status {
		for _, dd := range resp.GetCandidates() {
			fmt.Println(dd.Id, dd.Score, dd.Metadata.AsMap())
		}
	} else {
		log.Fatal(resp.Error.ErrorMessage)
	}
	fmt.Println("pure search")
	resp, err = dclient.FilterSearch(context.Background(), &edgeproto.SearchReq{
		CollectionName: "tcollection",
		Filter: map[string]string{
			"type": "12",
		},
		TopK: 5,
	})
	if err != nil {
		log.Fatal(err)
	}
	if resp.Status {
		for _, dd := range resp.GetCandidates() {
			fmt.Println(dd.Id, dd.Score, dd.Metadata.AsMap())
		}
	} else {
		log.Fatal(resp.Error.ErrorMessage)
	}
	fmt.Println("hybrid search")
	resp, err = dclient.HybridSearch(context.Background(), &edgeproto.SearchReq{
		CollectionName: "tcollection",
		Vector:         searchVec,
		Filter: map[string]string{
			"gender": "girl",
		},
		TopK: 5,
	})
	if err != nil {
		log.Fatal(err)
	}
	if resp.Status {
		for _, dd := range resp.GetCandidates() {
			fmt.Println(dd.Id, dd.Score, dd.Metadata.AsMap())
		}
	} else {
		log.Fatal(resp.Error.ErrorMessage)
	}
}

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

2024_11_11_release.md

2024_11_11_release.md

RPC Call

TEST Code

Create Collection

Create QuantizedCollection

Create Data

VectorSearch

FilterSearch

HybridSearch

FullCode(main, main2(quantizedCollection), Search)

main

main2

search

Files

2024_11_11_release.md

Latest commit

History

2024_11_11_release.md

File metadata and controls

RPC Call

TEST Code

Create Collection

Create QuantizedCollection

Create Data

VectorSearch

FilterSearch

HybridSearch

FullCode(main, main2(quantizedCollection), Search)

main

main2

search