Skip to content

98 feature optimize content add endpoints #103

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
LucRoy wants to merge 2 commits into
base: main
Choose a base branch
from
Open

98 feature optimize content add endpoints #103

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
cd9dd32
added db transaction to content add
LucRoy Jul 24, 2023
5bffc64
change /contents to pull all datasets and create transaction for create
LucRoy Jul 24, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
104 changes: 59 additions & 45 deletions api/contents.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,29 +113,34 @@ func ConfigureContentsRouter(e *echo.Group, dldm *core.DeltaDM) {
}
}

for _, cnt := range content {
// Check for bad data
if cnt.CommP == "" || cnt.PayloadCID == "" || cnt.PaddedSize == 0 || cnt.Size == 0 {
results.Fail = append(results.Fail, cnt.CommP)
continue
}
dldm.DB.Transaction(func(tx *gorm.DB) error {
for _, cnt := range content {
// Check for bad data
if cnt.CommP == "" || cnt.PayloadCID == "" || cnt.PaddedSize == 0 || cnt.Size == 0 {
results.Fail = append(results.Fail, cnt.CommP)
continue
}

cnt.DatasetID = dataset.ID
cnt.DatasetID = dataset.ID

err := dldm.DB.Create(&cnt).Error
if err != nil {
results.Fail = append(results.Fail, cnt.CommP)
continue
}
err := tx.Create(&cnt).Error
if err != nil {
results.Fail = append(results.Fail, cnt.CommP)
continue
}

results.Success = append(results.Success, cnt.CommP)
}
results.Success = append(results.Success, cnt.CommP)
}
return nil
})

return c.JSON(http.StatusOK, results)
})

contents.POST("", func(c echo.Context) error {
var content []ContentCollection
var datasets []db.Dataset

results := struct {
Success []string `json:"success"`
Fail []string `json:"fail"`
Expand All @@ -148,47 +153,56 @@ func ConfigureContentsRouter(e *echo.Group, dldm *core.DeltaDM) {
return err
}

cache := make(map[string]uint)
if err := dldm.DB.Model(&db.Dataset{}).Find(&datasets).Error; err != nil {
Copy link
Contributor

@jcace jcace Jul 25, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm, doesn't this .Find() only work on the primary key i.e, dataset ID ? We need to search via name instead here

Copy link
Contributor Author

@LucRoy LucRoy Jul 26, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

mmm here we are listing all datasets instead of querying per name. Let me know if you have a more efficient way of doing it.

return err
}

dldm.DB.Transaction(func(tx *gorm.DB) error {
for _, cnt := range content {
// Check for bad data
if cnt.CommP == "" || cnt.PayloadCID == "" || cnt.PaddedSize == 0 || cnt.Size == 0 ||
cnt.Collection == "" || cnt.ContentLocation == "" {
log.Debugf("Missing required parameters for commP: %s", cnt.CommP)
results.Fail = append(results.Fail, cnt.CommP)
continue
}

for _, cnt := range content {
var dataset db.Dataset
// Check for bad data
if cnt.CommP == "" || cnt.PayloadCID == "" || cnt.PaddedSize == 0 || cnt.Size == 0 ||
cnt.Collection == "" || cnt.ContentLocation == "" {
log.Debugf("Missing required parameters for commP: %s", cnt.CommP)
results.Fail = append(results.Fail, cnt.CommP)
continue
}
// Check if dataset exists
var collection *db.Dataset
for _, dataset := range datasets {
if dataset.Name == cnt.Collection {
collection = &dataset
break
}
}

// Check if dataset exists
if cache[cnt.Collection] == 0 {
if dldm.DB.Model(&db.Dataset{}).Where("name = ?", cnt.Collection).First(&dataset).Error != nil {
log.Debugf("Collection not found for: %s", cnt.Collection)
if collection == nil {
log.Debugf("Collection %s not found for Commp: %s", cnt.Collection, cnt.CommP)
results.Fail = append(results.Fail, cnt.CommP)
continue
}
}

cache[cnt.Collection] = dataset.ID
dbc := db.Content{
CommP: cnt.CommP,
PayloadCID: cnt.PayloadCID,
PaddedSize: cnt.PaddedSize,
Size: cnt.Size,
DatasetID: collection.ID,
ContentLocation: cnt.ContentLocation,
}

dbc := db.Content{
CommP: cnt.CommP,
PayloadCID: cnt.PayloadCID,
PaddedSize: cnt.PaddedSize,
Size: cnt.Size,
DatasetID: dataset.ID,
ContentLocation: cnt.ContentLocation,
}
err := tx.Create(&dbc).Error
if err != nil {
log.Debugf("Could not create DB record: %s", err.Error())
results.Fail = append(results.Fail, cnt.CommP)
continue
}

err := dldm.DB.Create(&dbc).Error
if err != nil {
log.Debugf("Could not create DB record: %s", err.Error())
results.Fail = append(results.Fail, cnt.CommP)
continue
results.Success = append(results.Success, cnt.CommP)
}

results.Success = append(results.Success, cnt.CommP)
}
return nil
})

return c.JSON(http.StatusOK, results)
})
Expand Down