Skip to content

Commit 7d7a1a6

Browse files
authored
feat: add url agent (#14368)
* feat: add url agent * feat: add url agent
1 parent 01b6e61 commit 7d7a1a6

22 files changed

Lines changed: 281 additions & 35 deletions

File tree

‎agent/app/dto/cronjob.go‎

Lines changed: 41 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
package dto
22

33
import (
4+
"bytes"
5+
"encoding/json"
6+
"fmt"
47
"time"
58
)
69

@@ -16,6 +19,35 @@ type CronjobSpec struct {
1619
Spec string `json:"spec" validate:"required"`
1720
}
1821

22+
// CronjobURL pairs a URL with its optional User-Agent value.
23+
type CronjobURL struct {
24+
URL string `json:"url"`
25+
Header string `json:"header"`
26+
}
27+
28+
// UnmarshalJSON accepts URL strings submitted by older frontends.
29+
func (item *CronjobURL) UnmarshalJSON(data []byte) error {
30+
data = bytes.TrimSpace(data)
31+
if bytes.Equal(data, []byte("null")) {
32+
return fmt.Errorf("URL group must not be null")
33+
}
34+
if len(data) > 0 && data[0] == '"' {
35+
var url string
36+
if err := json.Unmarshal(data, &url); err != nil {
37+
return err
38+
}
39+
*item = CronjobURL{URL: url}
40+
return nil
41+
}
42+
type plain CronjobURL
43+
var decoded plain
44+
if err := json.Unmarshal(data, &decoded); err != nil {
45+
return err
46+
}
47+
*item = CronjobURL(decoded)
48+
return nil
49+
}
50+
1951
type CronjobOperate struct {
2052
ID uint `json:"id"`
2153
Name string `json:"name" validate:"required"`
@@ -38,6 +70,7 @@ type CronjobOperate struct {
3870
DBType string `json:"dbType"`
3971
DBName string `json:"dbName"`
4072
URL string `json:"url"`
73+
URLItems []CronjobURL `json:"urlItems"`
4174
IsDir bool `json:"isDir"`
4275
SourceDir string `json:"sourceDir"`
4376
SnapshotRule SnapshotRule `json:"snapshotRule"`
@@ -141,13 +174,14 @@ type CronjobTrans struct {
141174
SpecCustom bool `json:"specCustom"`
142175
Spec string `json:"spec"`
143176

144-
Executor string `json:"executor"`
145-
ScriptMode string `json:"scriptMode"`
146-
Script string `json:"script"`
147-
Command string `json:"command"`
148-
ContainerName string `json:"containerName"`
149-
User string `json:"user"`
150-
URL string `json:"url"`
177+
Executor string `json:"executor"`
178+
ScriptMode string `json:"scriptMode"`
179+
Script string `json:"script"`
180+
Command string `json:"command"`
181+
ContainerName string `json:"containerName"`
182+
User string `json:"user"`
183+
URL string `json:"url"`
184+
URLItems []CronjobURL `json:"urlItems"`
151185

152186
ScriptName string `json:"scriptName"`
153187
Apps []TransHelper `json:"apps"`

‎agent/app/model/cronjob.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ type Cronjob struct {
2626
DBType string `json:"dbType"`
2727
DBName string `json:"dbName"`
2828
URL string `json:"url"`
29+
URLItemsJSON string `gorm:"column:url_items;type:text" json:"-"`
2930
IsDir bool `json:"isDir"`
3031
SourceDir string `json:"sourceDir"`
3132
SnapshotRule string `json:"snapshotRule"`

‎agent/app/service/cronjob.go‎

Lines changed: 103 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,9 @@ func (u *CronjobService) SearchWithPage(search dto.PageCronjob) (int64, interfac
5757
repo.WithByGroups(search.GroupIDs),
5858
repo.WithByLikeName(search.Info),
5959
repo.WithOrderRuleBy(search.OrderBy, search.Order))
60+
if err != nil {
61+
return 0, nil, err
62+
}
6063
var dtoCronjobs []dto.CronjobInfo
6164
for _, cronjob := range cronjobs {
6265
var item dto.CronjobInfo
@@ -92,6 +95,9 @@ func (u *CronjobService) SearchWithPage(search dto.PageCronjob) (int64, interfac
9295

9396
func (u *CronjobService) LoadInfo(req dto.OperateByID) (*dto.CronjobOperate, error) {
9497
cronjob, err := cronjobRepo.Get(repo.WithByID(req.ID))
98+
if err != nil {
99+
return nil, err
100+
}
95101
var item dto.CronjobOperate
96102
if err := copier.Copy(&item, &cronjob); err != nil {
97103
return nil, buserr.WithDetail("ErrStructTransform", err.Error(), nil)
@@ -115,6 +121,7 @@ func (u *CronjobService) LoadInfo(req dto.OperateByID) (*dto.CronjobOperate, err
115121
if cronjob.Type == "cleanLog" {
116122
_ = json.Unmarshal([]byte(cronjob.Config), &item.CleanLogConfig)
117123
}
124+
item.URLItems, err = loadCronjobURLs(cronjob)
118125
return &item, err
119126
}
120127

@@ -149,6 +156,10 @@ func (u *CronjobService) Export(req dto.OperateByIDs) (string, error) {
149156
Secret: cronjob.Secret,
150157
Args: cronjob.Args,
151158
}
159+
item.URLItems, err = loadCronjobURLs(cronjob)
160+
if err != nil {
161+
return "", err
162+
}
152163
switch cronjob.Type {
153164
case "app":
154165
if cronjob.AppID == "all" {
@@ -219,6 +230,15 @@ func (u *CronjobService) Export(req dto.OperateByIDs) (string, error) {
219230

220231
func (u *CronjobService) Import(req []dto.CronjobTrans, operator string) error {
221232
for _, item := range req {
233+
if item.Type == "curl" {
234+
items := item.URLItems
235+
if items == nil {
236+
items = legacyCronjobURLs(item.URL)
237+
}
238+
if err := validateCronjobURLs(items); err != nil {
239+
return err
240+
}
241+
}
222242
advanced, err := cronJobAlertAdvancedParams(item.AlertTriggerMode)
223243
if err != nil {
224244
return err
@@ -256,6 +276,9 @@ func (u *CronjobService) Import(req []dto.CronjobTrans, operator string) error {
256276
Secret: item.Secret,
257277
Args: item.Args,
258278
}
279+
if err := storeCronjobURLs(&cronjob, item.URLItems); err != nil {
280+
return err
281+
}
259282
hasNotFound := false
260283
switch item.Type {
261284
case "app":
@@ -605,6 +628,9 @@ func (u *CronjobService) Create(req dto.CronjobOperate, operator string) error {
605628
if err := copier.Copy(&cronjob, &req); err != nil {
606629
return buserr.WithDetail("ErrStructTransform", err.Error(), nil)
607630
}
631+
if err := storeCronjobURLs(&cronjob, req.URLItems); err != nil {
632+
return err
633+
}
608634
if cronjob.Type == "snapshot" {
609635
rule, err := json.Marshal(req.SnapshotRule)
610636
if err != nil {
@@ -754,6 +780,9 @@ func (u *CronjobService) Update(id uint, req dto.CronjobOperate, operator string
754780
upMap := make(map[string]interface{})
755781
cronjob.EntryIDs = cronModel.EntryIDs
756782
cronjob.Type = cronModel.Type
783+
if err := storeCronjobURLs(&cronjob, req.URLItems); err != nil {
784+
return err
785+
}
757786
spec := cronjob.Spec
758787
if cronModel.Status == constant.StatusEnable {
759788
newEntryIDs, err := u.StartJob(&cronjob, true)
@@ -789,7 +818,8 @@ func (u *CronjobService) Update(id uint, req dto.CronjobOperate, operator string
789818
upMap["exclusion_rules"] = req.ExclusionRules
790819
upMap["db_type"] = req.DBType
791820
upMap["db_name"] = req.DBName
792-
upMap["url"] = req.URL
821+
upMap["url"] = cronjob.URL
822+
upMap["url_items"] = cronjob.URLItemsJSON
793823
upMap["is_dir"] = req.IsDir
794824
upMap["source_dir"] = req.SourceDir
795825
upMap["snapshot_rule"] = cronjob.SnapshotRule
@@ -878,3 +908,75 @@ func (u *CronjobService) AddCronJob(cronjob *model.Cronjob) (int, error) {
878908
global.LOG.Infof("start cronjob entryID: %d", entryID)
879909
return int(entryID), nil
880910
}
911+
912+
func legacyCronjobURLs(raw string) []dto.CronjobURL {
913+
items := make([]dto.CronjobURL, 0)
914+
for _, value := range strings.Split(raw, ",") {
915+
if value = strings.TrimSpace(value); value != "" {
916+
items = append(items, dto.CronjobURL{URL: value})
917+
}
918+
}
919+
return items
920+
}
921+
922+
func loadCronjobURLs(job model.Cronjob) ([]dto.CronjobURL, error) {
923+
if job.URLItemsJSON == "" {
924+
return legacyCronjobURLs(job.URL), nil
925+
}
926+
var items []dto.CronjobURL
927+
if err := json.Unmarshal([]byte(job.URLItemsJSON), &items); err != nil {
928+
return nil, fmt.Errorf("invalid cronjob URL configuration")
929+
}
930+
return items, nil
931+
}
932+
933+
func validateCronjobURLs(items []dto.CronjobURL) error {
934+
if len(items) == 0 {
935+
return fmt.Errorf("at least one URL is required")
936+
}
937+
for i, item := range items {
938+
if strings.TrimSpace(item.URL) == "" || strings.ContainsAny(item.URL, "\r\n\x00") {
939+
return fmt.Errorf("invalid URL in group %d", i+1)
940+
}
941+
if strings.ContainsAny(item.Header, "\r\n\x00") {
942+
return fmt.Errorf("User-Agent in group %d must not contain line breaks or NUL", i+1)
943+
}
944+
}
945+
return nil
946+
}
947+
948+
func storeCronjobURLs(job *model.Cronjob, items []dto.CronjobURL) error {
949+
job.URLItemsJSON = ""
950+
if job.Type != "curl" {
951+
return nil
952+
}
953+
// A missing field indicates a legacy API request or import.
954+
if items == nil {
955+
items = legacyCronjobURLs(job.URL)
956+
}
957+
if err := validateCronjobURLs(items); err != nil {
958+
return err
959+
}
960+
normalized := make([]dto.CronjobURL, len(items))
961+
urls := make([]string, len(items))
962+
for i, item := range items {
963+
item.URL = strings.TrimSpace(item.URL)
964+
normalized[i] = item
965+
urls[i] = item.URL
966+
}
967+
data, err := json.Marshal(normalized)
968+
if err != nil {
969+
return err
970+
}
971+
job.URLItemsJSON = string(data)
972+
job.URL = strings.Join(urls, ",")
973+
return nil
974+
}
975+
976+
func cronjobCurlArgs(item dto.CronjobURL) []string {
977+
args := make([]string, 0, 4)
978+
if item.Header != "" {
979+
args = append(args, "--user-agent", item.Header)
980+
}
981+
return append(args, "--", item.URL)
982+
}

‎agent/app/service/cronjob_helper.go‎

Lines changed: 15 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -109,10 +109,9 @@ func (u *CronjobService) loadTask(cronjob *model.Cronjob, record *model.JobRecor
109109
u.handleShell(*cronjob, taskItem)
110110
u.removeExpiredLog(*cronjob)
111111
case "curl":
112-
if len(cronjob.URL) == 0 {
113-
return fmt.Errorf("the url is empty and is skipped")
112+
if err := u.handleCurl(*cronjob, taskItem); err != nil {
113+
return err
114114
}
115-
u.handleCurl(*cronjob, taskItem)
116115
u.removeExpiredLog(*cronjob)
117116
case "ntp":
118117
u.handleNtpSync(*cronjob, taskItem)
@@ -197,18 +196,24 @@ func (u *CronjobService) handleShell(cronjob model.Cronjob, taskItem *task.Task)
197196
}, nil, int(cronjob.RetryTimes), time.Duration(cronjob.Timeout)*time.Second)
198197
}
199198

200-
func (u *CronjobService) handleCurl(cronjob model.Cronjob, taskItem *task.Task) {
201-
urls := strings.Split(cronjob.URL, ",")
202-
for _, url := range urls {
203-
if len(strings.TrimSpace(url)) == 0 {
204-
continue
205-
}
199+
func (u *CronjobService) handleCurl(cronjob model.Cronjob, taskItem *task.Task) error {
200+
items, err := loadCronjobURLs(cronjob)
201+
if err != nil {
202+
return err
203+
}
204+
if err := validateCronjobURLs(items); err != nil {
205+
return err
206+
}
207+
for _, item := range items {
208+
url := item.URL
209+
args := cronjobCurlArgs(item)
206210
taskItem.AddSubTaskWithOps(i18n.GetWithName("HandleCurl", url), func(t *task.Task) error {
207211
taskItem.LogStart(i18n.GetWithName("HandleCurl", url))
208212
cmdMgr := cmd.NewCommandMgr(cmd.WithTask(*taskItem))
209-
return cmdMgr.Run("curl", url)
213+
return cmdMgr.Run("curl", args...)
210214
}, nil, int(cronjob.RetryTimes), time.Duration(cronjob.Timeout)*time.Second)
211215
}
216+
return nil
212217
}
213218

214219
func (u *CronjobService) handleNtpSync(cronjob model.Cronjob, taskItem *task.Task) {

‎agent/init/migration/migrate.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,7 @@ func agentDBMigrations() []*gormigrate.Migration {
115115
migrations.AddLocalModelAccountLink,
116116
migrations.AddOllamaModelAccountSync,
117117
migrations.AddContainerImageCheckUpdates,
118+
migrations.AddCronjobURLItems,
118119
migrations.AddModelQuickJump,
119120
migrations.MigrateVMDatabase,
120121
migrations.UpdateModelServicesQuickJumpRouter,

‎agent/init/migration/migrations/init.go‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1946,3 +1946,13 @@ var AddGPUGroup = &gormigrate.Migration{
19461946
return tx.AutoMigrate(&model.GPUGroup{})
19471947
},
19481948
}
1949+
1950+
var AddCronjobURLItems = &gormigrate.Migration{
1951+
ID: "20261109-cronjob-url-items",
1952+
Migrate: func(tx *gorm.DB) error {
1953+
if tx.Migrator().HasColumn(&model.Cronjob{}, "URLItemsJSON") {
1954+
return nil
1955+
}
1956+
return tx.Migrator().AddColumn(&model.Cronjob{}, "URLItemsJSON")
1957+
},
1958+
}

‎core/cmd/server/docs/swagger.json‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37335,6 +37335,16 @@
3733537335
},
3733637336
"type": "object"
3733737337
},
37338+
"dto.CronjobURL": {
37339+
"type": "object",
37340+
"properties": {
37341+
"url": { "type": "string" },
37342+
"header": {
37343+
"type": "string",
37344+
"description": "Optional User-Agent value for this URL"
37345+
}
37346+
}
37347+
},
3733837348
"dto.CronjobOperate": {
3733937349
"properties": {
3734037350
"alertCount": {
@@ -37436,6 +37446,12 @@
3743637446
"type": {
3743737447
"type": "string"
3743837448
},
37449+
"urlItems": {
37450+
"type": "array",
37451+
"items": {
37452+
"$ref": "#/definitions/dto.CronjobURL"
37453+
}
37454+
},
3743937455
"url": {
3744037456
"type": "string"
3744137457
},
@@ -37562,6 +37578,12 @@
3756237578
"type": {
3756337579
"type": "string"
3756437580
},
37581+
"urlItems": {
37582+
"type": "array",
37583+
"items": {
37584+
"$ref": "#/definitions/dto.CronjobURL"
37585+
}
37586+
},
3756537587
"url": {
3756637588
"type": "string"
3756737589
},

0 commit comments

Comments
 (0)