Skip to content

Commit cbc8b38

Browse files
WVerlaekWVerlaek
authored
[ws-daemon] Add OWI to logs (#18017)
1 parent 76b585e commit cbc8b38

File tree

13 files changed

+58
-44
lines changed

13 files changed

+58
-44
lines changed

‎components/ws-daemon/cmd/run.go

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212
"time"
1313

1414
"golang.org/x/xerrors"
15+
"k8s.io/klog/v2"
1516

1617
"github.com/bombsimon/logrusr/v2"
1718
"github.com/heptiolabs/healthcheck"
@@ -40,7 +41,10 @@ var runCmd = &cobra.Command{
4041

4142
createLVMDevices()
4243

43-
ctrl.SetLogger(logrusr.New(log.Log))
44+
baseLogger := logrusr.New(log.Log)
45+
ctrl.SetLogger(baseLogger)
46+
// Set the logger used by k8s (e.g. client-go).
47+
klog.SetLogger(baseLogger)
4448

4549
dmn, err := daemon.NewDaemon(cfg.Daemon)
4650
if err != nil {

‎components/ws-daemon/pkg/cgroup/plugin_iolimit_v2.go

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -58,29 +58,29 @@ func (c *IOLimiterV2) Apply(ctx context.Context, opts *PluginOptions) error {
5858
}()
5959

6060
go func() {
61-
log.WithField("cgroupPath", opts.CgroupPath).Debug("starting io limiting")
61+
log.WithFields(log.OWI("", "", opts.InstanceId)).WithField("cgroupPath", opts.CgroupPath).Debug("starting io limiting")
6262

6363
_, err := v2.NewManager(opts.BasePath, filepath.Join("/", opts.CgroupPath), c.limits)
6464
if err != nil {
65-
log.WithError(err).WithField("basePath", opts.BasePath).WithField("cgroupPath", opts.CgroupPath).WithField("limits", c.limits).Warn("cannot write IO limits")
65+
log.WithError(err).WithFields(log.OWI("", "", opts.InstanceId)).WithField("basePath", opts.BasePath).WithField("cgroupPath", opts.CgroupPath).WithField("limits", c.limits).Warn("cannot write IO limits")
6666
}
6767

6868
for {
6969
select {
7070
case <-update:
7171
_, err := v2.NewManager(opts.BasePath, filepath.Join("/", opts.CgroupPath), c.limits)
7272
if err != nil {
73-
log.WithError(err).WithField("basePath", opts.BasePath).WithField("cgroupPath", opts.CgroupPath).WithField("limits", c.limits).Error("cannot write IO limits")
73+
log.WithError(err).WithFields(log.OWI("", "", opts.InstanceId)).WithField("basePath", opts.BasePath).WithField("cgroupPath", opts.CgroupPath).WithField("limits", c.limits).Error("cannot write IO limits")
7474
}
7575
case <-ctx.Done():
7676
// Prior to shutting down though, we need to reset the IO limits to ensure we don't have
7777
// processes stuck in the uninterruptable "D" (disk sleep) state. This would prevent the
7878
// workspace pod from shutting down.
7979
_, err := v2.NewManager(opts.BasePath, filepath.Join("/", opts.CgroupPath), &v2.Resources{})
8080
if err != nil {
81-
log.WithError(err).WithField("cgroupPath", opts.CgroupPath).Error("cannot write IO limits")
81+
log.WithError(err).WithFields(log.OWI("", "", opts.InstanceId)).WithField("cgroupPath", opts.CgroupPath).Error("cannot write IO limits")
8282
}
83-
log.WithField("cgroupPath", opts.CgroupPath).Debug("stopping io limiting")
83+
log.WithFields(log.OWI("", "", opts.InstanceId)).WithField("cgroupPath", opts.CgroupPath).Debug("stopping io limiting")
8484
return
8585
}
8686
}

‎components/ws-daemon/pkg/cgroup/plugin_proc_limit_v2.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,15 +57,15 @@ func (c *ProcLimiterV2) Apply(ctx context.Context, opts *PluginOptions) error {
5757

5858
_, err := v2.NewManager(opts.BasePath, filepath.Join("/", opts.CgroupPath), c.limits)
5959
if err != nil {
60-
log.WithError(err).WithField("basePath", opts.BasePath).WithField("cgroupPath", opts.CgroupPath).WithField("limits", c.limits).Error("cannot write proc limits")
60+
log.WithError(err).WithFields(log.OWI("", "", opts.InstanceId)).WithField("basePath", opts.BasePath).WithField("cgroupPath", opts.CgroupPath).WithField("limits", c.limits).Error("cannot write proc limits")
6161
}
6262

6363
for {
6464
select {
6565
case <-update:
6666
_, err := v2.NewManager(opts.BasePath, filepath.Join("/", opts.CgroupPath), c.limits)
6767
if err != nil {
68-
log.WithError(err).WithField("basePath", opts.BasePath).WithField("cgroupPath", opts.CgroupPath).WithField("limits", c.limits).Error("cannot write proc limits")
68+
log.WithError(err).WithFields(log.OWI("", "", opts.InstanceId)).WithField("basePath", opts.BasePath).WithField("cgroupPath", opts.CgroupPath).WithField("limits", c.limits).Error("cannot write proc limits")
6969
}
7070
case <-ctx.Done():
7171
return

‎components/ws-daemon/pkg/cgroup/plugin_process_priority_v2.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ func (c *ProcessPriorityV2) Apply(ctx context.Context, opts *PluginOptions) erro
7878
// the target cgroup/workspace has gone
7979
return nil
8080
} else if err != nil {
81-
log.WithField("path", fullCgroupPath).WithError(err).Errorf("cannot read cgroup.procs file")
81+
log.WithFields(log.OWI("", "", opts.InstanceId)).WithField("path", fullCgroupPath).WithError(err).Errorf("cannot read cgroup.procs file")
8282
return err
8383
}
8484

@@ -92,7 +92,7 @@ func (c *ProcessPriorityV2) Apply(ctx context.Context, opts *PluginOptions) erro
9292

9393
pid, err := strconv.ParseInt(line, 10, 64)
9494
if err != nil {
95-
log.WithError(err).WithField("line", line).Warn("cannot parse pid")
95+
log.WithError(err).WithFields(log.OWI("", "", opts.InstanceId)).WithField("line", line).Warn("cannot parse pid")
9696
continue
9797
}
9898

@@ -102,7 +102,7 @@ func (c *ProcessPriorityV2) Apply(ctx context.Context, opts *PluginOptions) erro
102102
continue
103103
}
104104

105-
log.WithError(err).WithField("pid", pid).Warn("cannot get process")
105+
log.WithError(err).WithFields(log.OWI("", "", opts.InstanceId)).WithField("pid", pid).Warn("cannot get process")
106106
continue
107107
}
108108

‎components/ws-daemon/pkg/cgroup/plugin_psi.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -91,20 +91,20 @@ func (p *PSIMetrics) scrape(cpu *cgroups.Cpu, memory *cgroups.Memory, io *cgroup
9191
p.cpu.WithLabelValues(p.nodeName, instanceID, "some").Set(float64(psi.Some))
9292
p.cpu.WithLabelValues(p.nodeName, instanceID, "full").Set(float64(psi.Full))
9393
} else if !os.IsNotExist(err) {
94-
log.WithError(err).Warn("could not retrieve cpu psi")
94+
log.WithError(err).WithFields(log.OWI("", "", instanceID)).Warn("could not retrieve cpu psi")
9595
}
9696

9797
if psi, err := memory.PSI(); err == nil {
9898
p.memory.WithLabelValues(p.nodeName, instanceID, "some").Set(float64(psi.Some))
9999
p.memory.WithLabelValues(p.nodeName, instanceID, "full").Set(float64(psi.Full))
100100
} else if !os.IsNotExist(err) {
101-
log.WithError(err).Warn("could not retrieve memory psi")
101+
log.WithError(err).WithFields(log.OWI("", "", instanceID)).Warn("could not retrieve memory psi")
102102
}
103103

104104
if psi, err := io.PSI(); err == nil {
105105
p.io.WithLabelValues(p.nodeName, instanceID, "some").Set(float64(psi.Some))
106106
p.io.WithLabelValues(p.nodeName, instanceID, "full").Set(float64(psi.Full))
107107
} else if !os.IsNotExist(err) {
108-
log.WithError(err).Warn("could not retrieve io psi")
108+
log.WithError(err).WithFields(log.OWI("", "", instanceID)).Warn("could not retrieve io psi")
109109
}
110110
}

‎components/ws-daemon/pkg/container/containerd.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -300,7 +300,7 @@ func (s *Containerd) handleNewTask(cid string, rootfs []*types.Mount, pid uint32
300300
mnts, err := s.Client.SnapshotService(info.Snapshotter).Mounts(ctx, info.SnapshotKey)
301301
cancel()
302302
if err != nil {
303-
log.WithError(err).Warnf("cannot get mounts for container %v", cid)
303+
log.WithError(err).WithFields(log.OWI(info.OwnerID, info.WorkspaceID, info.InstanceID)).Warnf("cannot get mounts for container %v", cid)
304304
}
305305
for _, m := range mnts {
306306
rootfs = append(rootfs, &types.Mount{

‎components/ws-daemon/pkg/content/hooks.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,7 @@ func hookSetupWorkspaceLocation(ctx context.Context, ws *session.Workspace) (err
9191
// in the very unlikely event that the workspace Pod did not mount (and thus create) the workspace directory, create it
9292
err = os.Mkdir(location, 0755)
9393
if os.IsExist(err) {
94-
log.WithError(err).WithField("location", location).Debug("ran into non-atomic workspace location existence check")
94+
log.WithError(err).WithFields(ws.OWI()).WithField("location", location).Debug("ran into non-atomic workspace location existence check")
9595
} else if err != nil {
9696
return xerrors.Errorf("cannot create workspace: %w", err)
9797
}

‎components/ws-daemon/pkg/controller/workspace_controller.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,7 @@ func (wsc *WorkspaceController) Reconcile(ctx context.Context, req ctrl.Request)
123123
return ctrl.Result{}, client.IgnoreNotFound(err)
124124
}
125125

126-
glog.WithField("workspace", req.NamespacedName).WithField("phase", workspace.Status.Phase).Info("reconciling workspace")
126+
glog.WithFields(workspace.OWI()).WithField("workspace", req.NamespacedName).WithField("phase", workspace.Status.Phase).Info("reconciling workspace")
127127

128128
if workspace.Status.Phase == workspacev1.WorkspacePhaseCreating ||
129129
workspace.Status.Phase == workspacev1.WorkspacePhaseInitializing {
@@ -155,7 +155,7 @@ func (wsc *WorkspaceController) latestWorkspace(ctx context.Context, ws *workspa
155155

156156
err := wsc.Client.Status().Update(ctx, ws)
157157
if err != nil && !errors.IsConflict(err) {
158-
glog.Warnf("could not refresh workspace: %v", err)
158+
glog.WithFields(ws.OWI()).Warnf("could not refresh workspace: %v", err)
159159
}
160160

161161
return err
@@ -409,7 +409,7 @@ func (m *workspaceMetrics) recordInitializeTime(duration float64, ws *workspacev
409409

410410
hist, err := m.initializeTimeHistVec.GetMetricWithLabelValues(tpe, class)
411411
if err != nil {
412-
glog.WithError(err).WithField("type", tpe).WithField("class", class).Infof("could not retrieve initialize metric")
412+
glog.WithError(err).WithFields(ws.OWI()).WithField("type", tpe).WithField("class", class).Infof("could not retrieve initialize metric")
413413
}
414414

415415
hist.Observe(duration)
@@ -421,7 +421,7 @@ func (m *workspaceMetrics) recordFinalizeTime(duration float64, ws *workspacev1.
421421

422422
hist, err := m.finalizeTimeHistVec.GetMetricWithLabelValues(tpe, class)
423423
if err != nil {
424-
glog.WithError(err).WithField("type", tpe).WithField("class", class).Infof("could not retrieve finalize metric")
424+
glog.WithError(err).WithFields(ws.OWI()).WithField("type", tpe).WithField("class", class).Infof("could not retrieve finalize metric")
425425
}
426426

427427
hist.Observe(duration)

‎components/ws-daemon/pkg/controller/workspace_operations.go

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -169,12 +169,12 @@ func (wso *DefaultWorkspaceOperations) InitWorkspace(ctx context.Context, option
169169

170170
err = ensureCleanSlate(ws.Location)
171171
if err != nil {
172-
glog.Warnf("cannot ensure clean slate for workspace %s (this might break content init): %v", ws.InstanceID, err)
172+
glog.WithFields(ws.OWI()).Warnf("cannot ensure clean slate for workspace %s (this might break content init): %v", ws.InstanceID, err)
173173
}
174174

175175
err = content.RunInitializer(ctx, ws.Location, options.Initializer, remoteContent, opts)
176176
if err != nil {
177-
glog.Infof("error running initializer %v", err)
177+
glog.WithFields(ws.OWI()).Infof("error running initializer %v", err)
178178
return err.Error(), err
179179
}
180180

@@ -234,7 +234,7 @@ func (wso *DefaultWorkspaceOperations) BackupWorkspace(ctx context.Context, opts
234234
err := wso.uploadWorkspaceLogs(ctx, opts)
235235
if err != nil {
236236
// we do not fail the workspace yet because we still might succeed with its content!
237-
glog.WithError(err).Error("log backup failed")
237+
glog.WithError(err).WithFields(ws.OWI()).Error("log backup failed")
238238
}
239239
}
240240

@@ -253,7 +253,7 @@ func (wso *DefaultWorkspaceOperations) BackupWorkspace(ctx context.Context, opts
253253
// which can happen for various reasons, including user corrupting his .git folder somehow
254254
// instead we log the error and continue cleaning up workspace
255255
// todo(pavel): it would be great if we can somehow bubble this up to user without failing workspace
256-
glog.WithError(err).Warn("cannot get git status")
256+
glog.WithError(err).WithFields(ws.OWI()).Warn("cannot get git status")
257257
}
258258
}
259259

@@ -267,13 +267,13 @@ func (wso *DefaultWorkspaceOperations) DeleteWorkspace(ctx context.Context, inst
267267
}
268268

269269
if err = ws.Dispose(ctx, wso.provider.hooks[session.WorkspaceDisposed]); err != nil {
270-
glog.WithError(err).Error("cannot dispose session")
270+
glog.WithError(err).WithFields(ws.OWI()).Error("cannot dispose session")
271271
return err
272272
}
273273

274274
// remove workspace daemon directory in the node
275275
if err := os.RemoveAll(ws.ServiceLocDaemon); err != nil {
276-
glog.WithError(err).Error("cannot delete workspace daemon directory")
276+
glog.WithError(err).WithFields(ws.OWI()).Error("cannot delete workspace daemon directory")
277277
return err
278278
}
279279
wso.provider.Remove(ctx, instanceID)
@@ -369,12 +369,13 @@ func (wso *DefaultWorkspaceOperations) uploadWorkspaceLogs(ctx context.Context,
369369

370370
for _, absLogPath := range logFiles {
371371
taskID, parseErr := logs.ParseTaskIDFromPrebuildLogFilePath(absLogPath)
372+
owi := glog.OWI(opts.Meta.Owner, opts.Meta.WorkspaceID, opts.Meta.InstanceID)
372373
if parseErr != nil {
373-
glog.WithError(parseErr).Warn("cannot parse headless workspace log file name")
374+
glog.WithError(parseErr).WithFields(owi).Warn("cannot parse headless workspace log file name")
374375
continue
375376
}
376377

377-
err = retryIfErr(ctx, 5, glog.WithField("op", "upload log"), func(ctx context.Context) (err error) {
378+
err = retryIfErr(ctx, 5, glog.WithField("op", "upload log").WithFields(owi), func(ctx context.Context) (err error) {
378379
_, _, err = rs.UploadInstance(ctx, absLogPath, logs.UploadedHeadlessLogPath(taskID))
379380
if err != nil {
380381
return

‎components/ws-daemon/pkg/internal/session/workspace.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,7 @@ func (s *Workspace) Dispose(ctx context.Context, hooks []WorkspaceLivecycleHook)
111111
err = os.Remove(s.persistentStateLocation())
112112
if err != nil {
113113
if errors.Is(err, fs.ErrNotExist) {
114-
log.WithError(err).Warn("workspace persistent state location not exist")
114+
log.WithError(err).WithFields(s.OWI()).Warn("workspace persistent state location not exist")
115115
err = nil
116116
} else {
117117
return xerrors.Errorf("cannot remove workspace persistent state location: %w", err)

0 commit comments

Comments
 (0)