Skip to content

Commit 68b03c0

Browse files
Peter-J-JansenPeter-J-Jansen
committed
CPU Timer Fix Stopping 2nd Level VM ABENDs
There are (were) problems when running VM 2nd level under VM, in that the 2nd level VM would suffer form ABENDs (EXT105, DSP005 and PRG009). They would occur intermittently, even on idling systems. When left that way overnight, the 2nd level VM system would automatically restart, but in the morning the CPDUMP files would show these problems. Additionally, it was observed that the 2nd level VM system would intermittently go into a CPU loop, exhibiting 100% CPU but on a single CPU only. The ABENDs were observed to occur at that time. The analysis of the ABENDs showed a probably incorrect CPU Timer emulation in Hyperion. This was originally believed to be related to Hyperion issue #217, but this was proven to not be the case. Rather, the problem has now been determined to be caused by the CPU Timer implementation being based on thread_cputime(). Such CPU Timer emulation relied on three variables which were lacking being treated in a critical section. Also, basing the CPU Timer emulation on thread_cputime() would imply that a running mainframe CPU Timer would be stopped when the Hyperion host would pre-empt the host CPU thread emulating the mainframe CPU, which is incorrect. Additionally, the Windows emulation of thread_cputime() suffers from a poor resolution (1/16th of a second). This commit reverts the CPU Timer emulation to the earlier one, to not rely on thread_cputime(), but on hw_clock() and hw_tod. The processing around DIAG 204 and CPU MAXRATES has not been altered and still uses thread_cputime_us().
1 parent 5134922 commit 68b03c0

File tree

11 files changed

+34
-204
lines changed

11 files changed

+34
-204
lines changed

‎clock.c

Lines changed: 10 additions & 158 deletions
Original file line numberDiff line numberDiff line change
@@ -358,9 +358,8 @@ static void adjust_tod_offset(const S64 offset)
358358
}
359359

360360

361-
/* The CPU timer is internally kept as an offset to the thread CPU time
362-
* used in LPAR mode, or TOD clock in BASIC mode. The CPU timer counts
363-
* down as the clock approaches the timer epoch.
361+
/* The CPU timer is internally kept as an offset to the hw_clock()
362+
* the cpu timer counts down as the clock approaches the timer epoch
364363
*
365364
* To be in agreement with reporting of time in association with real
366365
* diagnose code x'204' for partition management and resource reporting,
@@ -369,21 +368,6 @@ static void adjust_tod_offset(const S64 offset)
369368
* management time).
370369
*/
371370

372-
TOD
373-
thread_cputime(const REGS *regs)
374-
{
375-
TOD result;
376-
int rc = -1;
377-
struct timespec cputime;
378-
379-
if (sysblk.cpuclockid[regs->cpuad])
380-
{
381-
rc = clock_gettime(sysblk.cpuclockid[regs->cpuad], &cputime);
382-
}
383-
result = (likely(rc == 0)) ? timespec2etod(&cputime) : host_tod();
384-
385-
return (result);
386-
}
387371

388372
U64
389373
thread_cputime_us(const REGS *regs)
@@ -402,158 +386,26 @@ thread_cputime_us(const REGS *regs)
402386
}
403387

404388

405-
static INLINE void
406-
set_cpu_timer_epoch(REGS *regs, const TOD epoch)
389+
void set_cpu_timer(REGS *regs, const S64 timer)
407390
{
408-
register REGS* guestregs = regs->guestregs;
409-
register REGS* hostregs = regs->hostregs;
410-
411-
/* Set CPU timer epoch */
412-
regs->cpu_timer_epoch = epoch;
413-
414-
/* Reset epoch value for host and guest */
415-
if (hostregs && hostregs != regs)
416-
hostregs->cpu_timer_epoch = epoch;
417-
if (guestregs && guestregs != regs)
418-
guestregs->cpu_timer_epoch = epoch;
419-
}
391+
U64 new_epoch_us = (sysblk.lparmode && !WAITSTATE(&regs->psw)) ? thread_cputime_us(regs) : etod2us(host_tod());
420392

421-
422-
static INLINE S64
423-
cpu_timer_update(REGS *regs, TOD new_epoch)
424-
{
425-
register S64 interval = (S64)(new_epoch - regs->cpu_timer_epoch);
426-
register S64 result = regs->cpu_timer;
427-
428-
if (interval > 0)
429-
{
430-
result -= interval;
431-
regs->cpu_timer_epoch = new_epoch;
432-
regs->cpu_timer = result;
433-
}
434-
435-
return (result);
436-
}
437-
438-
439-
static INLINE void
440-
cpu_timer_update_linked(REGS *regs, REGS *linked_regs, TOD new_epoch)
441-
{
442-
if (linked_regs && linked_regs != regs)
443-
cpu_timer_update(linked_regs, new_epoch);
444-
}
445-
446-
447-
static INLINE TOD
448-
mode_cputime(REGS *regs)
449-
{
450-
return (regs->cpu_timer_mode ? thread_cputime(regs) : host_tod());
451-
}
452-
453-
454-
static INLINE int
455-
cpu_timer_mode(REGS *regs)
456-
{
457-
return (sysblk.lparmode && !WAITSTATE(&regs->psw));
458-
}
459-
460-
461-
void
462-
set_cpu_timer_mode(REGS *regs)
463-
{
464-
int newmode = cpu_timer_mode(regs);
465-
466-
/* Update CPU timer epoch if changing mode */
467-
if ((U32)newmode != regs->cpu_timer_mode)
468-
{
469-
cpu_timer(regs);
470-
regs->cpu_timer_mode = newmode;
471-
set_cpu_timer_epoch(regs, mode_cputime(regs));
472-
}
473-
}
474-
475-
476-
S64 set_cpu_timer(REGS *regs, const TOD timer)
477-
{
478-
regs->cpu_timer_mode = cpu_timer_mode(regs);
479-
480-
/* Prepare new timer value and epoch value */
481-
regs->cpu_timer = tod2etod(timer);
482-
set_cpu_timer_epoch(regs, mode_cputime(regs));
483-
484-
return (timer);
485-
}
486-
487-
488-
void set_cpu_timers(REGS *hostregs, const TOD host_timer, REGS *regs, const TOD timer)
489-
{
490-
set_cpu_timer(hostregs, host_timer);
491-
if (regs != hostregs)
492-
{
493-
regs->cpu_timer = tod2etod(timer);
494-
regs->cpu_timer_epoch = hostregs->cpu_timer_epoch;
495-
}
496-
}
497-
498-
void save_cpu_timers(REGS *hostregs, TOD *host_timer, REGS *regs, TOD *timer)
499-
{
500-
*host_timer = cpu_timer(hostregs);
501-
*timer = (regs == hostregs) ?
502-
*host_timer : (TOD)cpu_timer_SIE(regs);
503-
}
504-
505-
506-
S64 cpu_timer(REGS *regs)
507-
{
508-
register TOD new_epoch = mode_cputime(regs);
509-
register U64 new_epoch_us = etod2us(new_epoch);
510-
register S64 result;
511-
512-
/* If no change from epoch, don't bother updating */
513-
if (new_epoch <= regs->cpu_timer_epoch)
514-
{
515-
result = regs->cpu_timer;
516-
}
517-
/* If CPU is stopped, return the CPU timer without updating */
518-
else if (regs->cpustate == CPUSTATE_STOPPED)
519-
{
520-
result = regs->cpu_timer;
521-
522-
/* Update base CPU time epoch */
523-
regs->bcputime = new_epoch_us;
524-
525-
/* Update CPU timer epoch */
526-
set_cpu_timer_epoch(regs, new_epoch);
527-
}
528-
else /* Update and return the CPU timer */
393+
if (regs->bcputime <= new_epoch_us)
529394
{
530395
/* Update real CPU time used and base CPU time epoch */
531396
regs->rcputime += new_epoch_us - regs->bcputime;
532397
regs->bcputime = new_epoch_us;
533-
534-
/* Process CPU timers */
535-
result = cpu_timer_update(regs, new_epoch);
536-
cpu_timer_update_linked(regs, regs->hostregs, new_epoch);
537-
cpu_timer_update_linked(regs, regs->guestregs, new_epoch);
538398
}
539399

540-
/* Change from ETOD format to TOD format */
541-
result = (S64)etod2tod(result);
542-
543-
return (result);
400+
regs->cpu_timer = tod2etod(timer) + hw_clock();
544401
}
545402

546403

547-
S64 cpu_timer_SIE(REGS *regs)
404+
S64 cpu_timer(REGS *regs)
548405
{
549-
register S64 result;
550-
551-
/* Ensure SIE guestregs are in use */
552-
if (regs == regs->hostregs)
553-
regs = regs->guestregs;
554-
555-
result = etod2tod(regs->cpu_timer);
556-
return (result);
406+
S64 timer;
407+
timer = (S64)etod2tod(regs->cpu_timer - hw_clock());
408+
return timer;
557409
}
558410

559411

‎clock.h

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -151,11 +151,7 @@ void adjust_tod_epoch(const S64); /* Adjust TOD epoch */
151151
S64 get_tod_epoch(void); /* Get TOD epoch */
152152
U64 hw_clock(void); /* Get hardware clock */
153153
S64 cpu_timer(REGS *); /* Retrieve CPU timer */
154-
S64 cpu_timer_SIE(REGS *); /* Retrieve SIE CPU timer */
155-
void set_cpu_timer_mode(REGS *); /* Set CPU timer source mode */
156-
S64 set_cpu_timer(REGS *, const TOD); /* Set CPU timer */
157-
void save_cpu_timers(REGS *, TOD *, REGS *, TOD *);
158-
void set_cpu_timers(REGS *, const TOD, REGS *, const TOD);
154+
void set_cpu_timer(REGS *, const S64); /* Set CPU timer */
159155
void set_int_timer(REGS *, const S32); /* Set interval timer */
160156
TOD tod_clock(REGS *); /* Get TOD clock non-unique */
161157
typedef enum

‎cpu.c

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1634,7 +1634,7 @@ void (ATTR_REGPARM(1) ARCH_DEP(process_interrupt))(REGS *regs)
16341634
/* This is where a stopped CPU will wait */
16351635
if (unlikely(regs->cpustate == CPUSTATE_STOPPED))
16361636
{
1637-
TOD saved_timer = cpu_timer(regs);
1637+
S64 saved_timer = cpu_timer(regs);
16381638
regs->ints_state = IC_INITIAL_STATE;
16391639
sysblk.started_mask ^= regs->cpubit;
16401640

@@ -1664,7 +1664,6 @@ void (ATTR_REGPARM(1) ARCH_DEP(process_interrupt))(REGS *regs)
16641664
if (WAITSTATE(&regs->psw))
16651665
{
16661666
regs->waittod = host_tod();
1667-
set_cpu_timer_mode(regs);
16681667

16691668
/* Test for disabled wait PSW and issue message */
16701669
if( IS_IC_DISABLED_WAIT_PSW(regs) )
@@ -1696,8 +1695,6 @@ void (ATTR_REGPARM(1) ARCH_DEP(process_interrupt))(REGS *regs)
16961695
regs->waittime += host_tod() - regs->waittod;
16971696
regs->waittod = 0;
16981697

1699-
set_cpu_timer_mode(regs);
1700-
17011698
/* If late state change to stopping, go reprocess */
17021699
if (unlikely(regs->cpustate == CPUSTATE_STOPPING))
17031700
goto cpustate_stopping;
@@ -1870,16 +1867,16 @@ int shouldstep = 0; /* 1=Wait for start command */
18701867
if (shouldstep)
18711868
{
18721869
REGS *hostregs = regs->hostregs;
1873-
TOD saved_timer[2];
1870+
S64 saved_timer[2];
18741871

18751872
OBTAIN_INTLOCK(hostregs);
18761873
#ifdef OPTION_MIPS_COUNTING
18771874
hostregs->waittod = host_tod();
18781875
#endif
18791876
/* The CPU timer is not decremented for a CPU that is in
18801877
the manual state (e.g. stopped in single step mode) */
1881-
save_cpu_timers(hostregs, &saved_timer[0],
1882-
regs, &saved_timer[1]);
1878+
saved_timer[0] = cpu_timer(regs);
1879+
saved_timer[1] = cpu_timer(hostregs);
18831880
hostregs->cpustate = CPUSTATE_STOPPED;
18841881
sysblk.started_mask &= ~hostregs->cpubit;
18851882
hostregs->stepwait = 1;
@@ -1891,8 +1888,8 @@ int shouldstep = 0; /* 1=Wait for start command */
18911888
sysblk.intowner = hostregs->cpuad;
18921889
hostregs->stepwait = 0;
18931890
sysblk.started_mask |= hostregs->cpubit;
1894-
set_cpu_timers(hostregs, saved_timer[0],
1895-
regs, saved_timer[1]);
1891+
set_cpu_timer(regs,saved_timer[0]);
1892+
set_cpu_timer(hostregs,saved_timer[1]);
18961893
#ifdef OPTION_MIPS_COUNTING
18971894
hostregs->waittime += host_tod() - hostregs->waittod;
18981895
hostregs->waittod = 0;

‎ecpsvm.c

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -451,8 +451,9 @@ VADR effective_addr1, effective_addr2; \
451451

452452
#define SPT(_x) \
453453
{ \
454+
set_cpu_timer(regs,EVM_LD(_x)); \
454455
OBTAIN_INTLOCK(regs); \
455-
if(set_cpu_timer(regs,EVM_LD(_x)) < 0) \
456+
if(CPU_TIMER(regs) < 0) \
456457
{ \
457458
ON_IC_PTIMER(regs); \
458459
} \

‎esame.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2177,7 +2177,7 @@ U64 gr0, gr1; /* Result register workareas */
21772177
dreg = cpu_timer(regs);
21782178

21792179
/* Reset the cpu timer pending flag according to its value */
2180-
if( dreg < 0 )
2180+
if( CPU_TIMER(regs) < 0 )
21812181
{
21822182
ON_IC_PTIMER(regs);
21832183

‎external.c

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -159,7 +159,6 @@ void ARCH_DEP(perform_external_interrupt) (REGS *regs)
159159
{
160160
PSA *psa; /* -> Prefixed storage area */
161161
U16 cpuad; /* Originating CPU address */
162-
S64 dreg; /* Double word workarea */
163162
#if defined(FEATURE_VM_BLOCKIO)
164163
#if defined(FEATURE_ESAME)
165164
RADR servpadr; /* Address of 64-bit block I/O interrupt */
@@ -287,13 +286,12 @@ U16 servcode; /* Service Signal or Block I/O Interrupt code */
287286
}
288287

289288
/* External interrupt if CPU timer is negative */
290-
dreg = cpu_timer(regs);
291-
if ( dreg < 0
289+
if ( CPU_TIMER(regs) < 0
292290
&& OPEN_IC_PTIMER(regs) )
293291
{
294292
if (CPU_STEPPING_OR_TRACING_ALL)
295293
{
296-
WRMSG (HHC00842, "I", dreg );
294+
WRMSG (HHC00842, "I", CPU_TIMER(regs) );
297295
}
298296
ARCH_DEP(external_interrupt) (EXT_CPU_TIMER_INTERRUPT, regs);
299297
}

‎hinlines.h

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -384,11 +384,6 @@ setCpuId(const unsigned int cpu,
384384
/* Set new CPU ID */
385385
setCpuIdregs(regs, cpu, arg_model, arg_version, arg_serial, arg_MCEL);
386386

387-
/* Set CPU timer source (a "strange" place, but here as the CPU ID
388-
* must be updated when the LPAR mode or number is update).
389-
*/
390-
set_cpu_timer_mode(regs);
391-
392387
}
393388

394389

‎hscpufun.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -793,14 +793,14 @@ int rc = 0;
793793
hw_now.low = hw_tod.low;
794794
epoch_now = regs->tod_epoch;
795795
clkc_now = regs->clkc;
796-
cpt_now = cpu_timer(regs);
796+
cpt_now = CPU_TIMER(regs);
797797
#if defined(_FEATURE_SIE)
798798
if ( regs->sie_active )
799799
{
800800
vtod_now = TOD_CLOCK(regs->guestregs);
801801
vepoch_now = regs->guestregs->tod_epoch;
802802
vclkc_now = regs->guestregs->clkc;
803-
vcpt_now = cpu_timer_SIE(regs->guestregs);
803+
vcpt_now = CPU_TIMER(regs->guestregs);
804804
sie_flag = 1;
805805
}
806806
#endif

‎hstructs.h

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -169,9 +169,7 @@ struct REGS { /* Processor registers */
169169
S64 tod_epoch; /* TOD epoch for this CPU */
170170
TOD clkc; /* 0-7=Clock comparator epoch,
171171
8-63=Comparator bits 0-55 */
172-
TOD cpu_timer; /* CPU timer */
173-
TOD cpu_timer_epoch; /* CPU timer epoch */
174-
U32 cpu_timer_mode; /* CPU timer source mode */
172+
S64 cpu_timer; /* CPU timer */
175173
U32 todpr; /* TOD programmable register */
176174

177175
S64 int_timer; /* S/370 Interval timer */

‎sie.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -678,7 +678,7 @@ U64 dreg;
678678
OBTAIN_INTLOCK(regs);
679679

680680
/* CPU timer */
681-
if(cpu_timer(GUESTREGS) < 0)
681+
if(CPU_TIMER(GUESTREGS) < 0)
682682
ON_IC_PTIMER(GUESTREGS);
683683

684684
/* Clock comparator */

0 commit comments

Comments
 (0)