OpenSolaris_b135/uts/i86pc/os/memscrub.c
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or http://www.opensolaris.org/os/licensing.
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2010 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
/*
* i86pc Memory Scrubbing
*
* On detection of a correctable memory ECC error, the i86pc hardware
* returns the corrected data to the requester and may re-write it
* to memory (DRAM or NVRAM). Machines which do not re-write this to
* memory should add an NMI handler to correct and rewrite.
*
* Scrubbing thus reduces the likelyhood that multiple transient errors
* will occur in the same memory word, making uncorrectable errors due
* to transients less likely.
*
* Thus is born the desire that every memory location be periodically
* accessed.
*
* This file implements a memory scrubbing thread. This scrubber
* guarantees that all of physical memory is accessed periodically
* (memscrub_period_sec -- 12 hours).
*
* It attempts to do this as unobtrusively as possible. The thread
* schedules itself to wake up at an interval such that if it reads
* memscrub_span_pages (4MB) on each wakeup, it will read all of physical
* memory in in memscrub_period_sec (12 hours).
*
* The scrubber uses the REP LODS so it reads 4MB in 0.15 secs (on P5-200).
* When it completes a span, if all the CPUs are idle, it reads another span.
* Typically it soaks up idle time this way to reach its deadline early
* -- and sleeps until the next period begins.
*
* Maximal Cost Estimate: 8GB @ xxMB/s = xxx seconds spent in 640 wakeups
* that run for 0.15 seconds at intervals of 67 seconds.
*
* In practice, the scrubber finds enough idle time to finish in a few
* minutes, and sleeps until its 12 hour deadline.
*
* The scrubber maintains a private copy of the phys_install memory list
* to keep track of what memory should be scrubbed.
*
* The following parameters can be set via /etc/system
*
* memscrub_span_pages = MEMSCRUB_DFL_SPAN_PAGES (4MB)
* memscrub_period_sec = MEMSCRUB_DFL_PERIOD_SEC (12 hours)
* memscrub_thread_pri = MEMSCRUB_DFL_THREAD_PRI (0)
* memscrub_delay_start_sec = (10 seconds)
* disable_memscrub = (0)
*
* the scrubber will exit (or never be started) if it finds the variable
* "disable_memscrub" set.
*
* MEMSCRUB_DFL_SPAN_PAGES is based on the guess that 0.15 sec
* is a "good" amount of minimum time for the thread to run at a time.
*
* MEMSCRUB_DFL_PERIOD_SEC (12 hours) is nearly a total guess --
* twice the frequency the hardware folk estimated would be necessary.
*
* MEMSCRUB_DFL_THREAD_PRI (0) is based on the assumption that nearly
* any other use of the system should be higher priority than scrubbing.
*/
#include <sys/types.h>
#include <sys/systm.h> /* timeout, types, t_lock */
#include <sys/cmn_err.h>
#include <sys/sysmacros.h> /* MIN */
#include <sys/memlist.h> /* memlist */
#include <sys/kmem.h> /* KMEM_NOSLEEP */
#include <sys/cpuvar.h> /* ncpus_online */
#include <sys/debug.h> /* ASSERTs */
#include <sys/vmem.h>
#include <sys/mman.h>
#include <vm/seg_kmem.h>
#include <vm/seg_kpm.h>
#include <vm/hat_i86.h>
#include <sys/callb.h> /* CPR callback */
static caddr_t memscrub_window;
static hat_mempte_t memscrub_pte;
/*
* Global Data:
*/
/*
* scan all of physical memory at least once every MEMSCRUB_PERIOD_SEC
*/
#define MEMSCRUB_DFL_PERIOD_SEC (12 * 60 * 60) /* 12 hours */
/*
* start only if at least MEMSCRUB_MIN_PAGES in system
*/
#define MEMSCRUB_MIN_PAGES ((32 * 1024 * 1024) / PAGESIZE)
/*
* scan at least MEMSCRUB_DFL_SPAN_PAGES each iteration
*/
#define MEMSCRUB_DFL_SPAN_PAGES ((4 * 1024 * 1024) / PAGESIZE)
/*
* almost anything is higher priority than scrubbing
*/
#define MEMSCRUB_DFL_THREAD_PRI 0
/*
* we can patch these defaults in /etc/system if necessary
*/
uint_t disable_memscrub = 0;
static uint_t disable_memscrub_quietly = 0;
pgcnt_t memscrub_min_pages = MEMSCRUB_MIN_PAGES;
pgcnt_t memscrub_span_pages = MEMSCRUB_DFL_SPAN_PAGES;
time_t memscrub_period_sec = MEMSCRUB_DFL_PERIOD_SEC;
uint_t memscrub_thread_pri = MEMSCRUB_DFL_THREAD_PRI;
time_t memscrub_delay_start_sec = 10;
/*
* Static Routines
*/
static void memscrubber(void);
static int system_is_idle(void);
static int memscrub_add_span(uint64_t, uint64_t);
/*
* Static Data
*/
static struct memlist *memscrub_memlist;
static uint_t memscrub_phys_pages;
static kcondvar_t memscrub_cv;
static kmutex_t memscrub_lock;
/*
* memscrub_lock protects memscrub_memlist
*/
uint_t memscrub_scans_done;
uint_t memscrub_done_early;
uint_t memscrub_early_sec;
uint_t memscrub_done_late;
time_t memscrub_late_sec;
/*
* create memscrub_memlist from phys_install list
* initialize locks, set memscrub_phys_pages.
*/
void
memscrub_init()
{
struct memlist *src;
if (physmem < memscrub_min_pages)
return;
if (!kpm_enable) {
memscrub_window = vmem_alloc(heap_arena, PAGESIZE, VM_SLEEP);
memscrub_pte = hat_mempte_setup(memscrub_window);
}
/*
* copy phys_install to memscrub_memlist
*/
for (src = phys_install; src; src = src->ml_next) {
if (memscrub_add_span(src->ml_address, src->ml_size)) {
cmn_err(CE_WARN,
"Software memory scrubber failed to initialize\n");
return;
}
}
mutex_init(&memscrub_lock, NULL, MUTEX_DRIVER, NULL);
cv_init(&memscrub_cv, NULL, CV_DRIVER, NULL);
/*
* create memscrubber thread
*/
(void) thread_create(NULL, 0, (void (*)())memscrubber, NULL, 0, &p0,
TS_RUN, memscrub_thread_pri);
}
/*
* Function to cause the software memscrubber to exit quietly if the
* platform support has located a hardware scrubber and enabled it.
*/
void
memscrub_disable(void)
{
disable_memscrub_quietly = 1;
}
#ifdef MEMSCRUB_DEBUG
static void
memscrub_printmemlist(char *title, struct memlist *listp)
{
struct memlist *list;
cmn_err(CE_CONT, "%s:\n", title);
for (list = listp; list; list = list->next) {
cmn_err(CE_CONT, "addr = 0x%llx, size = 0x%llx\n",
list->address, list->size);
}
}
#endif /* MEMSCRUB_DEBUG */
/* ARGSUSED */
static void
memscrub_wakeup(void *c)
{
/*
* grab mutex to guarantee that our wakeup call
* arrives after we go to sleep -- so we can't sleep forever.
*/
mutex_enter(&memscrub_lock);
cv_signal(&memscrub_cv);
mutex_exit(&memscrub_lock);
}
/*
* this calculation doesn't account for the time that the actual scan
* consumes -- so we'd fall slightly behind schedule with this
* interval_sec. but the idle loop optimization below usually makes us
* come in way ahead of schedule.
*/
static int
compute_interval_sec()
{
if (memscrub_phys_pages <= memscrub_span_pages)
return (memscrub_period_sec);
else
return (memscrub_period_sec/
(memscrub_phys_pages/memscrub_span_pages));
}
static void
memscrubber()
{
time_t deadline;
uint64_t mlp_last_addr;
uint64_t mlp_next_addr;
int reached_end = 1;
time_t interval_sec = 0;
struct memlist *mlp;
extern void scan_memory(caddr_t, size_t);
callb_cpr_t cprinfo;
/*
* notify CPR of our existence
*/
CALLB_CPR_INIT(&cprinfo, &memscrub_lock, callb_generic_cpr, "memscrub");
if (memscrub_memlist == NULL) {
cmn_err(CE_WARN, "memscrub_memlist not initialized.");
goto memscrub_exit;
}
mlp = memscrub_memlist;
mlp_next_addr = mlp->ml_address;
mlp_last_addr = mlp->ml_address + mlp->ml_size;
deadline = gethrestime_sec() + memscrub_delay_start_sec;
for (;;) {
if (disable_memscrub || disable_memscrub_quietly)
break;
mutex_enter(&memscrub_lock);
/*
* did we just reach the end of memory?
*/
if (reached_end) {
time_t now = gethrestime_sec();
if (now >= deadline) {
memscrub_done_late++;
memscrub_late_sec += (now - deadline);
/*
* past deadline, start right away
*/
interval_sec = 0;
deadline = now + memscrub_period_sec;
} else {
/*
* we finished ahead of schedule.
* wait till previous dealine before re-start.
*/
interval_sec = deadline - now;
memscrub_done_early++;
memscrub_early_sec += interval_sec;
deadline += memscrub_period_sec;
}
} else {
interval_sec = compute_interval_sec();
}
/*
* it is safe from our standpoint for CPR to
* suspend the system
*/
CALLB_CPR_SAFE_BEGIN(&cprinfo);
/*
* hit the snooze bar
*/
(void) timeout(memscrub_wakeup, NULL, interval_sec * hz);
/*
* go to sleep
*/
cv_wait(&memscrub_cv, &memscrub_lock);
/* we need to goto work */
CALLB_CPR_SAFE_END(&cprinfo, &memscrub_lock);
mutex_exit(&memscrub_lock);
do {
pgcnt_t pages = memscrub_span_pages;
uint64_t address = mlp_next_addr;
if (disable_memscrub || disable_memscrub_quietly)
break;
mutex_enter(&memscrub_lock);
/*
* Make sure we don't try to scan beyond the end of
* the current memlist. If we would, then resize
* our scan target for this iteration, and prepare
* to read the next memlist entry on the next
* iteration.
*/
reached_end = 0;
if (address + mmu_ptob(pages) >= mlp_last_addr) {
pages = mmu_btop(mlp_last_addr - address);
mlp = mlp->ml_next;
if (mlp == NULL) {
reached_end = 1;
mlp = memscrub_memlist;
}
mlp_next_addr = mlp->ml_address;
mlp_last_addr = mlp->ml_address + mlp->ml_size;
} else {
mlp_next_addr += mmu_ptob(pages);
}
mutex_exit(&memscrub_lock);
while (pages--) {
pfn_t pfn = btop(address);
/*
* Without segkpm, the memscrubber cannot
* be allowed to migrate across CPUs, as
* the CPU-specific mapping of
* memscrub_window would be incorrect.
* With segkpm, switching CPUs is legal, but
* inefficient. We don't use
* kpreempt_disable as it might hold a
* higher priority thread (eg, RT) too long
* off CPU.
*/
thread_affinity_set(curthread, CPU_CURRENT);
if (kpm_enable)
memscrub_window = hat_kpm_pfn2va(pfn);
else
hat_mempte_remap(pfn, memscrub_window,
memscrub_pte,
PROT_READ, HAT_LOAD_NOCONSIST);
scan_memory(memscrub_window, PAGESIZE);
thread_affinity_clear(curthread);
address += MMU_PAGESIZE;
}
memscrub_scans_done++;
} while (!reached_end && system_is_idle());
}
memscrub_exit:
if (!disable_memscrub_quietly)
cmn_err(CE_NOTE, "Software memory scrubber exiting.");
/*
* We are about to bail, but don't have the memscrub_lock,
* and it is needed for CALLB_CPR_EXIT.
*/
mutex_enter(&memscrub_lock);
CALLB_CPR_EXIT(&cprinfo);
cv_destroy(&memscrub_cv);
thread_exit();
}
/*
* return 1 if we're MP and all the other CPUs are idle
*/
static int
system_is_idle()
{
int cpu_id;
int found = 0;
if (1 == ncpus_online)
return (0);
for (cpu_id = 0; cpu_id < NCPU; ++cpu_id) {
if (!cpu[cpu_id])
continue;
found++;
if (cpu[cpu_id]->cpu_thread != cpu[cpu_id]->cpu_idle_thread) {
if (CPU->cpu_id == cpu_id &&
CPU->cpu_disp->disp_nrunnable == 0)
continue;
return (0);
}
if (found == ncpus)
break;
}
return (1);
}
/*
* add a span to the memscrub list
*/
static int
memscrub_add_span(uint64_t start, uint64_t bytes)
{
struct memlist *dst;
struct memlist *prev, *next;
uint64_t end = start + bytes - 1;
int retval = 0;
mutex_enter(&memscrub_lock);
#ifdef MEMSCRUB_DEBUG
memscrub_printmemlist("memscrub_memlist before", memscrub_memlist);
cmn_err(CE_CONT, "memscrub_phys_pages: 0x%x\n", memscrub_phys_pages);
cmn_err(CE_CONT, "memscrub_add_span: address: 0x%llx"
" size: 0x%llx\n", start, bytes);
#endif /* MEMSCRUB_DEBUG */
/*
* Scan through the list to find the proper place to install it.
*/
prev = NULL;
next = memscrub_memlist;
while (next) {
uint64_t ns = next->ml_address;
uint64_t ne = next->ml_address + next->ml_size - 1;
/*
* If this span overlaps with an existing span, then
* something has gone horribly wrong with the phys_install
* list. In fact, I'm surprised we made it this far.
*/
if ((start >= ns && start <= ne) || (end >= ns && end <= ne) ||
(start < ns && end > ne))
panic("memscrub found overlapping memory ranges "
"(0x%p-0x%p) and (0x%p-0x%p)",
(void *)(uintptr_t)start, (void *)(uintptr_t)end,
(void *)(uintptr_t)ns, (void *)(uintptr_t)ne);
/*
* New span can be appended to an existing one.
*/
if (start == ne + 1) {
next->ml_size += bytes;
goto add_done;
}
/*
* New span can be prepended to an existing one.
*/
if (end + 1 == ns) {
next->ml_size += bytes;
next->ml_address = start;
goto add_done;
}
/*
* If the next span has a higher start address than the new
* one, then we have found the right spot for our
* insertion.
*/
if (ns > start)
break;
prev = next;
next = next->ml_next;
}
/*
* allocate a new struct memlist
*/
dst = kmem_alloc(sizeof (struct memlist), KM_NOSLEEP);
if (dst == NULL) {
retval = -1;
goto add_done;
}
dst->ml_address = start;
dst->ml_size = bytes;
dst->ml_prev = prev;
dst->ml_next = next;
if (prev)
prev->ml_next = dst;
else
memscrub_memlist = dst;
if (next)
next->ml_prev = dst;
add_done:
if (retval != -1)
memscrub_phys_pages += mmu_btop(bytes);
#ifdef MEMSCRUB_DEBUG
memscrub_printmemlist("memscrub_memlist after", memscrub_memlist);
cmn_err(CE_CONT, "memscrub_phys_pages: 0x%x\n", memscrub_phys_pages);
#endif /* MEMSCRUB_DEBUG */
mutex_exit(&memscrub_lock);
return (retval);
}