OpenSolaris_b135/uts/sun4/vm/sfmmu.c
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or http://www.opensolaris.org/os/licensing.
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2010 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
#include <sys/types.h>
#include <vm/hat.h>
#include <vm/hat_sfmmu.h>
#include <vm/page.h>
#include <sys/pte.h>
#include <sys/systm.h>
#include <sys/mman.h>
#include <sys/sysmacros.h>
#include <sys/machparam.h>
#include <sys/vtrace.h>
#include <sys/kmem.h>
#include <sys/mmu.h>
#include <sys/cmn_err.h>
#include <sys/cpu.h>
#include <sys/cpuvar.h>
#include <sys/debug.h>
#include <sys/lgrp.h>
#include <sys/archsystm.h>
#include <sys/machsystm.h>
#include <sys/vmsystm.h>
#include <sys/bitmap.h>
#include <vm/as.h>
#include <vm/seg.h>
#include <vm/seg_kmem.h>
#include <vm/seg_kp.h>
#include <vm/seg_kpm.h>
#include <vm/rm.h>
#include <vm/vm_dep.h>
#include <sys/t_lock.h>
#include <sys/vm_machparam.h>
#include <sys/promif.h>
#include <sys/prom_isa.h>
#include <sys/prom_plat.h>
#include <sys/prom_debug.h>
#include <sys/privregs.h>
#include <sys/bootconf.h>
#include <sys/memlist.h>
#include <sys/memlist_plat.h>
#include <sys/cpu_module.h>
#include <sys/reboot.h>
#include <sys/kdi.h>
/*
* Static routines
*/
static void sfmmu_map_prom_mappings(struct translation *, size_t);
static struct translation *read_prom_mappings(size_t *);
static void sfmmu_reloc_trap_handler(void *, void *, size_t);
/*
* External routines
*/
extern void sfmmu_remap_kernel(void);
extern void sfmmu_patch_utsb(void);
/*
* Global Data:
*/
extern caddr_t textva, datava;
extern tte_t ktext_tte, kdata_tte; /* ttes for kernel text and data */
extern int enable_bigktsb;
extern int kmem64_smchunks;
uint64_t memsegspa = (uintptr_t)MSEG_NULLPTR_PA; /* memsegs physical linkage */
uint64_t memseg_phash[N_MEM_SLOTS]; /* use physical memseg addresses */
int sfmmu_kern_mapped = 0;
/*
* DMMU primary context register for the kernel context. Machine specific code
* inserts correct page size codes when necessary
*/
uint64_t kcontextreg = KCONTEXT;
#ifdef DEBUG
static int ndata_middle_hole_detected = 0;
#endif
/* Extern Global Data */
extern int page_relocate_ready;
/*
* Controls the logic which enables the use of the
* QUAD_LDD_PHYS ASI for TSB accesses.
*/
extern int ktsb_phys;
/*
* Global Routines called from within:
* usr/src/uts/sun4u
* usr/src/uts/sfmmu
* usr/src/uts/sun
*/
pfn_t
va_to_pfn(void *vaddr)
{
u_longlong_t physaddr;
int mode, valid;
if (tba_taken_over)
return (hat_getpfnum(kas.a_hat, (caddr_t)vaddr));
#if !defined(C_OBP)
if (!kmem64_smchunks &&
(caddr_t)vaddr >= kmem64_base && (caddr_t)vaddr < kmem64_end) {
if (kmem64_pabase == (uint64_t)-1)
prom_panic("va_to_pfn: kmem64_pabase not init");
physaddr = kmem64_pabase + ((caddr_t)vaddr - kmem64_base);
return ((pfn_t)physaddr >> MMU_PAGESHIFT);
}
#endif /* !C_OBP */
if ((prom_translate_virt(vaddr, &valid, &physaddr, &mode) != -1) &&
(valid == -1)) {
return ((pfn_t)(physaddr >> MMU_PAGESHIFT));
}
return (PFN_INVALID);
}
uint64_t
va_to_pa(void *vaddr)
{
pfn_t pfn;
if ((pfn = va_to_pfn(vaddr)) == PFN_INVALID)
return ((uint64_t)-1);
return (((uint64_t)pfn << MMU_PAGESHIFT) |
((uint64_t)vaddr & MMU_PAGEOFFSET));
}
void
hat_kern_setup(void)
{
struct translation *trans_root;
size_t ntrans_root;
extern void startup_fixup_physavail(void);
/*
* These are the steps we take to take over the mmu from the prom.
*
* (1) Read the prom's mappings through the translation property.
* (2) Remap the kernel text and kernel data with 2 locked 4MB ttes.
* Create the the hmeblks for these 2 ttes at this time.
* (3) Create hat structures for all other prom mappings. Since the
* kernel text and data hme_blks have already been created we
* skip the equivalent prom's mappings.
* (4) Initialize the tsb and its corresponding hardware regs.
* (5) Take over the trap table (currently in startup).
* (6) Up to this point it is possible the prom required some of its
* locked tte's. Now that we own the trap table we remove them.
*/
ktsb_pbase = va_to_pa(ktsb_base);
ktsb4m_pbase = va_to_pa(ktsb4m_base);
PRM_DEBUG(ktsb_pbase);
PRM_DEBUG(ktsb4m_pbase);
sfmmu_patch_ktsb();
sfmmu_patch_utsb();
sfmmu_patch_mmu_asi(ktsb_phys);
sfmmu_init_tsbs();
if (kpm_enable) {
sfmmu_kpm_patch_tlbm();
if (kpm_smallpages == 0) {
sfmmu_kpm_patch_tsbm();
}
}
if (!shctx_on) {
sfmmu_patch_shctx();
}
/*
* The 8K-indexed kernel TSB space is used to hold
* translations below...
*/
trans_root = read_prom_mappings(&ntrans_root);
sfmmu_remap_kernel();
startup_fixup_physavail();
mmu_init_kernel_pgsz(kas.a_hat);
sfmmu_map_prom_mappings(trans_root, ntrans_root);
/*
* We invalidate 8K kernel TSB because we used it in
* sfmmu_map_prom_mappings()
*/
sfmmu_inv_tsb(ktsb_base, ktsb_sz);
sfmmu_inv_tsb(ktsb4m_base, ktsb4m_sz);
sfmmu_init_ktsbinfo();
sfmmu_kern_mapped = 1;
/*
* hments have been created for mapped pages, and thus we're ready
* for kmdb to start using its own trap table. It walks the hments
* to resolve TLB misses, and can't be used until they're ready.
*/
if (boothowto & RB_DEBUG)
kdi_dvec_vmready();
}
/*
* Macro used below to convert the prom's 32-bit high and low fields into
* a value appropriate for the 64-bit kernel.
*/
#define COMBINE(hi, lo) (((uint64_t)(uint32_t)(hi) << 32) | (uint32_t)(lo))
/*
* Track larges pages used.
* Provides observability for this feature on non-debug kernels.
*/
ulong_t map_prom_lpcount[MMU_PAGE_SIZES];
/*
* This function traverses the prom mapping list and creates equivalent
* mappings in the sfmmu mapping hash.
*/
static void
sfmmu_map_prom_mappings(struct translation *trans_root, size_t ntrans_root)
{
struct translation *promt;
tte_t tte, oldtte, *ttep;
pfn_t pfn, oldpfn, basepfn;
caddr_t vaddr;
size_t size, offset;
unsigned long i;
uint_t attr;
page_t *pp;
extern struct memlist *virt_avail;
char buf[256];
ttep = &tte;
for (i = 0, promt = trans_root; i < ntrans_root; i++, promt++) {
ASSERT(promt->tte_hi != 0);
ASSERT32(promt->virt_hi == 0 && promt->size_hi == 0);
vaddr = (caddr_t)COMBINE(promt->virt_hi, promt->virt_lo);
/*
* hack until we get rid of map-for-unix
*/
if (vaddr < (caddr_t)KERNELBASE)
continue;
ttep->tte_inthi = promt->tte_hi;
ttep->tte_intlo = promt->tte_lo;
attr = PROC_DATA | HAT_NOSYNC;
#if defined(TTE_IS_GLOBAL)
if (TTE_IS_GLOBAL(ttep)) {
/*
* The prom better not use global translations
* because a user process might use the same
* virtual addresses
*/
prom_panic("sfmmu_map_prom_mappings: global"
" translation");
TTE_SET_LOFLAGS(ttep, TTE_GLB_INT, 0);
}
#endif
if (TTE_IS_LOCKED(ttep)) {
/* clear the lock bits */
TTE_CLR_LOCKED(ttep);
}
attr |= (TTE_IS_VCACHEABLE(ttep)) ? 0 : SFMMU_UNCACHEVTTE;
attr |= (TTE_IS_PCACHEABLE(ttep)) ? 0 : SFMMU_UNCACHEPTTE;
attr |= (TTE_IS_SIDEFFECT(ttep)) ? SFMMU_SIDEFFECT : 0;
attr |= (TTE_IS_IE(ttep)) ? HAT_STRUCTURE_LE : 0;
size = COMBINE(promt->size_hi, promt->size_lo);
offset = 0;
basepfn = TTE_TO_PFN((caddr_t)COMBINE(promt->virt_hi,
promt->virt_lo), ttep);
while (size) {
vaddr = (caddr_t)(COMBINE(promt->virt_hi,
promt->virt_lo) + offset);
/*
* make sure address is not in virt-avail list
*/
if (address_in_memlist(virt_avail, (uint64_t)vaddr,
size)) {
prom_panic("sfmmu_map_prom_mappings:"
" inconsistent translation/avail lists");
}
pfn = basepfn + mmu_btop(offset);
if (pf_is_memory(pfn)) {
if (attr & SFMMU_UNCACHEPTTE) {
prom_panic("sfmmu_map_prom_mappings:"
" uncached prom memory page");
}
} else {
if (!(attr & SFMMU_SIDEFFECT)) {
prom_panic("sfmmu_map_prom_mappings:"
" prom i/o page without"
" side-effect");
}
}
/*
* skip kmem64 area
*/
if (!kmem64_smchunks &&
vaddr >= kmem64_base &&
vaddr < kmem64_aligned_end) {
#if !defined(C_OBP)
prom_panic("sfmmu_map_prom_mappings:"
" unexpected kmem64 prom mapping");
#else /* !C_OBP */
size_t mapsz;
if (ptob(pfn) !=
kmem64_pabase + (vaddr - kmem64_base)) {
prom_panic("sfmmu_map_prom_mappings:"
" unexpected kmem64 prom mapping");
}
mapsz = kmem64_aligned_end - vaddr;
if (mapsz >= size) {
break;
}
size -= mapsz;
offset += mapsz;
continue;
#endif /* !C_OBP */
}
oldpfn = sfmmu_vatopfn(vaddr, KHATID, &oldtte);
ASSERT(oldpfn != PFN_SUSPENDED);
ASSERT(page_relocate_ready == 0);
if (oldpfn != PFN_INVALID) {
/*
* mapping already exists.
* Verify they are equal
*/
if (pfn != oldpfn) {
(void) snprintf(buf, sizeof (buf),
"sfmmu_map_prom_mappings: mapping"
" conflict (va = 0x%p, pfn = 0x%p,"
" oldpfn = 0x%p)", (void *)vaddr,
(void *)pfn, (void *)oldpfn);
prom_panic(buf);
}
size -= MMU_PAGESIZE;
offset += MMU_PAGESIZE;
continue;
}
pp = page_numtopp_nolock(pfn);
if ((pp != NULL) && PP_ISFREE((page_t *)pp)) {
(void) snprintf(buf, sizeof (buf),
"sfmmu_map_prom_mappings: prom-mapped"
" page (va = 0x%p, pfn = 0x%p) on free list",
(void *)vaddr, (void *)pfn);
prom_panic(buf);
}
sfmmu_memtte(ttep, pfn, attr, TTE8K);
sfmmu_tteload(kas.a_hat, ttep, vaddr, pp,
HAT_LOAD_LOCK | SFMMU_NO_TSBLOAD);
size -= MMU_PAGESIZE;
offset += MMU_PAGESIZE;
}
}
/*
* We claimed kmem64 from prom, so now we need to load tte.
*/
if (!kmem64_smchunks && kmem64_base != NULL) {
pgcnt_t pages;
size_t psize;
int pszc;
pszc = kmem64_szc;
#ifdef sun4u
if (pszc > TTE8K) {
pszc = segkmem_lpszc;
}
#endif /* sun4u */
psize = TTEBYTES(pszc);
pages = btop(psize);
basepfn = kmem64_pabase >> MMU_PAGESHIFT;
vaddr = kmem64_base;
while (vaddr < kmem64_end) {
sfmmu_memtte(ttep, basepfn,
PROC_DATA | HAT_NOSYNC, pszc);
sfmmu_tteload(kas.a_hat, ttep, vaddr, NULL,
HAT_LOAD_LOCK | SFMMU_NO_TSBLOAD);
vaddr += psize;
basepfn += pages;
}
map_prom_lpcount[pszc] =
((caddr_t)P2ROUNDUP((uintptr_t)kmem64_end, psize) -
kmem64_base) >> TTE_PAGE_SHIFT(pszc);
}
}
#undef COMBINE /* local to previous routine */
/*
* This routine reads in the "translations" property in to a buffer and
* returns a pointer to this buffer and the number of translations.
*/
static struct translation *
read_prom_mappings(size_t *ntransrootp)
{
char *prop = "translations";
size_t translen;
pnode_t node;
struct translation *transroot;
/*
* the "translations" property is associated with the mmu node
*/
node = (pnode_t)prom_getphandle(prom_mmu_ihandle());
/*
* We use the TSB space to read in the prom mappings. This space
* is currently not being used because we haven't taken over the
* trap table yet. It should be big enough to hold the mappings.
*/
if ((translen = prom_getproplen(node, prop)) == -1)
cmn_err(CE_PANIC, "no translations property");
*ntransrootp = translen / sizeof (*transroot);
translen = roundup(translen, MMU_PAGESIZE);
PRM_DEBUG(translen);
if (translen > TSB_BYTES(ktsb_szcode))
cmn_err(CE_PANIC, "not enough space for translations");
transroot = (struct translation *)ktsb_base;
ASSERT(transroot);
if (prom_getprop(node, prop, (caddr_t)transroot) == -1) {
cmn_err(CE_PANIC, "translations getprop failed");
}
return (transroot);
}
/*
* Init routine of the nucleus data memory allocator.
*
* The nucleus data memory allocator is organized in ecache_alignsize'd
* memory chunks. Memory allocated by ndata_alloc() will never be freed.
*
* The ndata argument is used as header of the ndata freelist.
* Other freelist nodes are placed in the nucleus memory itself
* at the beginning of a free memory chunk. Therefore a freelist
* node (struct memlist) must fit into the smallest allocatable
* memory chunk (ecache_alignsize bytes).
*
* The memory interval [base, end] passed to ndata_alloc_init() must be
* bzero'd to allow the allocator to return bzero'd memory easily.
*/
void
ndata_alloc_init(struct memlist *ndata, uintptr_t base, uintptr_t end)
{
ASSERT(sizeof (struct memlist) <= ecache_alignsize);
base = roundup(base, ecache_alignsize);
end = end - end % ecache_alignsize;
ASSERT(base < end);
ndata->ml_address = base;
ndata->ml_size = end - base;
ndata->ml_next = NULL;
ndata->ml_prev = NULL;
}
/*
* Deliver the size of the largest free memory chunk.
*/
size_t
ndata_maxsize(struct memlist *ndata)
{
size_t chunksize = ndata->ml_size;
while ((ndata = ndata->ml_next) != NULL) {
if (chunksize < ndata->ml_size)
chunksize = ndata->ml_size;
}
return (chunksize);
}
/*
* Allocate the last properly aligned memory chunk.
* This function is called when no more large nucleus memory chunks
* will be allocated. The remaining free nucleus memory at the end
* of the nucleus can be added to the phys_avail list.
*/
void *
ndata_extra_base(struct memlist *ndata, size_t alignment, caddr_t endaddr)
{
uintptr_t base;
size_t wasteage = 0;
#ifdef DEBUG
static int called = 0;
if (called++ > 0)
cmn_err(CE_PANIC, "ndata_extra_base() called more than once");
#endif /* DEBUG */
/*
* The alignment needs to be a multiple of ecache_alignsize.
*/
ASSERT((alignment % ecache_alignsize) == 0);
while (ndata->ml_next != NULL) {
wasteage += ndata->ml_size;
ndata = ndata->ml_next;
}
base = roundup(ndata->ml_address, alignment);
if (base >= ndata->ml_address + ndata->ml_size)
return (NULL);
if ((caddr_t)(ndata->ml_address + ndata->ml_size) != endaddr) {
#ifdef DEBUG
ndata_middle_hole_detected = 1; /* see if we hit this again */
#endif
return (NULL);
}
if (base == ndata->ml_address) {
if (ndata->ml_prev != NULL)
ndata->ml_prev->ml_next = NULL;
else
ndata->ml_size = 0;
bzero((void *)base, sizeof (struct memlist));
} else {
ndata->ml_size = base - ndata->ml_address;
wasteage += ndata->ml_size;
}
PRM_DEBUG(wasteage);
return ((void *)base);
}
/*
* Select the best matching buffer, avoid memory fragmentation.
*/
static struct memlist *
ndata_select_chunk(struct memlist *ndata, size_t wanted, size_t alignment)
{
struct memlist *fnd_below = NULL;
struct memlist *fnd_above = NULL;
struct memlist *fnd_unused = NULL;
struct memlist *frlist;
uintptr_t base;
uintptr_t end;
size_t below;
size_t above;
size_t unused;
size_t best_below = ULONG_MAX;
size_t best_above = ULONG_MAX;
size_t best_unused = ULONG_MAX;
ASSERT(ndata != NULL);
/*
* Look for the best matching buffer, avoid memory fragmentation.
* The following strategy is used, try to find
* 1. an exact fitting buffer
* 2. avoid wasting any space below the buffer, take first
* fitting buffer
* 3. avoid wasting any space above the buffer, take first
* fitting buffer
* 4. avoid wasting space, take first fitting buffer
* 5. take the last buffer in chain
*/
for (frlist = ndata; frlist != NULL; frlist = frlist->ml_next) {
base = roundup(frlist->ml_address, alignment);
end = roundup(base + wanted, ecache_alignsize);
if (end > frlist->ml_address + frlist->ml_size)
continue;
below = (base - frlist->ml_address) / ecache_alignsize;
above = (frlist->ml_address + frlist->ml_size - end) /
ecache_alignsize;
unused = below + above;
if (unused == 0)
return (frlist);
if (frlist->ml_next == NULL)
break;
if (below < best_below) {
best_below = below;
fnd_below = frlist;
}
if (above < best_above) {
best_above = above;
fnd_above = frlist;
}
if (unused < best_unused) {
best_unused = unused;
fnd_unused = frlist;
}
}
if (best_below == 0)
return (fnd_below);
if (best_above == 0)
return (fnd_above);
if (best_unused < ULONG_MAX)
return (fnd_unused);
return (frlist);
}
/*
* Nucleus data memory allocator.
* The granularity of the allocator is ecache_alignsize.
* See also comment for ndata_alloc_init().
*/
void *
ndata_alloc(struct memlist *ndata, size_t wanted, size_t alignment)
{
struct memlist *found;
struct memlist *fnd_above;
uintptr_t base;
uintptr_t end;
size_t below;
size_t above;
/*
* Look for the best matching buffer, avoid memory fragmentation.
*/
if ((found = ndata_select_chunk(ndata, wanted, alignment)) == NULL)
return (NULL);
/*
* Allocate the nucleus data buffer.
*/
base = roundup(found->ml_address, alignment);
end = roundup(base + wanted, ecache_alignsize);
ASSERT(end <= found->ml_address + found->ml_size);
below = base - found->ml_address;
above = found->ml_address + found->ml_size - end;
ASSERT(above == 0 || (above % ecache_alignsize) == 0);
if (below >= ecache_alignsize) {
/*
* There is free memory below the allocated memory chunk.
*/
found->ml_size = below - below % ecache_alignsize;
if (above) {
fnd_above = (struct memlist *)end;
fnd_above->ml_address = end;
fnd_above->ml_size = above;
if ((fnd_above->ml_next = found->ml_next) != NULL)
found->ml_next->ml_prev = fnd_above;
fnd_above->ml_prev = found;
found->ml_next = fnd_above;
}
return ((void *)base);
}
if (found->ml_prev == NULL) {
/*
* The first chunk (ndata) is selected.
*/
ASSERT(found == ndata);
if (above) {
found->ml_address = end;
found->ml_size = above;
} else if (found->ml_next != NULL) {
found->ml_address = found->ml_next->ml_address;
found->ml_size = found->ml_next->ml_size;
if ((found->ml_next = found->ml_next->ml_next) != NULL)
found->ml_next->ml_prev = found;
bzero((void *)found->ml_address,
sizeof (struct memlist));
} else {
found->ml_address = end;
found->ml_size = 0;
}
return ((void *)base);
}
/*
* Not the first chunk.
*/
if (above) {
fnd_above = (struct memlist *)end;
fnd_above->ml_address = end;
fnd_above->ml_size = above;
if ((fnd_above->ml_next = found->ml_next) != NULL)
fnd_above->ml_next->ml_prev = fnd_above;
fnd_above->ml_prev = found->ml_prev;
found->ml_prev->ml_next = fnd_above;
} else {
if ((found->ml_prev->ml_next = found->ml_next) != NULL)
found->ml_next->ml_prev = found->ml_prev;
}
bzero((void *)found->ml_address, sizeof (struct memlist));
return ((void *)base);
}
/*
* Size the kernel TSBs based upon the amount of physical
* memory in the system.
*/
static void
calc_tsb_sizes(pgcnt_t npages)
{
PRM_DEBUG(npages);
if (npages <= TSB_FREEMEM_MIN) {
ktsb_szcode = TSB_128K_SZCODE;
enable_bigktsb = 0;
} else if (npages <= TSB_FREEMEM_LARGE / 2) {
ktsb_szcode = TSB_256K_SZCODE;
enable_bigktsb = 0;
} else if (npages <= TSB_FREEMEM_LARGE) {
ktsb_szcode = TSB_512K_SZCODE;
enable_bigktsb = 0;
} else if (npages <= TSB_FREEMEM_LARGE * 2 ||
enable_bigktsb == 0) {
ktsb_szcode = TSB_1M_SZCODE;
enable_bigktsb = 0;
} else {
ktsb_szcode = highbit(npages - 1);
ktsb_szcode -= TSB_START_SIZE;
ktsb_szcode = MAX(ktsb_szcode, MIN_BIGKTSB_SZCODE);
ktsb_szcode = MIN(ktsb_szcode, MAX_BIGKTSB_SZCODE);
}
/*
* We choose the TSB to hold kernel 4M mappings to have twice
* the reach as the primary kernel TSB since this TSB will
* potentially (currently) be shared by both mappings to all of
* physical memory plus user TSBs. If this TSB has to be in nucleus
* (only for Spitfire and Cheetah) limit its size to 64K.
*/
ktsb4m_szcode = highbit((2 * npages) / TTEPAGES(TTE4M) - 1);
ktsb4m_szcode -= TSB_START_SIZE;
ktsb4m_szcode = MAX(ktsb4m_szcode, TSB_MIN_SZCODE);
ktsb4m_szcode = MIN(ktsb4m_szcode, TSB_SOFTSZ_MASK);
if ((enable_bigktsb == 0 || ktsb_phys == 0) && ktsb4m_szcode >
TSB_64K_SZCODE) {
ktsb4m_szcode = TSB_64K_SZCODE;
max_bootlp_tteszc = TTE8K;
}
ktsb_sz = TSB_BYTES(ktsb_szcode); /* kernel 8K tsb size */
ktsb4m_sz = TSB_BYTES(ktsb4m_szcode); /* kernel 4M tsb size */
}
/*
* Allocate kernel TSBs from nucleus data memory.
* The function return 0 on success and -1 on failure.
*/
int
ndata_alloc_tsbs(struct memlist *ndata, pgcnt_t npages)
{
/*
* Set ktsb_phys to 1 if the processor supports ASI_QUAD_LDD_PHYS.
*/
(void) sfmmu_setup_4lp();
/*
* Size the kernel TSBs based upon the amount of physical
* memory in the system.
*/
calc_tsb_sizes(npages);
/*
* Allocate the 8K kernel TSB if it belongs inside the nucleus.
*/
if (enable_bigktsb == 0) {
if ((ktsb_base = ndata_alloc(ndata, ktsb_sz, ktsb_sz)) == NULL)
return (-1);
ASSERT(!((uintptr_t)ktsb_base & (ktsb_sz - 1)));
PRM_DEBUG(ktsb_base);
PRM_DEBUG(ktsb_sz);
PRM_DEBUG(ktsb_szcode);
}
/*
* Next, allocate 4M kernel TSB from the nucleus since it's small.
*/
if (ktsb4m_szcode <= TSB_64K_SZCODE) {
ktsb4m_base = ndata_alloc(ndata, ktsb4m_sz, ktsb4m_sz);
if (ktsb4m_base == NULL)
return (-1);
ASSERT(!((uintptr_t)ktsb4m_base & (ktsb4m_sz - 1)));
PRM_DEBUG(ktsb4m_base);
PRM_DEBUG(ktsb4m_sz);
PRM_DEBUG(ktsb4m_szcode);
}
return (0);
}
size_t
calc_hmehash_sz(pgcnt_t npages)
{
ulong_t hme_buckets;
/*
* The number of buckets in the hme hash tables
* is a power of 2 such that the average hash chain length is
* HMENT_HASHAVELEN. The number of buckets for the user hash is
* a function of physical memory and a predefined overmapping factor.
* The number of buckets for the kernel hash is a function of
* physical memory only.
*/
hme_buckets = (npages * HMEHASH_FACTOR) /
(HMENT_HASHAVELEN * (HMEBLK_SPAN(TTE8K) >> MMU_PAGESHIFT));
uhmehash_num = (int)MIN(hme_buckets, MAX_UHME_BUCKETS);
if (uhmehash_num > USER_BUCKETS_THRESHOLD) {
/*
* if uhmehash_num is not power of 2 round it down to the
* next power of 2.
*/
uint_t align = 1 << (highbit(uhmehash_num - 1) - 1);
uhmehash_num = P2ALIGN(uhmehash_num, align);
} else
uhmehash_num = 1 << highbit(uhmehash_num - 1);
hme_buckets = npages / (HMEBLK_SPAN(TTE8K) >> MMU_PAGESHIFT);
khmehash_num = (int)MIN(hme_buckets, MAX_KHME_BUCKETS);
khmehash_num = 1 << highbit(khmehash_num - 1);
khmehash_num = MAX(khmehash_num, MIN_KHME_BUCKETS);
return ((uhmehash_num + khmehash_num) * sizeof (struct hmehash_bucket));
}
caddr_t
alloc_hmehash(caddr_t alloc_base)
{
size_t khmehash_sz, uhmehash_sz;
khme_hash = (struct hmehash_bucket *)alloc_base;
khmehash_sz = khmehash_num * sizeof (struct hmehash_bucket);
alloc_base += khmehash_sz;
uhme_hash = (struct hmehash_bucket *)alloc_base;
uhmehash_sz = uhmehash_num * sizeof (struct hmehash_bucket);
alloc_base += uhmehash_sz;
PRM_DEBUG(khme_hash);
PRM_DEBUG(uhme_hash);
return (alloc_base);
}
/*
* Allocate hat structs from the nucleus data memory.
*/
int
ndata_alloc_hat(struct memlist *ndata, pgcnt_t npages)
{
size_t mml_alloc_sz;
size_t cb_alloc_sz;
/*
* For the page mapping list mutex array we allocate one mutex
* for every 128 pages (1 MB) with a minimum of 64 entries and
* a maximum of 8K entries. For the initial computation npages
* is rounded up (ie. 1 << highbit(npages * 1.5 / 128))
*
* mml_shift is roughly log2(mml_table_sz) + 3 for MLIST_HASH
*/
mml_table_sz = 1 << highbit((npages * 3) / 256);
if (mml_table_sz < 64)
mml_table_sz = 64;
else if (mml_table_sz > 8192)
mml_table_sz = 8192;
mml_shift = highbit(mml_table_sz) + 3;
PRM_DEBUG(mml_table_sz);
PRM_DEBUG(mml_shift);
mml_alloc_sz = mml_table_sz * sizeof (kmutex_t);
mml_table = ndata_alloc(ndata, mml_alloc_sz, ecache_alignsize);
if (mml_table == NULL)
return (-1);
PRM_DEBUG(mml_table);
cb_alloc_sz = sfmmu_max_cb_id * sizeof (struct sfmmu_callback);
PRM_DEBUG(cb_alloc_sz);
sfmmu_cb_table = ndata_alloc(ndata, cb_alloc_sz, ecache_alignsize);
if (sfmmu_cb_table == NULL)
return (-1);
PRM_DEBUG(sfmmu_cb_table);
return (0);
}
int
ndata_alloc_kpm(struct memlist *ndata, pgcnt_t kpm_npages)
{
size_t kpmp_alloc_sz;
/*
* For the kpm_page mutex array we allocate one mutex every 16
* kpm pages (64MB). In smallpage mode we allocate one mutex
* every 8K pages. The minimum is set to 64 entries and the
* maximum to 8K entries.
*/
if (kpm_smallpages == 0) {
kpmp_shift = highbit(sizeof (kpm_page_t)) - 1;
kpmp_table_sz = 1 << highbit(kpm_npages / 16);
kpmp_table_sz = (kpmp_table_sz < 64) ? 64 :
((kpmp_table_sz > 8192) ? 8192 : kpmp_table_sz);
kpmp_alloc_sz = kpmp_table_sz * sizeof (kpm_hlk_t);
kpmp_table = ndata_alloc(ndata, kpmp_alloc_sz,
ecache_alignsize);
if (kpmp_table == NULL)
return (-1);
PRM_DEBUG(kpmp_table);
PRM_DEBUG(kpmp_table_sz);
kpmp_stable_sz = 0;
kpmp_stable = NULL;
} else {
ASSERT(kpm_pgsz == PAGESIZE);
kpmp_shift = highbit(sizeof (kpm_shlk_t)) + 1;
kpmp_stable_sz = 1 << highbit(kpm_npages / 8192);
kpmp_stable_sz = (kpmp_stable_sz < 64) ? 64 :
((kpmp_stable_sz > 8192) ? 8192 : kpmp_stable_sz);
kpmp_alloc_sz = kpmp_stable_sz * sizeof (kpm_shlk_t);
kpmp_stable = ndata_alloc(ndata, kpmp_alloc_sz,
ecache_alignsize);
if (kpmp_stable == NULL)
return (-1);
PRM_DEBUG(kpmp_stable);
PRM_DEBUG(kpmp_stable_sz);
kpmp_table_sz = 0;
kpmp_table = NULL;
}
PRM_DEBUG(kpmp_shift);
return (0);
}
/*
* This function bop allocs kernel TSBs.
*/
caddr_t
sfmmu_ktsb_alloc(caddr_t tsbbase)
{
caddr_t vaddr;
if (enable_bigktsb) {
ktsb_base = (caddr_t)roundup((uintptr_t)tsbbase, ktsb_sz);
vaddr = prom_alloc(ktsb_base, ktsb_sz, ktsb_sz);
if (vaddr != ktsb_base)
cmn_err(CE_PANIC, "sfmmu_ktsb_alloc: can't alloc"
" 8K bigktsb");
ktsb_base = vaddr;
tsbbase = ktsb_base + ktsb_sz;
PRM_DEBUG(ktsb_base);
PRM_DEBUG(tsbbase);
}
if (ktsb4m_szcode > TSB_64K_SZCODE) {
ASSERT(ktsb_phys && enable_bigktsb);
ktsb4m_base = (caddr_t)roundup((uintptr_t)tsbbase, ktsb4m_sz);
vaddr = (caddr_t)BOP_ALLOC(bootops, ktsb4m_base, ktsb4m_sz,
ktsb4m_sz);
if (vaddr != ktsb4m_base)
cmn_err(CE_PANIC, "sfmmu_ktsb_alloc: can't alloc"
" 4M bigktsb");
ktsb4m_base = vaddr;
tsbbase = ktsb4m_base + ktsb4m_sz;
PRM_DEBUG(ktsb4m_base);
PRM_DEBUG(tsbbase);
}
return (tsbbase);
}
/*
* Moves code assembled outside of the trap table into the trap
* table taking care to relocate relative branches to code outside
* of the trap handler.
*/
static void
sfmmu_reloc_trap_handler(void *tablep, void *start, size_t count)
{
size_t i;
uint32_t *src;
uint32_t *dst;
uint32_t inst;
int op, op2;
int32_t offset;
int disp;
src = start;
dst = tablep;
offset = src - dst;
for (src = start, i = 0; i < count; i++, src++, dst++) {
inst = *dst = *src;
op = (inst >> 30) & 0x2;
if (op == 1) {
/* call */
disp = ((int32_t)inst << 2) >> 2; /* sign-extend */
if (disp + i >= 0 && disp + i < count)
continue;
disp += offset;
inst = 0x40000000u | (disp & 0x3fffffffu);
*dst = inst;
} else if (op == 0) {
/* branch or sethi */
op2 = (inst >> 22) & 0x7;
switch (op2) {
case 0x3: /* BPr */
disp = (((inst >> 20) & 0x3) << 14) |
(inst & 0x3fff);
disp = (disp << 16) >> 16; /* sign-extend */
if (disp + i >= 0 && disp + i < count)
continue;
disp += offset;
if (((disp << 16) >> 16) != disp)
cmn_err(CE_PANIC, "bad reloc");
inst &= ~0x303fff;
inst |= (disp & 0x3fff);
inst |= (disp & 0xc000) << 6;
break;
case 0x2: /* Bicc */
disp = ((int32_t)inst << 10) >> 10;
if (disp + i >= 0 && disp + i < count)
continue;
disp += offset;
if (((disp << 10) >> 10) != disp)
cmn_err(CE_PANIC, "bad reloc");
inst &= ~0x3fffff;
inst |= (disp & 0x3fffff);
break;
case 0x1: /* Bpcc */
disp = ((int32_t)inst << 13) >> 13;
if (disp + i >= 0 && disp + i < count)
continue;
disp += offset;
if (((disp << 13) >> 13) != disp)
cmn_err(CE_PANIC, "bad reloc");
inst &= ~0x7ffff;
inst |= (disp & 0x7ffffu);
break;
}
*dst = inst;
}
}
flush_instr_mem(tablep, count * sizeof (uint32_t));
}
/*
* Routine to allocate a large page to use in the TSB caches.
*/
/*ARGSUSED*/
static page_t *
sfmmu_tsb_page_create(void *addr, size_t size, int vmflag, void *arg)
{
int pgflags;
pgflags = PG_EXCL;
if ((vmflag & VM_NOSLEEP) == 0)
pgflags |= PG_WAIT;
if (vmflag & VM_PANIC)
pgflags |= PG_PANIC;
if (vmflag & VM_PUSHPAGE)
pgflags |= PG_PUSHPAGE;
return (page_create_va_large(&kvp, (u_offset_t)(uintptr_t)addr, size,
pgflags, &kvseg, addr, arg));
}
/*
* Allocate a large page to back the virtual address range
* [addr, addr + size). If addr is NULL, allocate the virtual address
* space as well.
*/
static void *
sfmmu_tsb_xalloc(vmem_t *vmp, void *inaddr, size_t size, int vmflag,
uint_t attr, page_t *(*page_create_func)(void *, size_t, int, void *),
void *pcarg)
{
page_t *ppl;
page_t *rootpp;
caddr_t addr = inaddr;
pgcnt_t npages = btopr(size);
page_t **ppa;
int i = 0;
/*
* Assuming that only TSBs will call this with size > PAGESIZE
* There is no reason why this couldn't be expanded to 8k pages as
* well, or other page sizes in the future .... but for now, we
* only support fixed sized page requests.
*/
if ((inaddr == NULL) && ((addr = vmem_xalloc(vmp, size, size, 0, 0,
NULL, NULL, vmflag)) == NULL))
return (NULL);
if (page_resv(npages, vmflag & VM_KMFLAGS) == 0) {
if (inaddr == NULL)
vmem_xfree(vmp, addr, size);
return (NULL);
}
ppl = page_create_func(addr, size, vmflag, pcarg);
if (ppl == NULL) {
if (inaddr == NULL)
vmem_xfree(vmp, addr, size);
page_unresv(npages);
return (NULL);
}
rootpp = ppl;
ppa = kmem_zalloc(npages * sizeof (page_t *), KM_SLEEP);
while (ppl != NULL) {
page_t *pp = ppl;
ppa[i++] = pp;
page_sub(&ppl, pp);
ASSERT(page_iolock_assert(pp));
page_io_unlock(pp);
}
/*
* Load the locked entry. It's OK to preload the entry into
* the TSB since we now support large mappings in the kernel TSB.
*/
hat_memload_array(kas.a_hat, (caddr_t)rootpp->p_offset, size,
ppa, (PROT_ALL & ~PROT_USER) | HAT_NOSYNC | attr, HAT_LOAD_LOCK);
for (--i; i >= 0; --i) {
(void) page_pp_lock(ppa[i], 0, 1);
page_unlock(ppa[i]);
}
kmem_free(ppa, npages * sizeof (page_t *));
return (addr);
}
/* Called to import new spans into the TSB vmem arenas */
void *
sfmmu_tsb_segkmem_alloc(vmem_t *vmp, size_t size, int vmflag)
{
lgrp_id_t lgrpid = LGRP_NONE;
if (tsb_lgrp_affinity) {
/*
* Search for the vmp->lgrpid mapping by brute force;
* some day vmp will have an lgrp, until then we have
* to do this the hard way.
*/
for (lgrpid = 0; lgrpid < NLGRPS_MAX &&
vmp != kmem_tsb_default_arena[lgrpid]; lgrpid++)
;
if (lgrpid == NLGRPS_MAX)
lgrpid = LGRP_NONE;
}
return (sfmmu_tsb_xalloc(vmp, NULL, size, vmflag, 0,
sfmmu_tsb_page_create, lgrpid != LGRP_NONE? &lgrpid : NULL));
}
/* Called to free spans from the TSB vmem arenas */
void
sfmmu_tsb_segkmem_free(vmem_t *vmp, void *inaddr, size_t size)
{
page_t *pp;
caddr_t addr = inaddr;
caddr_t eaddr;
pgcnt_t npages = btopr(size);
pgcnt_t pgs_left = npages;
page_t *rootpp = NULL;
hat_unload(kas.a_hat, addr, size, HAT_UNLOAD_UNLOCK);
for (eaddr = addr + size; addr < eaddr; addr += PAGESIZE) {
pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)addr, SE_EXCL);
if (pp == NULL)
panic("sfmmu_tsb_segkmem_free: page not found");
ASSERT(PAGE_EXCL(pp));
page_pp_unlock(pp, 0, 1);
if (rootpp == NULL)
rootpp = pp;
if (--pgs_left == 0) {
/*
* similar logic to segspt_free_pages, but we know we
* have one large page.
*/
page_destroy_pages(rootpp);
}
}
page_unresv(npages);
if (vmp != NULL)
vmem_xfree(vmp, inaddr, size);
}