OpenSolaris_b135/uts/i86pc/os/lgrpplat.c
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or http://www.opensolaris.org/os/licensing.
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2010 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
/*
* LOCALITY GROUP (LGROUP) PLATFORM SUPPORT FOR X86/AMD64 PLATFORMS
* ================================================================
* Multiprocessor AMD and Intel systems may have Non Uniform Memory Access
* (NUMA). A NUMA machine consists of one or more "nodes" that each consist of
* one or more CPUs and some local memory. The CPUs in each node can access
* the memory in the other nodes but at a higher latency than accessing their
* local memory. Typically, a system with only one node has Uniform Memory
* Access (UMA), but it may be possible to have a one node system that has
* some global memory outside of the node which is higher latency.
*
* Module Description
* ------------------
* This module provides a platform interface for determining which CPUs and
* which memory (and how much) are in a NUMA node and how far each node is from
* each other. The interface is used by the Virtual Memory (VM) system and the
* common lgroup framework. The VM system uses the plat_*() routines to fill
* in its memory node (memnode) array with the physical address range spanned
* by each NUMA node to know which memory belongs to which node, so it can
* build and manage a physical page free list for each NUMA node and allocate
* local memory from each node as needed. The common lgroup framework uses the
* exported lgrp_plat_*() routines to figure out which CPUs and memory belong
* to each node (leaf lgroup) and how far each node is from each other, so it
* can build the latency (lgroup) topology for the machine in order to optimize
* for locality. Also, an lgroup platform handle instead of lgroups are used
* in the interface with this module, so this module shouldn't need to know
* anything about lgroups. Instead, it just needs to know which CPUs, memory,
* etc. are in each NUMA node, how far each node is from each other, and to use
* a unique lgroup platform handle to refer to each node through the interface.
*
* Determining NUMA Configuration
* ------------------------------
* By default, this module will try to determine the NUMA configuration of the
* machine by reading the ACPI System Resource Affinity Table (SRAT) and System
* Locality Information Table (SLIT). The SRAT contains info to tell which
* CPUs and memory are local to a given proximity domain (NUMA node). The SLIT
* is a matrix that gives the distance between each system locality (which is
* a NUMA node and should correspond to proximity domains in the SRAT). For
* more details on the SRAT and SLIT, please refer to an ACPI 3.0 or newer
* specification.
*
* If the SRAT doesn't exist on a system with AMD Opteron processors, we
* examine registers in PCI configuration space to determine how many nodes are
* in the system and which CPUs and memory are in each node.
* do while booting the kernel.
*
* NOTE: Using these PCI configuration space registers to determine this
* locality info is not guaranteed to work or be compatible across all
* Opteron processor families.
*
* If the SLIT does not exist or look right, the kernel will probe to determine
* the distance between nodes as long as the NUMA CPU and memory configuration
* has been determined (see lgrp_plat_probe() for details).
*
* Data Structures
* ---------------
* The main data structures used by this code are the following:
*
* - lgrp_plat_cpu_node[] CPU to node ID mapping table indexed by
* CPU ID (only used for SRAT)
*
* - lgrp_plat_lat_stats.latencies[][] Table of latencies between same and
* different nodes indexed by node ID
*
* - lgrp_plat_node_cnt Number of NUMA nodes in system
*
* - lgrp_plat_node_domain[] Node ID to proximity domain ID mapping
* table indexed by node ID (only used
* for SRAT)
*
* - lgrp_plat_node_memory[] Table with physical address range for
* each node indexed by node ID
*
* The code is implemented to make the following always be true:
*
* lgroup platform handle == node ID == memnode ID
*
* Moreover, it allows for the proximity domain ID to be equal to all of the
* above as long as the proximity domains IDs are numbered from 0 to <number of
* nodes - 1>. This is done by hashing each proximity domain ID into the range
* from 0 to <number of nodes - 1>. Then proximity ID N will hash into node ID
* N and proximity domain ID N will be entered into lgrp_plat_node_domain[N]
* and be assigned node ID N. If the proximity domain IDs aren't numbered
* from 0 to <number of nodes - 1>, then hashing the proximity domain IDs into
* lgrp_plat_node_domain[] will still work for assigning proximity domain IDs
* to node IDs. However, the proximity domain IDs may not map to the
* equivalent node ID since we want to keep the node IDs numbered from 0 to
* <number of nodes - 1> to minimize cost of searching and potentially space.
*
* The code below really tries to do the above. However, the virtual memory
* system expects the memnodes which describe the physical address range for
* each NUMA node to be arranged in ascending order by physical address. (:-(
* Otherwise, the kernel will panic in different semi-random places in the VM
* system.
*
* Consequently, this module has to try to sort the nodes in ascending order by
* each node's starting physical address to try to meet this "constraint" in
* the VM system (see lgrp_plat_node_sort()). Also, the lowest numbered
* proximity domain ID in the system is deteremined and used to make the lowest
* numbered proximity domain map to node 0 in hopes that the proximity domains
* are sorted in ascending order by physical address already even if their IDs
* don't start at 0 (see NODE_DOMAIN_HASH() and lgrp_plat_srat_domains()).
* Finally, it is important to note that these workarounds may not be
* sufficient if/when memory hotplugging is supported and the VM system may
* ultimately need to be fixed to handle this....
*/
#include <sys/archsystm.h> /* for {in,out}{b,w,l}() */
#include <sys/bootconf.h>
#include <sys/cmn_err.h>
#include <sys/controlregs.h>
#include <sys/cpupart.h>
#include <sys/cpuvar.h>
#include <sys/lgrp.h>
#include <sys/machsystm.h>
#include <sys/memlist.h>
#include <sys/memnode.h>
#include <sys/mman.h>
#include <sys/pci_cfgspace.h>
#include <sys/pci_impl.h>
#include <sys/param.h>
#include <sys/pghw.h>
#include <sys/promif.h> /* for prom_printf() */
#include <sys/sysmacros.h>
#include <sys/systm.h>
#include <sys/thread.h>
#include <sys/types.h>
#include <sys/var.h>
#include <sys/x86_archext.h> /* for x86_feature and X86_AMD */
#include <vm/hat_i86.h>
#include <vm/seg_kmem.h>
#include <vm/vm_dep.h>
#include "acpi_fw.h" /* for SRAT and SLIT */
#define MAX_NODES 8
#define NLGRP (MAX_NODES * (MAX_NODES - 1) + 1)
/*
* Constants for configuring probing
*/
#define LGRP_PLAT_PROBE_NROUNDS 64 /* default laps for probing */
#define LGRP_PLAT_PROBE_NSAMPLES 1 /* default samples to take */
#define LGRP_PLAT_PROBE_NREADS 256 /* number of vendor ID reads */
/*
* Flags for probing
*/
#define LGRP_PLAT_PROBE_ENABLE 0x1 /* enable probing */
#define LGRP_PLAT_PROBE_PGCPY 0x2 /* probe using page copy */
#define LGRP_PLAT_PROBE_VENDOR 0x4 /* probe vendor ID register */
/*
* Hash proximity domain ID into node to domain mapping table "mod" number of
* nodes to minimize span of entries used and try to have lowest numbered
* proximity domain be node 0
*/
#define NODE_DOMAIN_HASH(domain, node_cnt) \
((lgrp_plat_prox_domain_min == UINT32_MAX) ? (domain) % node_cnt : \
((domain) - lgrp_plat_prox_domain_min) % node_cnt)
/*
* CPU to node ID mapping structure (only used with SRAT)
*/
typedef struct cpu_node_map {
int exists;
uint_t node;
uint32_t apicid;
uint32_t prox_domain;
} cpu_node_map_t;
/*
* Latency statistics
*/
typedef struct lgrp_plat_latency_stats {
hrtime_t latencies[MAX_NODES][MAX_NODES];
hrtime_t latency_max;
hrtime_t latency_min;
} lgrp_plat_latency_stats_t;
/*
* Memory configuration for probing
*/
typedef struct lgrp_plat_probe_mem_config {
size_t probe_memsize; /* how much memory to probe per node */
caddr_t probe_va[MAX_NODES]; /* where memory mapped for probing */
pfn_t probe_pfn[MAX_NODES]; /* physical pages to map for probing */
} lgrp_plat_probe_mem_config_t;
/*
* Statistics kept for probing
*/
typedef struct lgrp_plat_probe_stats {
hrtime_t flush_cost;
hrtime_t probe_cost;
hrtime_t probe_cost_total;
hrtime_t probe_error_code;
hrtime_t probe_errors[MAX_NODES][MAX_NODES];
int probe_suspect[MAX_NODES][MAX_NODES];
hrtime_t probe_max[MAX_NODES][MAX_NODES];
hrtime_t probe_min[MAX_NODES][MAX_NODES];
} lgrp_plat_probe_stats_t;
/*
* Node to proximity domain ID mapping structure (only used with SRAT)
*/
typedef struct node_domain_map {
int exists;
uint32_t prox_domain;
} node_domain_map_t;
/*
* Node ID and starting and ending page for physical memory in node
*/
typedef struct node_phys_addr_map {
pfn_t start;
pfn_t end;
int exists;
uint32_t prox_domain;
} node_phys_addr_map_t;
/*
* Number of CPUs for which we got APIC IDs
*/
static int lgrp_plat_apic_ncpus = 0;
/*
* CPU to node ID mapping table (only used for SRAT) and its max number of
* entries
*/
static cpu_node_map_t *lgrp_plat_cpu_node = NULL;
static uint_t lgrp_plat_cpu_node_nentries = 0;
/*
* Latency statistics
*/
lgrp_plat_latency_stats_t lgrp_plat_lat_stats;
/*
* Whether memory is interleaved across nodes causing MPO to be disabled
*/
static int lgrp_plat_mem_intrlv = 0;
/*
* Node ID to proximity domain ID mapping table (only used for SRAT)
*/
static node_domain_map_t lgrp_plat_node_domain[MAX_NODES];
/*
* Physical address range for memory in each node
*/
static node_phys_addr_map_t lgrp_plat_node_memory[MAX_NODES];
/*
* Statistics gotten from probing
*/
static lgrp_plat_probe_stats_t lgrp_plat_probe_stats;
/*
* Memory configuration for probing
*/
static lgrp_plat_probe_mem_config_t lgrp_plat_probe_mem_config;
/*
* Lowest proximity domain ID seen in ACPI SRAT
*/
static uint32_t lgrp_plat_prox_domain_min = UINT32_MAX;
/*
* Error code from processing ACPI SRAT
*/
static int lgrp_plat_srat_error = 0;
/*
* Error code from processing ACPI SLIT
*/
static int lgrp_plat_slit_error = 0;
/*
* Allocate lgroup array statically
*/
static lgrp_t lgrp_space[NLGRP];
static int nlgrps_alloc;
/*
* Enable finding and using minimum proximity domain ID when hashing
*/
int lgrp_plat_domain_min_enable = 1;
/*
* Number of nodes in system
*/
uint_t lgrp_plat_node_cnt = 1;
/*
* Enable sorting nodes in ascending order by starting physical address
*/
int lgrp_plat_node_sort_enable = 1;
/*
* Configuration Parameters for Probing
* - lgrp_plat_probe_flags Flags to specify enabling probing, probe
* operation, etc.
* - lgrp_plat_probe_nrounds How many rounds of probing to do
* - lgrp_plat_probe_nsamples Number of samples to take when probing each
* node
* - lgrp_plat_probe_nreads Number of times to read vendor ID from
* Northbridge for each probe
*/
uint_t lgrp_plat_probe_flags = 0;
int lgrp_plat_probe_nrounds = LGRP_PLAT_PROBE_NROUNDS;
int lgrp_plat_probe_nsamples = LGRP_PLAT_PROBE_NSAMPLES;
int lgrp_plat_probe_nreads = LGRP_PLAT_PROBE_NREADS;
/*
* Enable use of ACPI System Resource Affinity Table (SRAT) and System
* Locality Information Table (SLIT)
*/
int lgrp_plat_srat_enable = 1;
int lgrp_plat_slit_enable = 1;
/*
* mnode_xwa: set to non-zero value to initiate workaround if large pages are
* found to be crossing memory node boundaries. The workaround will eliminate
* a base size page at the end of each memory node boundary to ensure that
* a large page with constituent pages that span more than 1 memory node
* can never be formed.
*
*/
int mnode_xwa = 1;
/*
* Static array to hold lgroup statistics
*/
struct lgrp_stats lgrp_stats[NLGRP];
/*
* Forward declarations of platform interface routines
*/
void plat_build_mem_nodes(struct memlist *list);
int plat_lgrphand_to_mem_node(lgrp_handle_t hand);
lgrp_handle_t plat_mem_node_to_lgrphand(int mnode);
int plat_mnode_xcheck(pfn_t pfncnt);
int plat_pfn_to_mem_node(pfn_t pfn);
/*
* Forward declarations of lgroup platform interface routines
*/
lgrp_t *lgrp_plat_alloc(lgrp_id_t lgrpid);
void lgrp_plat_config(lgrp_config_flag_t flag, uintptr_t arg);
lgrp_handle_t lgrp_plat_cpu_to_hand(processorid_t id);
void lgrp_plat_init(lgrp_init_stages_t stage);
int lgrp_plat_latency(lgrp_handle_t from, lgrp_handle_t to);
int lgrp_plat_max_lgrps(void);
pgcnt_t lgrp_plat_mem_size(lgrp_handle_t plathand,
lgrp_mem_query_t query);
lgrp_handle_t lgrp_plat_pfn_to_hand(pfn_t pfn);
void lgrp_plat_probe(void);
lgrp_handle_t lgrp_plat_root_hand(void);
/*
* Forward declarations of local routines
*/
static int is_opteron(void);
static int lgrp_plat_cpu_node_update(node_domain_map_t *node_domain,
int node_cnt, cpu_node_map_t *cpu_node, int nentries, uint32_t apicid,
uint32_t domain);
static int lgrp_plat_cpu_to_node(cpu_t *cp, cpu_node_map_t *cpu_node,
int cpu_node_nentries);
static int lgrp_plat_domain_to_node(node_domain_map_t *node_domain,
int node_cnt, uint32_t domain);
static void lgrp_plat_get_numa_config(void);
static void lgrp_plat_latency_adjust(node_phys_addr_map_t *node_memory,
lgrp_plat_latency_stats_t *lat_stats,
lgrp_plat_probe_stats_t *probe_stats);
static int lgrp_plat_latency_verify(node_phys_addr_map_t *node_memory,
lgrp_plat_latency_stats_t *lat_stats);
static void lgrp_plat_main_init(void);
static pgcnt_t lgrp_plat_mem_size_default(lgrp_handle_t, lgrp_mem_query_t);
static int lgrp_plat_node_domain_update(node_domain_map_t *node_domain,
int node_cnt, uint32_t domain);
static int lgrp_plat_node_memory_update(node_domain_map_t *node_domain,
int node_cnt, node_phys_addr_map_t *node_memory, uint64_t start,
uint64_t end, uint32_t domain);
static void lgrp_plat_node_sort(node_domain_map_t *node_domain,
int node_cnt, cpu_node_map_t *cpu_node, int cpu_count,
node_phys_addr_map_t *node_memory);
static hrtime_t lgrp_plat_probe_time(int to, cpu_node_map_t *cpu_node,
int cpu_node_nentries, lgrp_plat_probe_mem_config_t *probe_mem_config,
lgrp_plat_latency_stats_t *lat_stats, lgrp_plat_probe_stats_t *probe_stats);
static int lgrp_plat_process_cpu_apicids(cpu_node_map_t *cpu_node);
static int lgrp_plat_process_slit(struct slit *tp, uint_t node_cnt,
node_phys_addr_map_t *node_memory, lgrp_plat_latency_stats_t *lat_stats);
static int lgrp_plat_process_srat(struct srat *tp,
uint32_t *prox_domain_min, node_domain_map_t *node_domain,
cpu_node_map_t *cpu_node, int cpu_count,
node_phys_addr_map_t *node_memory);
static void lgrp_plat_release_bootstrap(void);
static int lgrp_plat_srat_domains(struct srat *tp,
uint32_t *prox_domain_min);
static void lgrp_plat_2level_setup(node_phys_addr_map_t *node_memory,
lgrp_plat_latency_stats_t *lat_stats);
static void opt_get_numa_config(uint_t *node_cnt, int *mem_intrlv,
node_phys_addr_map_t *node_memory);
static hrtime_t opt_probe_vendor(int dest_node, int nreads);
/*
* PLATFORM INTERFACE ROUTINES
*/
/*
* Configure memory nodes for machines with more than one node (ie NUMA)
*/
void
plat_build_mem_nodes(struct memlist *list)
{
pfn_t cur_start; /* start addr of subrange */
pfn_t cur_end; /* end addr of subrange */
pfn_t start; /* start addr of whole range */
pfn_t end; /* end addr of whole range */
pgcnt_t endcnt; /* pages to sacrifice */
/*
* Boot install lists are arranged <addr, len>, ...
*/
while (list) {
int node;
start = list->ml_address >> PAGESHIFT;
end = (list->ml_address + list->ml_size - 1) >> PAGESHIFT;
if (start > physmax) {
list = list->ml_next;
continue;
}
if (end > physmax)
end = physmax;
/*
* When there is only one memnode, just add memory to memnode
*/
if (max_mem_nodes == 1) {
mem_node_add_slice(start, end);
list = list->ml_next;
continue;
}
/*
* mem_node_add_slice() expects to get a memory range that
* is within one memnode, so need to split any memory range
* that spans multiple memnodes into subranges that are each
* contained within one memnode when feeding them to
* mem_node_add_slice()
*/
cur_start = start;
do {
node = plat_pfn_to_mem_node(cur_start);
/*
* Panic if DRAM address map registers or SRAT say
* memory in node doesn't exist or address from
* boot installed memory list entry isn't in this node.
* This shouldn't happen and rest of code can't deal
* with this if it does.
*/
if (node < 0 || node >= lgrp_plat_node_cnt ||
!lgrp_plat_node_memory[node].exists ||
cur_start < lgrp_plat_node_memory[node].start ||
cur_start > lgrp_plat_node_memory[node].end) {
cmn_err(CE_PANIC, "Don't know which memnode "
"to add installed memory address 0x%lx\n",
cur_start);
}
/*
* End of current subrange should not span memnodes
*/
cur_end = end;
endcnt = 0;
if (lgrp_plat_node_memory[node].exists &&
cur_end > lgrp_plat_node_memory[node].end) {
cur_end = lgrp_plat_node_memory[node].end;
if (mnode_xwa > 1) {
/*
* sacrifice the last page in each
* node to eliminate large pages
* that span more than 1 memory node.
*/
endcnt = 1;
physinstalled--;
}
}
mem_node_add_slice(cur_start, cur_end - endcnt);
/*
* Next subrange starts after end of current one
*/
cur_start = cur_end + 1;
} while (cur_end < end);
list = list->ml_next;
}
mem_node_physalign = 0;
mem_node_pfn_shift = 0;
}
int
plat_lgrphand_to_mem_node(lgrp_handle_t hand)
{
if (max_mem_nodes == 1)
return (0);
return ((int)hand);
}
/*
* plat_mnode_xcheck: checks the node memory ranges to see if there is a pfncnt
* range of pages aligned on pfncnt that crosses an node boundary. Returns 1 if
* a crossing is found and returns 0 otherwise.
*/
int
plat_mnode_xcheck(pfn_t pfncnt)
{
int node, prevnode = -1, basenode;
pfn_t ea, sa;
for (node = 0; node < lgrp_plat_node_cnt; node++) {
if (lgrp_plat_node_memory[node].exists == 0)
continue;
if (prevnode == -1) {
prevnode = node;
basenode = node;
continue;
}
/* assume x86 node pfn ranges are in increasing order */
ASSERT(lgrp_plat_node_memory[node].start >
lgrp_plat_node_memory[prevnode].end);
/*
* continue if the starting address of node is not contiguous
* with the previous node.
*/
if (lgrp_plat_node_memory[node].start !=
(lgrp_plat_node_memory[prevnode].end + 1)) {
basenode = node;
prevnode = node;
continue;
}
/* check if the starting address of node is pfncnt aligned */
if ((lgrp_plat_node_memory[node].start & (pfncnt - 1)) != 0) {
/*
* at this point, node starts at an unaligned boundary
* and is contiguous with the previous node(s) to
* basenode. Check if there is an aligned contiguous
* range of length pfncnt that crosses this boundary.
*/
sa = P2ALIGN(lgrp_plat_node_memory[prevnode].end,
pfncnt);
ea = P2ROUNDUP((lgrp_plat_node_memory[node].start),
pfncnt);
ASSERT((ea - sa) == pfncnt);
if (sa >= lgrp_plat_node_memory[basenode].start &&
ea <= (lgrp_plat_node_memory[node].end + 1)) {
/*
* large page found to cross mnode boundary.
* Return Failure if workaround not enabled.
*/
if (mnode_xwa == 0)
return (1);
mnode_xwa++;
}
}
prevnode = node;
}
return (0);
}
lgrp_handle_t
plat_mem_node_to_lgrphand(int mnode)
{
if (max_mem_nodes == 1)
return (LGRP_DEFAULT_HANDLE);
return ((lgrp_handle_t)mnode);
}
int
plat_pfn_to_mem_node(pfn_t pfn)
{
int node;
if (max_mem_nodes == 1)
return (0);
for (node = 0; node < lgrp_plat_node_cnt; node++) {
/*
* Skip nodes with no memory
*/
if (!lgrp_plat_node_memory[node].exists)
continue;
if (pfn >= lgrp_plat_node_memory[node].start &&
pfn <= lgrp_plat_node_memory[node].end)
return (node);
}
/*
* Didn't find memnode where this PFN lives which should never happen
*/
ASSERT(node < lgrp_plat_node_cnt);
return (-1);
}
/*
* LGROUP PLATFORM INTERFACE ROUTINES
*/
/*
* Allocate additional space for an lgroup.
*/
/* ARGSUSED */
lgrp_t *
lgrp_plat_alloc(lgrp_id_t lgrpid)
{
lgrp_t *lgrp;
lgrp = &lgrp_space[nlgrps_alloc++];
if (lgrpid >= NLGRP || nlgrps_alloc > NLGRP)
return (NULL);
return (lgrp);
}
/*
* Platform handling for (re)configuration changes
*/
/* ARGSUSED */
void
lgrp_plat_config(lgrp_config_flag_t flag, uintptr_t arg)
{
}
/*
* Return the platform handle for the lgroup containing the given CPU
*/
/* ARGSUSED */
lgrp_handle_t
lgrp_plat_cpu_to_hand(processorid_t id)
{
lgrp_handle_t hand;
if (lgrp_plat_node_cnt == 1)
return (LGRP_DEFAULT_HANDLE);
hand = (lgrp_handle_t)lgrp_plat_cpu_to_node(cpu[id],
lgrp_plat_cpu_node, lgrp_plat_cpu_node_nentries);
ASSERT(hand != (lgrp_handle_t)-1);
if (hand == (lgrp_handle_t)-1)
return (LGRP_NULL_HANDLE);
return (hand);
}
/*
* Platform-specific initialization of lgroups
*/
void
lgrp_plat_init(lgrp_init_stages_t stage)
{
#if defined(__xpv)
#else /* __xpv */
u_longlong_t value;
#endif /* __xpv */
switch (stage) {
case LGRP_INIT_STAGE1:
#if defined(__xpv)
/*
* XXPV For now, the hypervisor treats all memory equally.
*/
lgrp_plat_node_cnt = max_mem_nodes = 1;
#else /* __xpv */
/*
* Get boot property for lgroup topology height limit
*/
if (bootprop_getval(BP_LGRP_TOPO_LEVELS, &value) == 0)
(void) lgrp_topo_ht_limit_set((int)value);
/*
* Get boot property for enabling/disabling SRAT
*/
if (bootprop_getval(BP_LGRP_SRAT_ENABLE, &value) == 0)
lgrp_plat_srat_enable = (int)value;
/*
* Get boot property for enabling/disabling SLIT
*/
if (bootprop_getval(BP_LGRP_SLIT_ENABLE, &value) == 0)
lgrp_plat_slit_enable = (int)value;
/*
* Initialize as a UMA machine
*/
if (lgrp_topo_ht_limit() == 1) {
lgrp_plat_node_cnt = max_mem_nodes = 1;
return;
}
lgrp_plat_get_numa_config();
#endif /* __xpv */
break;
case LGRP_INIT_STAGE3:
lgrp_plat_probe();
lgrp_plat_release_bootstrap();
break;
case LGRP_INIT_STAGE4:
lgrp_plat_main_init();
break;
default:
break;
}
}
/*
* Return latency between "from" and "to" lgroups
*
* This latency number can only be used for relative comparison
* between lgroups on the running system, cannot be used across platforms,
* and may not reflect the actual latency. It is platform and implementation
* specific, so platform gets to decide its value. It would be nice if the
* number was at least proportional to make comparisons more meaningful though.
*/
/* ARGSUSED */
int
lgrp_plat_latency(lgrp_handle_t from, lgrp_handle_t to)
{
lgrp_handle_t src, dest;
int node;
if (max_mem_nodes == 1)
return (0);
/*
* Return max latency for root lgroup
*/
if (from == LGRP_DEFAULT_HANDLE || to == LGRP_DEFAULT_HANDLE)
return (lgrp_plat_lat_stats.latency_max);
src = from;
dest = to;
/*
* Return 0 for nodes (lgroup platform handles) out of range
*/
if (src < 0 || src >= MAX_NODES || dest < 0 || dest >= MAX_NODES)
return (0);
/*
* Probe from current CPU if its lgroup latencies haven't been set yet
* and we are trying to get latency from current CPU to some node
*/
node = lgrp_plat_cpu_to_node(CPU, lgrp_plat_cpu_node,
lgrp_plat_cpu_node_nentries);
ASSERT(node >= 0 && node < lgrp_plat_node_cnt);
if (lgrp_plat_lat_stats.latencies[src][src] == 0 && node == src)
lgrp_plat_probe();
return (lgrp_plat_lat_stats.latencies[src][dest]);
}
/*
* Return the maximum number of lgrps supported by the platform.
* Before lgrp topology is known it returns an estimate based on the number of
* nodes. Once topology is known it returns the actual maximim number of lgrps
* created. Since x86/x64 doesn't support Dynamic Reconfiguration (DR) and
* dynamic addition of new nodes, this number may not grow during system
* lifetime (yet).
*/
int
lgrp_plat_max_lgrps(void)
{
return (lgrp_topo_initialized ?
lgrp_alloc_max + 1 :
lgrp_plat_node_cnt * (lgrp_plat_node_cnt - 1) + 1);
}
/*
* Return the number of free pages in an lgroup.
*
* For query of LGRP_MEM_SIZE_FREE, return the number of base pagesize
* pages on freelists. For query of LGRP_MEM_SIZE_AVAIL, return the
* number of allocatable base pagesize pages corresponding to the
* lgroup (e.g. do not include page_t's, BOP_ALLOC()'ed memory, ..)
* For query of LGRP_MEM_SIZE_INSTALL, return the amount of physical
* memory installed, regardless of whether or not it's usable.
*/
pgcnt_t
lgrp_plat_mem_size(lgrp_handle_t plathand, lgrp_mem_query_t query)
{
int mnode;
pgcnt_t npgs = (pgcnt_t)0;
extern struct memlist *phys_avail;
extern struct memlist *phys_install;
if (plathand == LGRP_DEFAULT_HANDLE)
return (lgrp_plat_mem_size_default(plathand, query));
if (plathand != LGRP_NULL_HANDLE) {
mnode = plat_lgrphand_to_mem_node(plathand);
if (mnode >= 0 && mem_node_config[mnode].exists) {
switch (query) {
case LGRP_MEM_SIZE_FREE:
npgs = MNODE_PGCNT(mnode);
break;
case LGRP_MEM_SIZE_AVAIL:
npgs = mem_node_memlist_pages(mnode,
phys_avail);
break;
case LGRP_MEM_SIZE_INSTALL:
npgs = mem_node_memlist_pages(mnode,
phys_install);
break;
default:
break;
}
}
}
return (npgs);
}
/*
* Return the platform handle of the lgroup that contains the physical memory
* corresponding to the given page frame number
*/
/* ARGSUSED */
lgrp_handle_t
lgrp_plat_pfn_to_hand(pfn_t pfn)
{
int mnode;
if (max_mem_nodes == 1)
return (LGRP_DEFAULT_HANDLE);
if (pfn > physmax)
return (LGRP_NULL_HANDLE);
mnode = plat_pfn_to_mem_node(pfn);
if (mnode < 0)
return (LGRP_NULL_HANDLE);
return (MEM_NODE_2_LGRPHAND(mnode));
}
/*
* Probe memory in each node from current CPU to determine latency topology
*
* The probing code will probe the vendor ID register on the Northbridge of
* Opteron processors and probe memory for other processors by default.
*
* Since probing is inherently error prone, the code takes laps across all the
* nodes probing from each node to each of the other nodes some number of
* times. Furthermore, each node is probed some number of times before moving
* onto the next one during each lap. The minimum latency gotten between nodes
* is kept as the latency between the nodes.
*
* After all that, the probe times are adjusted by normalizing values that are
* close to each other and local latencies are made the same. Lastly, the
* latencies are verified to make sure that certain conditions are met (eg.
* local < remote, latency(a, b) == latency(b, a), etc.).
*
* If any of the conditions aren't met, the code will export a NUMA
* configuration with the local CPUs and memory given by the SRAT or PCI config
* space registers and one remote memory latency since it can't tell exactly
* how far each node is from each other.
*/
void
lgrp_plat_probe(void)
{
int from;
int i;
lgrp_plat_latency_stats_t *lat_stats;
boolean_t probed;
hrtime_t probe_time;
int to;
if (!(lgrp_plat_probe_flags & LGRP_PLAT_PROBE_ENABLE) ||
max_mem_nodes == 1 || lgrp_topo_ht_limit() <= 2)
return;
/*
* Determine ID of node containing current CPU
*/
from = lgrp_plat_cpu_to_node(CPU, lgrp_plat_cpu_node,
lgrp_plat_cpu_node_nentries);
ASSERT(from >= 0 && from < lgrp_plat_node_cnt);
if (srat_ptr && lgrp_plat_srat_enable && !lgrp_plat_srat_error)
ASSERT(lgrp_plat_node_domain[from].exists);
/*
* Don't need to probe if got times already
*/
lat_stats = &lgrp_plat_lat_stats;
if (lat_stats->latencies[from][from] != 0)
return;
/*
* Read vendor ID in Northbridge or read and write page(s)
* in each node from current CPU and remember how long it takes,
* so we can build latency topology of machine later.
* This should approximate the memory latency between each node.
*/
probed = B_FALSE;
for (i = 0; i < lgrp_plat_probe_nrounds; i++) {
for (to = 0; to < lgrp_plat_node_cnt; to++) {
/*
* Get probe time and skip over any nodes that can't be
* probed yet or don't have memory
*/
probe_time = lgrp_plat_probe_time(to,
lgrp_plat_cpu_node, lgrp_plat_cpu_node_nentries,
&lgrp_plat_probe_mem_config, &lgrp_plat_lat_stats,
&lgrp_plat_probe_stats);
if (probe_time == 0)
continue;
probed = B_TRUE;
/*
* Keep lowest probe time as latency between nodes
*/
if (lat_stats->latencies[from][to] == 0 ||
probe_time < lat_stats->latencies[from][to])
lat_stats->latencies[from][to] = probe_time;
/*
* Update overall minimum and maximum probe times
* across all nodes
*/
if (probe_time < lat_stats->latency_min ||
lat_stats->latency_min == -1)
lat_stats->latency_min = probe_time;
if (probe_time > lat_stats->latency_max)
lat_stats->latency_max = probe_time;
}
}
/*
* Bail out if weren't able to probe any nodes from current CPU
*/
if (probed == B_FALSE)
return;
/*
* - Fix up latencies such that local latencies are same,
* latency(i, j) == latency(j, i), etc. (if possible)
*
* - Verify that latencies look ok
*
* - Fallback to just optimizing for local and remote if
* latencies didn't look right
*/
lgrp_plat_latency_adjust(lgrp_plat_node_memory, &lgrp_plat_lat_stats,
&lgrp_plat_probe_stats);
lgrp_plat_probe_stats.probe_error_code =
lgrp_plat_latency_verify(lgrp_plat_node_memory,
&lgrp_plat_lat_stats);
if (lgrp_plat_probe_stats.probe_error_code)
lgrp_plat_2level_setup(lgrp_plat_node_memory,
&lgrp_plat_lat_stats);
}
/*
* Return platform handle for root lgroup
*/
lgrp_handle_t
lgrp_plat_root_hand(void)
{
return (LGRP_DEFAULT_HANDLE);
}
/*
* INTERNAL ROUTINES
*/
/*
* Update CPU to node mapping for given CPU and proximity domain (and returns
* negative numbers for errors and positive ones for success)
*/
static int
lgrp_plat_cpu_node_update(node_domain_map_t *node_domain, int node_cnt,
cpu_node_map_t *cpu_node, int nentries, uint32_t apicid, uint32_t domain)
{
uint_t i;
int node;
/*
* Get node number for proximity domain
*/
node = lgrp_plat_domain_to_node(node_domain, node_cnt, domain);
if (node == -1) {
node = lgrp_plat_node_domain_update(node_domain, node_cnt,
domain);
if (node == -1)
return (-1);
}
/*
* Search for entry with given APIC ID and fill in its node and
* proximity domain IDs (if they haven't been set already)
*/
for (i = 0; i < nentries; i++) {
/*
* Skip nonexistent entries and ones without matching APIC ID
*/
if (!cpu_node[i].exists || cpu_node[i].apicid != apicid)
continue;
/*
* Just return if entry completely and correctly filled in
* already
*/
if (cpu_node[i].prox_domain == domain &&
cpu_node[i].node == node)
return (1);
/*
* Fill in node and proximity domain IDs
*/
cpu_node[i].prox_domain = domain;
cpu_node[i].node = node;
return (0);
}
/*
* Return error when entry for APIC ID wasn't found in table
*/
return (-2);
}
/*
* Get node ID for given CPU
*/
static int
lgrp_plat_cpu_to_node(cpu_t *cp, cpu_node_map_t *cpu_node,
int cpu_node_nentries)
{
processorid_t cpuid;
if (cp == NULL)
return (-1);
cpuid = cp->cpu_id;
if (cpuid < 0 || cpuid >= max_ncpus)
return (-1);
/*
* SRAT doesn't exist, isn't enabled, or there was an error processing
* it, so return node ID for Opteron and -1 otherwise.
*/
if (srat_ptr == NULL || !lgrp_plat_srat_enable ||
lgrp_plat_srat_error) {
if (is_opteron())
return (pg_plat_hw_instance_id(cp, PGHW_PROCNODE));
return (-1);
}
/*
* Return -1 when CPU to node ID mapping entry doesn't exist for given
* CPU
*/
if (cpuid >= cpu_node_nentries || !cpu_node[cpuid].exists)
return (-1);
return (cpu_node[cpuid].node);
}
/*
* Return node number for given proximity domain/system locality
*/
static int
lgrp_plat_domain_to_node(node_domain_map_t *node_domain, int node_cnt,
uint32_t domain)
{
uint_t node;
uint_t start;
/*
* Hash proximity domain ID into node to domain mapping table (array),
* search for entry with matching proximity domain ID, and return index
* of matching entry as node ID.
*/
node = start = NODE_DOMAIN_HASH(domain, node_cnt);
do {
if (node_domain[node].prox_domain == domain &&
node_domain[node].exists)
return (node);
node = (node + 1) % node_cnt;
} while (node != start);
return (-1);
}
/*
* Get NUMA configuration of machine
*/
static void
lgrp_plat_get_numa_config(void)
{
uint_t probe_op;
/*
* Read boot property with CPU to APIC ID mapping table/array to
* determine number of CPUs
*/
lgrp_plat_apic_ncpus = lgrp_plat_process_cpu_apicids(NULL);
/*
* Determine which CPUs and memory are local to each other and number
* of NUMA nodes by reading ACPI System Resource Affinity Table (SRAT)
*/
if (lgrp_plat_apic_ncpus > 0) {
int retval;
/*
* Temporarily allocate boot memory to use for CPU to node
* mapping since kernel memory allocator isn't alive yet
*/
lgrp_plat_cpu_node = (cpu_node_map_t *)BOP_ALLOC(bootops,
NULL, lgrp_plat_apic_ncpus * sizeof (cpu_node_map_t),
sizeof (int));
ASSERT(lgrp_plat_cpu_node != NULL);
if (lgrp_plat_cpu_node) {
lgrp_plat_cpu_node_nentries = lgrp_plat_apic_ncpus;
bzero(lgrp_plat_cpu_node, lgrp_plat_cpu_node_nentries *
sizeof (cpu_node_map_t));
}
/*
* Fill in CPU to node ID mapping table with APIC ID for each
* CPU
*/
(void) lgrp_plat_process_cpu_apicids(lgrp_plat_cpu_node);
retval = lgrp_plat_process_srat(srat_ptr,
&lgrp_plat_prox_domain_min,
lgrp_plat_node_domain, lgrp_plat_cpu_node,
lgrp_plat_apic_ncpus, lgrp_plat_node_memory);
if (retval <= 0) {
lgrp_plat_srat_error = retval;
lgrp_plat_node_cnt = 1;
} else {
lgrp_plat_srat_error = 0;
lgrp_plat_node_cnt = retval;
}
}
/*
* Try to use PCI config space registers on Opteron if there's an error
* processing CPU to APIC ID mapping or SRAT
*/
if ((lgrp_plat_apic_ncpus <= 0 || lgrp_plat_srat_error != 0) &&
is_opteron())
opt_get_numa_config(&lgrp_plat_node_cnt, &lgrp_plat_mem_intrlv,
lgrp_plat_node_memory);
/*
* Don't bother to setup system for multiple lgroups and only use one
* memory node when memory is interleaved between any nodes or there is
* only one NUMA node
*
* NOTE: May need to change this for Dynamic Reconfiguration (DR)
* when and if it happens for x86/x64
*/
if (lgrp_plat_mem_intrlv || lgrp_plat_node_cnt == 1) {
lgrp_plat_node_cnt = max_mem_nodes = 1;
(void) lgrp_topo_ht_limit_set(1);
return;
}
/*
* Leaf lgroups on x86/x64 architectures contain one physical
* processor chip. Tune lgrp_expand_proc_thresh and
* lgrp_expand_proc_diff so that lgrp_choose() will spread
* things out aggressively.
*/
lgrp_expand_proc_thresh = LGRP_LOADAVG_THREAD_MAX / 2;
lgrp_expand_proc_diff = 0;
/*
* There should be one memnode (physical page free list(s)) for
* each node
*/
max_mem_nodes = lgrp_plat_node_cnt;
/*
* Initialize min and max latency before reading SLIT or probing
*/
lgrp_plat_lat_stats.latency_min = -1;
lgrp_plat_lat_stats.latency_max = 0;
/*
* Determine how far each NUMA node is from each other by
* reading ACPI System Locality Information Table (SLIT) if it
* exists
*/
lgrp_plat_slit_error = lgrp_plat_process_slit(slit_ptr,
lgrp_plat_node_cnt, lgrp_plat_node_memory,
&lgrp_plat_lat_stats);
if (lgrp_plat_slit_error == 0)
return;
/*
* Probe to determine latency between NUMA nodes when SLIT
* doesn't exist or make sense
*/
lgrp_plat_probe_flags |= LGRP_PLAT_PROBE_ENABLE;
/*
* Specify whether to probe using vendor ID register or page copy
* if hasn't been specified already or is overspecified
*/
probe_op = lgrp_plat_probe_flags &
(LGRP_PLAT_PROBE_PGCPY|LGRP_PLAT_PROBE_VENDOR);
if (probe_op == 0 ||
probe_op == (LGRP_PLAT_PROBE_PGCPY|LGRP_PLAT_PROBE_VENDOR)) {
lgrp_plat_probe_flags &=
~(LGRP_PLAT_PROBE_PGCPY|LGRP_PLAT_PROBE_VENDOR);
if (is_opteron())
lgrp_plat_probe_flags |=
LGRP_PLAT_PROBE_VENDOR;
else
lgrp_plat_probe_flags |= LGRP_PLAT_PROBE_PGCPY;
}
/*
* Probing errors can mess up the lgroup topology and
* force us fall back to a 2 level lgroup topology.
* Here we bound how tall the lgroup topology can grow
* in hopes of avoiding any anamolies in probing from
* messing up the lgroup topology by limiting the
* accuracy of the latency topology.
*
* Assume that nodes will at least be configured in a
* ring, so limit height of lgroup topology to be less
* than number of nodes on a system with 4 or more
* nodes
*/
if (lgrp_plat_node_cnt >= 4 && lgrp_topo_ht_limit() ==
lgrp_topo_ht_limit_default())
(void) lgrp_topo_ht_limit_set(lgrp_plat_node_cnt - 1);
}
/*
* Latencies must be within 1/(2**LGRP_LAT_TOLERANCE_SHIFT) of each other to
* be considered same
*/
#define LGRP_LAT_TOLERANCE_SHIFT 4
int lgrp_plat_probe_lt_shift = LGRP_LAT_TOLERANCE_SHIFT;
/*
* Adjust latencies between nodes to be symmetric, normalize latencies between
* any nodes that are within some tolerance to be same, and make local
* latencies be same
*/
static void
lgrp_plat_latency_adjust(node_phys_addr_map_t *node_memory,
lgrp_plat_latency_stats_t *lat_stats, lgrp_plat_probe_stats_t *probe_stats)
{
int i;
int j;
int k;
int l;
u_longlong_t max;
u_longlong_t min;
u_longlong_t t;
u_longlong_t t1;
u_longlong_t t2;
const lgrp_config_flag_t cflag = LGRP_CONFIG_LAT_CHANGE_ALL;
int lat_corrected[MAX_NODES][MAX_NODES];
/*
* Nothing to do when this is an UMA machine or don't have args needed
*/
if (max_mem_nodes == 1)
return;
ASSERT(node_memory != NULL && lat_stats != NULL &&
probe_stats != NULL);
/*
* Make sure that latencies are symmetric between any two nodes
* (ie. latency(node0, node1) == latency(node1, node0))
*/
for (i = 0; i < lgrp_plat_node_cnt; i++) {
if (!node_memory[i].exists)
continue;
for (j = 0; j < lgrp_plat_node_cnt; j++) {
if (!node_memory[j].exists)
continue;
t1 = lat_stats->latencies[i][j];
t2 = lat_stats->latencies[j][i];
if (t1 == 0 || t2 == 0 || t1 == t2)
continue;
/*
* Latencies should be same
* - Use minimum of two latencies which should be same
* - Track suspect probe times not within tolerance of
* min value
* - Remember how much values are corrected by
*/
if (t1 > t2) {
t = t2;
probe_stats->probe_errors[i][j] += t1 - t2;
if (t1 - t2 > t2 >> lgrp_plat_probe_lt_shift) {
probe_stats->probe_suspect[i][j]++;
probe_stats->probe_suspect[j][i]++;
}
} else if (t2 > t1) {
t = t1;
probe_stats->probe_errors[j][i] += t2 - t1;
if (t2 - t1 > t1 >> lgrp_plat_probe_lt_shift) {
probe_stats->probe_suspect[i][j]++;
probe_stats->probe_suspect[j][i]++;
}
}
lat_stats->latencies[i][j] =
lat_stats->latencies[j][i] = t;
lgrp_config(cflag, t1, t);
lgrp_config(cflag, t2, t);
}
}
/*
* Keep track of which latencies get corrected
*/
for (i = 0; i < MAX_NODES; i++)
for (j = 0; j < MAX_NODES; j++)
lat_corrected[i][j] = 0;
/*
* For every two nodes, see whether there is another pair of nodes which
* are about the same distance apart and make the latencies be the same
* if they are close enough together
*/
for (i = 0; i < lgrp_plat_node_cnt; i++) {
for (j = 0; j < lgrp_plat_node_cnt; j++) {
if (!node_memory[j].exists)
continue;
/*
* Pick one pair of nodes (i, j)
* and get latency between them
*/
t1 = lat_stats->latencies[i][j];
/*
* Skip this pair of nodes if there isn't a latency
* for it yet
*/
if (t1 == 0)
continue;
for (k = 0; k < lgrp_plat_node_cnt; k++) {
for (l = 0; l < lgrp_plat_node_cnt; l++) {
if (!node_memory[l].exists)
continue;
/*
* Pick another pair of nodes (k, l)
* not same as (i, j) and get latency
* between them
*/
if (k == i && l == j)
continue;
t2 = lat_stats->latencies[k][l];
/*
* Skip this pair of nodes if there
* isn't a latency for it yet
*/
if (t2 == 0)
continue;
/*
* Skip nodes (k, l) if they already
* have same latency as (i, j) or
* their latency isn't close enough to
* be considered/made the same
*/
if (t1 == t2 || (t1 > t2 && t1 - t2 >
t1 >> lgrp_plat_probe_lt_shift) ||
(t2 > t1 && t2 - t1 >
t2 >> lgrp_plat_probe_lt_shift))
continue;
/*
* Make latency(i, j) same as
* latency(k, l), try to use latency
* that has been adjusted already to get
* more consistency (if possible), and
* remember which latencies were
* adjusted for next time
*/
if (lat_corrected[i][j]) {
t = t1;
lgrp_config(cflag, t2, t);
t2 = t;
} else if (lat_corrected[k][l]) {
t = t2;
lgrp_config(cflag, t1, t);
t1 = t;
} else {
if (t1 > t2)
t = t2;
else
t = t1;
lgrp_config(cflag, t1, t);
lgrp_config(cflag, t2, t);
t1 = t2 = t;
}
lat_stats->latencies[i][j] =
lat_stats->latencies[k][l] = t;
lat_corrected[i][j] =
lat_corrected[k][l] = 1;
}
}
}
}
/*
* Local latencies should be same
* - Find min and max local latencies
* - Make all local latencies be minimum
*/
min = -1;
max = 0;
for (i = 0; i < lgrp_plat_node_cnt; i++) {
if (!node_memory[i].exists)
continue;
t = lat_stats->latencies[i][i];
if (t == 0)
continue;
if (min == -1 || t < min)
min = t;
if (t > max)
max = t;
}
if (min != max) {
for (i = 0; i < lgrp_plat_node_cnt; i++) {
int local;
if (!node_memory[i].exists)
continue;
local = lat_stats->latencies[i][i];
if (local == 0)
continue;
/*
* Track suspect probe times that aren't within
* tolerance of minimum local latency and how much
* probe times are corrected by
*/
if (local - min > min >> lgrp_plat_probe_lt_shift)
probe_stats->probe_suspect[i][i]++;
probe_stats->probe_errors[i][i] += local - min;
/*
* Make local latencies be minimum
*/
lgrp_config(LGRP_CONFIG_LAT_CHANGE, i, min);
lat_stats->latencies[i][i] = min;
}
}
/*
* Determine max probe time again since just adjusted latencies
*/
lat_stats->latency_max = 0;
for (i = 0; i < lgrp_plat_node_cnt; i++) {
for (j = 0; j < lgrp_plat_node_cnt; j++) {
if (!node_memory[j].exists)
continue;
t = lat_stats->latencies[i][j];
if (t > lat_stats->latency_max)
lat_stats->latency_max = t;
}
}
}
/*
* Verify following about latencies between nodes:
*
* - Latencies should be symmetric (ie. latency(a, b) == latency(b, a))
* - Local latencies same
* - Local < remote
* - Number of latencies seen is reasonable
* - Number of occurrences of a given latency should be more than 1
*
* Returns:
* 0 Success
* -1 Not symmetric
* -2 Local latencies not same
* -3 Local >= remote
*/
static int
lgrp_plat_latency_verify(node_phys_addr_map_t *node_memory,
lgrp_plat_latency_stats_t *lat_stats)
{
int i;
int j;
u_longlong_t t1;
u_longlong_t t2;
ASSERT(node_memory != NULL && lat_stats != NULL);
/*
* Nothing to do when this is an UMA machine, lgroup topology is
* limited to 2 levels, or there aren't any probe times yet
*/
if (max_mem_nodes == 1 || lgrp_topo_levels < 2 ||
lat_stats->latencies[0][0] == 0)
return (0);
/*
* Make sure that latencies are symmetric between any two nodes
* (ie. latency(node0, node1) == latency(node1, node0))
*/
for (i = 0; i < lgrp_plat_node_cnt; i++) {
if (!node_memory[i].exists)
continue;
for (j = 0; j < lgrp_plat_node_cnt; j++) {
if (!node_memory[j].exists)
continue;
t1 = lat_stats->latencies[i][j];
t2 = lat_stats->latencies[j][i];
if (t1 == 0 || t2 == 0 || t1 == t2)
continue;
return (-1);
}
}
/*
* Local latencies should be same
*/
t1 = lat_stats->latencies[0][0];
for (i = 1; i < lgrp_plat_node_cnt; i++) {
if (!node_memory[i].exists)
continue;
t2 = lat_stats->latencies[i][i];
if (t2 == 0)
continue;
if (t1 == 0) {
t1 = t2;
continue;
}
if (t1 != t2)
return (-2);
}
/*
* Local latencies should be less than remote
*/
if (t1) {
for (i = 0; i < lgrp_plat_node_cnt; i++) {
for (j = 0; j < lgrp_plat_node_cnt; j++) {
if (!node_memory[j].exists)
continue;
t2 = lat_stats->latencies[i][j];
if (i == j || t2 == 0)
continue;
if (t1 >= t2)
return (-3);
}
}
}
return (0);
}
/*
* Platform-specific initialization
*/
static void
lgrp_plat_main_init(void)
{
int curnode;
int ht_limit;
int i;
/*
* Print a notice that MPO is disabled when memory is interleaved
* across nodes....Would do this when it is discovered, but can't
* because it happens way too early during boot....
*/
if (lgrp_plat_mem_intrlv)
cmn_err(CE_NOTE,
"MPO disabled because memory is interleaved\n");
/*
* Don't bother to do any probing if it is disabled, there is only one
* node, or the height of the lgroup topology less than or equal to 2
*/
ht_limit = lgrp_topo_ht_limit();
if (!(lgrp_plat_probe_flags & LGRP_PLAT_PROBE_ENABLE) ||
max_mem_nodes == 1 || ht_limit <= 2) {
/*
* Setup lgroup latencies for 2 level lgroup topology
* (ie. local and remote only) if they haven't been set yet
*/
if (ht_limit == 2 && lgrp_plat_lat_stats.latency_min == -1 &&
lgrp_plat_lat_stats.latency_max == 0)
lgrp_plat_2level_setup(lgrp_plat_node_memory,
&lgrp_plat_lat_stats);
return;
}
if (lgrp_plat_probe_flags & LGRP_PLAT_PROBE_VENDOR) {
/*
* Should have been able to probe from CPU 0 when it was added
* to lgroup hierarchy, but may not have been able to then
* because it happens so early in boot that gethrtime() hasn't
* been initialized. (:-(
*/
curnode = lgrp_plat_cpu_to_node(CPU, lgrp_plat_cpu_node,
lgrp_plat_cpu_node_nentries);
ASSERT(curnode >= 0 && curnode < lgrp_plat_node_cnt);
if (lgrp_plat_lat_stats.latencies[curnode][curnode] == 0)
lgrp_plat_probe();
return;
}
/*
* When probing memory, use one page for every sample to determine
* lgroup topology and taking multiple samples
*/
if (lgrp_plat_probe_mem_config.probe_memsize == 0)
lgrp_plat_probe_mem_config.probe_memsize = PAGESIZE *
lgrp_plat_probe_nsamples;
/*
* Map memory in each node needed for probing to determine latency
* topology
*/
for (i = 0; i < lgrp_plat_node_cnt; i++) {
int mnode;
/*
* Skip this node and leave its probe page NULL
* if it doesn't have any memory
*/
mnode = plat_lgrphand_to_mem_node((lgrp_handle_t)i);
if (!mem_node_config[mnode].exists) {
lgrp_plat_probe_mem_config.probe_va[i] = NULL;
continue;
}
/*
* Allocate one kernel virtual page
*/
lgrp_plat_probe_mem_config.probe_va[i] = vmem_alloc(heap_arena,
lgrp_plat_probe_mem_config.probe_memsize, VM_NOSLEEP);
if (lgrp_plat_probe_mem_config.probe_va[i] == NULL) {
cmn_err(CE_WARN,
"lgrp_plat_main_init: couldn't allocate memory");
return;
}
/*
* Get PFN for first page in each node
*/
lgrp_plat_probe_mem_config.probe_pfn[i] =
mem_node_config[mnode].physbase;
/*
* Map virtual page to first page in node
*/
hat_devload(kas.a_hat, lgrp_plat_probe_mem_config.probe_va[i],
lgrp_plat_probe_mem_config.probe_memsize,
lgrp_plat_probe_mem_config.probe_pfn[i],
PROT_READ | PROT_WRITE | HAT_PLAT_NOCACHE,
HAT_LOAD_NOCONSIST);
}
/*
* Probe from current CPU
*/
lgrp_plat_probe();
}
/*
* Return the number of free, allocatable, or installed
* pages in an lgroup
* This is a copy of the MAX_MEM_NODES == 1 version of the routine
* used when MPO is disabled (i.e. single lgroup) or this is the root lgroup
*/
/* ARGSUSED */
static pgcnt_t
lgrp_plat_mem_size_default(lgrp_handle_t lgrphand, lgrp_mem_query_t query)
{
struct memlist *mlist;
pgcnt_t npgs = 0;
extern struct memlist *phys_avail;
extern struct memlist *phys_install;
switch (query) {
case LGRP_MEM_SIZE_FREE:
return ((pgcnt_t)freemem);
case LGRP_MEM_SIZE_AVAIL:
memlist_read_lock();
for (mlist = phys_avail; mlist; mlist = mlist->ml_next)
npgs += btop(mlist->ml_size);
memlist_read_unlock();
return (npgs);
case LGRP_MEM_SIZE_INSTALL:
memlist_read_lock();
for (mlist = phys_install; mlist; mlist = mlist->ml_next)
npgs += btop(mlist->ml_size);
memlist_read_unlock();
return (npgs);
default:
return ((pgcnt_t)0);
}
}
/*
* Update node to proximity domain mappings for given domain and return node ID
*/
static int
lgrp_plat_node_domain_update(node_domain_map_t *node_domain, int node_cnt,
uint32_t domain)
{
uint_t node;
uint_t start;
/*
* Hash proximity domain ID into node to domain mapping table (array)
* and add entry for it into first non-existent or matching entry found
*/
node = start = NODE_DOMAIN_HASH(domain, node_cnt);
do {
/*
* Entry doesn't exist yet, so create one for this proximity
* domain and return node ID which is index into mapping table.
*/
if (!node_domain[node].exists) {
node_domain[node].exists = 1;
node_domain[node].prox_domain = domain;
return (node);
}
/*
* Entry exists for this proximity domain already, so just
* return node ID (index into table).
*/
if (node_domain[node].prox_domain == domain)
return (node);
node = NODE_DOMAIN_HASH(node + 1, node_cnt);
} while (node != start);
/*
* Ran out of supported number of entries which shouldn't happen....
*/
ASSERT(node != start);
return (-1);
}
/*
* Update node memory information for given proximity domain with specified
* starting and ending physical address range (and return positive numbers for
* success and negative ones for errors)
*/
static int
lgrp_plat_node_memory_update(node_domain_map_t *node_domain, int node_cnt,
node_phys_addr_map_t *node_memory, uint64_t start, uint64_t end,
uint32_t domain)
{
int node;
/*
* Get node number for proximity domain
*/
node = lgrp_plat_domain_to_node(node_domain, node_cnt, domain);
if (node == -1) {
node = lgrp_plat_node_domain_update(node_domain, node_cnt,
domain);
if (node == -1)
return (-1);
}
/*
* Create entry in table for node if it doesn't exist
*/
if (!node_memory[node].exists) {
node_memory[node].exists = 1;
node_memory[node].start = btop(start);
node_memory[node].end = btop(end);
node_memory[node].prox_domain = domain;
return (0);
}
/*
* Entry already exists for this proximity domain
*
* There may be more than one SRAT memory entry for a domain, so we may
* need to update existing start or end address for the node.
*/
if (node_memory[node].prox_domain == domain) {
if (btop(start) < node_memory[node].start)
node_memory[node].start = btop(start);
if (btop(end) > node_memory[node].end)
node_memory[node].end = btop(end);
return (1);
}
return (-2);
}
/*
* Have to sort node by starting physical address because VM system (physical
* page free list management) assumes and expects memnodes to be sorted in
* ascending order by physical address. If not, the kernel will panic in
* potentially a number of different places. (:-(
* NOTE: This workaround will not be sufficient if/when hotplugging memory is
* supported on x86/x64.
*/
static void
lgrp_plat_node_sort(node_domain_map_t *node_domain, int node_cnt,
cpu_node_map_t *cpu_node, int cpu_count, node_phys_addr_map_t *node_memory)
{
boolean_t found;
int i;
int j;
int n;
boolean_t sorted;
boolean_t swapped;
if (!lgrp_plat_node_sort_enable || node_cnt <= 1 ||
node_domain == NULL || node_memory == NULL)
return;
/*
* Sorted already?
*/
sorted = B_TRUE;
for (i = 0; i < node_cnt - 1; i++) {
/*
* Skip entries that don't exist
*/
if (!node_memory[i].exists)
continue;
/*
* Try to find next existing entry to compare against
*/
found = B_FALSE;
for (j = i + 1; j < node_cnt; j++) {
if (node_memory[j].exists) {
found = B_TRUE;
break;
}
}
/*
* Done if no more existing entries to compare against
*/
if (found == B_FALSE)
break;
/*
* Not sorted if starting address of current entry is bigger
* than starting address of next existing entry
*/
if (node_memory[i].start > node_memory[j].start) {
sorted = B_FALSE;
break;
}
}
/*
* Don't need to sort if sorted already
*/
if (sorted == B_TRUE)
return;
/*
* Just use bubble sort since number of nodes is small
*/
n = node_cnt;
do {
swapped = B_FALSE;
n--;
for (i = 0; i < n; i++) {
/*
* Skip entries that don't exist
*/
if (!node_memory[i].exists)
continue;
/*
* Try to find next existing entry to compare against
*/
found = B_FALSE;
for (j = i + 1; j <= n; j++) {
if (node_memory[j].exists) {
found = B_TRUE;
break;
}
}
/*
* Done if no more existing entries to compare against
*/
if (found == B_FALSE)
break;
if (node_memory[i].start > node_memory[j].start) {
node_phys_addr_map_t save_addr;
node_domain_map_t save_node;
/*
* Swap node to proxmity domain ID assignments
*/
bcopy(&node_domain[i], &save_node,
sizeof (node_domain_map_t));
bcopy(&node_domain[j], &node_domain[i],
sizeof (node_domain_map_t));
bcopy(&save_node, &node_domain[j],
sizeof (node_domain_map_t));
/*
* Swap node to physical memory assignments
*/
bcopy(&node_memory[i], &save_addr,
sizeof (node_phys_addr_map_t));
bcopy(&node_memory[j], &node_memory[i],
sizeof (node_phys_addr_map_t));
bcopy(&save_addr, &node_memory[j],
sizeof (node_phys_addr_map_t));
swapped = B_TRUE;
}
}
} while (swapped == B_TRUE);
/*
* Check to make sure that CPUs assigned to correct node IDs now since
* node to proximity domain ID assignments may have been changed above
*/
if (n == node_cnt - 1 || cpu_node == NULL || cpu_count < 1)
return;
for (i = 0; i < cpu_count; i++) {
int node;
node = lgrp_plat_domain_to_node(node_domain, node_cnt,
cpu_node[i].prox_domain);
if (cpu_node[i].node != node)
cpu_node[i].node = node;
}
}
/*
* Return time needed to probe from current CPU to memory in given node
*/
static hrtime_t
lgrp_plat_probe_time(int to, cpu_node_map_t *cpu_node, int cpu_node_nentries,
lgrp_plat_probe_mem_config_t *probe_mem_config,
lgrp_plat_latency_stats_t *lat_stats, lgrp_plat_probe_stats_t *probe_stats)
{
caddr_t buf;
hrtime_t elapsed;
hrtime_t end;
int from;
int i;
int ipl;
hrtime_t max;
hrtime_t min;
hrtime_t start;
extern int use_sse_pagecopy;
/*
* Determine ID of node containing current CPU
*/
from = lgrp_plat_cpu_to_node(CPU, cpu_node, cpu_node_nentries);
ASSERT(from >= 0 && from < lgrp_plat_node_cnt);
/*
* Do common work for probing main memory
*/
if (lgrp_plat_probe_flags & LGRP_PLAT_PROBE_PGCPY) {
/*
* Skip probing any nodes without memory and
* set probe time to 0
*/
if (probe_mem_config->probe_va[to] == NULL) {
lat_stats->latencies[from][to] = 0;
return (0);
}
/*
* Invalidate caches once instead of once every sample
* which should cut cost of probing by a lot
*/
probe_stats->flush_cost = gethrtime();
invalidate_cache();
probe_stats->flush_cost = gethrtime() -
probe_stats->flush_cost;
probe_stats->probe_cost_total += probe_stats->flush_cost;
}
/*
* Probe from current CPU to given memory using specified operation
* and take specified number of samples
*/
max = 0;
min = -1;
for (i = 0; i < lgrp_plat_probe_nsamples; i++) {
probe_stats->probe_cost = gethrtime();
/*
* Can't measure probe time if gethrtime() isn't working yet
*/
if (probe_stats->probe_cost == 0 && gethrtime() == 0)
return (0);
if (lgrp_plat_probe_flags & LGRP_PLAT_PROBE_VENDOR) {
/*
* Measure how long it takes to read vendor ID from
* Northbridge
*/
elapsed = opt_probe_vendor(to, lgrp_plat_probe_nreads);
} else {
/*
* Measure how long it takes to copy page
* on top of itself
*/
buf = probe_mem_config->probe_va[to] + (i * PAGESIZE);
kpreempt_disable();
ipl = splhigh();
start = gethrtime();
if (use_sse_pagecopy)
hwblkpagecopy(buf, buf);
else
bcopy(buf, buf, PAGESIZE);
end = gethrtime();
elapsed = end - start;
splx(ipl);
kpreempt_enable();
}
probe_stats->probe_cost = gethrtime() -
probe_stats->probe_cost;
probe_stats->probe_cost_total += probe_stats->probe_cost;
if (min == -1 || elapsed < min)
min = elapsed;
if (elapsed > max)
max = elapsed;
}
/*
* Update minimum and maximum probe times between
* these two nodes
*/
if (min < probe_stats->probe_min[from][to] ||
probe_stats->probe_min[from][to] == 0)
probe_stats->probe_min[from][to] = min;
if (max > probe_stats->probe_max[from][to])
probe_stats->probe_max[from][to] = max;
return (min);
}
/*
* Read boot property with CPU to APIC ID array, fill in CPU to node ID
* mapping table with APIC ID for each CPU (if pointer to table isn't NULL),
* and return number of CPU APIC IDs.
*
* NOTE: This code assumes that CPU IDs are assigned in order that they appear
* in in cpu_apicid_array boot property which is based on and follows
* same ordering as processor list in ACPI MADT. If the code in
* usr/src/uts/i86pc/io/pcplusmp/apic.c that reads MADT and assigns
* CPU IDs ever changes, then this code will need to change too....
*/
static int
lgrp_plat_process_cpu_apicids(cpu_node_map_t *cpu_node)
{
int boot_prop_len;
char *boot_prop_name = BP_CPU_APICID_ARRAY;
uint8_t cpu_apicid_array[UINT8_MAX + 1];
int i;
int n;
/*
* Check length of property value
*/
boot_prop_len = BOP_GETPROPLEN(bootops, boot_prop_name);
if (boot_prop_len <= 0 || boot_prop_len > sizeof (cpu_apicid_array))
return (-1);
/*
* Calculate number of entries in array and return when there's just
* one CPU since that's not very interesting for NUMA
*/
n = boot_prop_len / sizeof (uint8_t);
if (n == 1)
return (-2);
/*
* Get CPU to APIC ID property value
*/
if (BOP_GETPROP(bootops, boot_prop_name, cpu_apicid_array) < 0)
return (-3);
/*
* Just return number of CPU APIC IDs if CPU to node mapping table is
* NULL
*/
if (cpu_node == NULL)
return (n);
/*
* Fill in CPU to node ID mapping table with APIC ID for each CPU
*/
for (i = 0; i < n; i++) {
cpu_node[i].exists = 1;
cpu_node[i].apicid = cpu_apicid_array[i];
}
/*
* Return number of CPUs based on number of APIC IDs
*/
return (n);
}
/*
* Read ACPI System Locality Information Table (SLIT) to determine how far each
* NUMA node is from each other
*/
static int
lgrp_plat_process_slit(struct slit *tp, uint_t node_cnt,
node_phys_addr_map_t *node_memory, lgrp_plat_latency_stats_t *lat_stats)
{
int i;
int j;
int localities;
hrtime_t max;
hrtime_t min;
int retval;
uint8_t *slit_entries;
if (tp == NULL || !lgrp_plat_slit_enable)
return (1);
if (lat_stats == NULL)
return (2);
localities = tp->number;
if (localities != node_cnt)
return (3);
min = lat_stats->latency_min;
max = lat_stats->latency_max;
/*
* Fill in latency matrix based on SLIT entries
*/
slit_entries = tp->entry;
for (i = 0; i < localities; i++) {
for (j = 0; j < localities; j++) {
uint8_t latency;
latency = slit_entries[(i * localities) + j];
lat_stats->latencies[i][j] = latency;
if (latency < min || min == -1)
min = latency;
if (latency > max)
max = latency;
}
}
/*
* Verify that latencies/distances given in SLIT look reasonable
*/
retval = lgrp_plat_latency_verify(node_memory, lat_stats);
if (retval) {
/*
* Reinitialize (zero) latency table since SLIT doesn't look
* right
*/
for (i = 0; i < localities; i++) {
for (j = 0; j < localities; j++)
lat_stats->latencies[i][j] = 0;
}
} else {
/*
* Update min and max latencies seen since SLIT looks valid
*/
lat_stats->latency_min = min;
lat_stats->latency_max = max;
}
return (retval);
}
/*
* Read ACPI System Resource Affinity Table (SRAT) to determine which CPUs
* and memory are local to each other in the same NUMA node and return number
* of nodes
*/
static int
lgrp_plat_process_srat(struct srat *tp, uint32_t *prox_domain_min,
node_domain_map_t *node_domain, cpu_node_map_t *cpu_node, int cpu_count,
node_phys_addr_map_t *node_memory)
{
struct srat_item *srat_end;
int i;
struct srat_item *item;
int node_cnt;
int proc_entry_count;
/*
* Nothing to do when no SRAT or disabled
*/
if (tp == NULL || !lgrp_plat_srat_enable)
return (-1);
/*
* Determine number of nodes by counting number of proximity domains in
* SRAT and return if number of nodes is 1 or less since don't need to
* read SRAT then
*/
node_cnt = lgrp_plat_srat_domains(tp, prox_domain_min);
if (node_cnt == 1)
return (1);
else if (node_cnt <= 0)
return (-2);
/*
* Walk through SRAT, examining each CPU and memory entry to determine
* which CPUs and memory belong to which node.
*/
item = tp->list;
srat_end = (struct srat_item *)(tp->hdr.len + (uintptr_t)tp);
proc_entry_count = 0;
while (item < srat_end) {
uint32_t apic_id;
uint32_t domain;
uint64_t end;
uint64_t length;
uint64_t start;
switch (item->type) {
case SRAT_PROCESSOR: /* CPU entry */
if (!(item->i.p.flags & SRAT_ENABLED) ||
cpu_node == NULL)
break;
/*
* Calculate domain (node) ID and fill in APIC ID to
* domain/node mapping table
*/
domain = item->i.p.domain1;
for (i = 0; i < 3; i++) {
domain += item->i.p.domain2[i] <<
((i + 1) * 8);
}
apic_id = item->i.p.apic_id;
if (lgrp_plat_cpu_node_update(node_domain, node_cnt,
cpu_node, cpu_count, apic_id, domain) < 0)
return (-3);
proc_entry_count++;
break;
case SRAT_MEMORY: /* memory entry */
if (!(item->i.m.flags & SRAT_ENABLED) ||
node_memory == NULL)
break;
/*
* Get domain (node) ID and fill in domain/node
* to memory mapping table
*/
domain = item->i.m.domain;
start = item->i.m.base_addr;
length = item->i.m.len;
end = start + length - 1;
if (lgrp_plat_node_memory_update(node_domain, node_cnt,
node_memory, start, end, domain) < 0)
return (-4);
break;
case SRAT_X2APIC: /* x2apic CPU entry */
if (!(item->i.xp.flags & SRAT_ENABLED) ||
cpu_node == NULL)
break;
/*
* Calculate domain (node) ID and fill in APIC ID to
* domain/node mapping table
*/
domain = item->i.xp.domain;
apic_id = item->i.xp.x2apic_id;
if (lgrp_plat_cpu_node_update(node_domain, node_cnt,
cpu_node, cpu_count, apic_id, domain) < 0)
return (-3);
proc_entry_count++;
break;
default:
break;
}
item = (struct srat_item *)((uintptr_t)item + item->len);
}
/*
* Should have seen at least as many SRAT processor entries as CPUs
*/
if (proc_entry_count < cpu_count)
return (-5);
/*
* Need to sort nodes by starting physical address since VM system
* assumes and expects memnodes to be sorted in ascending order by
* physical address
*/
lgrp_plat_node_sort(node_domain, node_cnt, cpu_node, cpu_count,
node_memory);
return (node_cnt);
}
/*
* Allocate permanent memory for any temporary memory that we needed to
* allocate using BOP_ALLOC() before kmem_alloc() and VM system were
* initialized and copy everything from temporary to permanent memory since
* temporary boot memory will eventually be released during boot
*/
static void
lgrp_plat_release_bootstrap(void)
{
void *buf;
size_t size;
if (lgrp_plat_cpu_node_nentries > 0) {
size = lgrp_plat_cpu_node_nentries * sizeof (cpu_node_map_t);
buf = kmem_alloc(size, KM_SLEEP);
bcopy(lgrp_plat_cpu_node, buf, size);
lgrp_plat_cpu_node = buf;
}
}
/*
* Return number of proximity domains given in ACPI SRAT
*/
static int
lgrp_plat_srat_domains(struct srat *tp, uint32_t *prox_domain_min)
{
int domain_cnt;
uint32_t domain_min;
struct srat_item *end;
int i;
struct srat_item *item;
node_domain_map_t node_domain[MAX_NODES];
if (tp == NULL || !lgrp_plat_srat_enable)
return (1);
/*
* Walk through SRAT to find minimum proximity domain ID
*/
domain_min = UINT32_MAX;
item = tp->list;
end = (struct srat_item *)(tp->hdr.len + (uintptr_t)tp);
while (item < end) {
uint32_t domain;
switch (item->type) {
case SRAT_PROCESSOR: /* CPU entry */
if (!(item->i.p.flags & SRAT_ENABLED)) {
item = (struct srat_item *)((uintptr_t)item +
item->len);
continue;
}
domain = item->i.p.domain1;
for (i = 0; i < 3; i++) {
domain += item->i.p.domain2[i] <<
((i + 1) * 8);
}
break;
case SRAT_MEMORY: /* memory entry */
if (!(item->i.m.flags & SRAT_ENABLED)) {
item = (struct srat_item *)((uintptr_t)item +
item->len);
continue;
}
domain = item->i.m.domain;
break;
case SRAT_X2APIC: /* x2apic CPU entry */
if (!(item->i.xp.flags & SRAT_ENABLED)) {
item = (struct srat_item *)((uintptr_t)item +
item->len);
continue;
}
domain = item->i.xp.domain;
break;
default:
item = (struct srat_item *)((uintptr_t)item +
item->len);
continue;
}
/*
* Keep track of minimum proximity domain ID
*/
if (domain < domain_min)
domain_min = domain;
item = (struct srat_item *)((uintptr_t)item + item->len);
}
if (lgrp_plat_domain_min_enable && prox_domain_min != NULL)
*prox_domain_min = domain_min;
/*
* Walk through SRAT, examining each CPU and memory entry to determine
* proximity domain ID for each.
*/
domain_cnt = 0;
item = tp->list;
end = (struct srat_item *)(tp->hdr.len + (uintptr_t)tp);
bzero(node_domain, MAX_NODES * sizeof (node_domain_map_t));
while (item < end) {
uint32_t domain;
boolean_t overflow;
uint_t start;
switch (item->type) {
case SRAT_PROCESSOR: /* CPU entry */
if (!(item->i.p.flags & SRAT_ENABLED)) {
item = (struct srat_item *)((uintptr_t)item +
item->len);
continue;
}
domain = item->i.p.domain1;
for (i = 0; i < 3; i++) {
domain += item->i.p.domain2[i] <<
((i + 1) * 8);
}
break;
case SRAT_MEMORY: /* memory entry */
if (!(item->i.m.flags & SRAT_ENABLED)) {
item = (struct srat_item *)((uintptr_t)item +
item->len);
continue;
}
domain = item->i.m.domain;
break;
case SRAT_X2APIC: /* x2apic CPU entry */
if (!(item->i.xp.flags & SRAT_ENABLED)) {
item = (struct srat_item *)((uintptr_t)item +
item->len);
continue;
}
domain = item->i.xp.domain;
break;
default:
item = (struct srat_item *)((uintptr_t)item +
item->len);
continue;
}
/*
* Count and keep track of which proximity domain IDs seen
*/
start = i = domain % MAX_NODES;
overflow = B_TRUE;
do {
/*
* Create entry for proximity domain and increment
* count when no entry exists where proximity domain
* hashed
*/
if (!node_domain[i].exists) {
node_domain[i].exists = 1;
node_domain[i].prox_domain = domain;
domain_cnt++;
overflow = B_FALSE;
break;
}
/*
* Nothing to do when proximity domain seen already
* and its entry exists
*/
if (node_domain[i].prox_domain == domain) {
overflow = B_FALSE;
break;
}
/*
* Entry exists where proximity domain hashed, but for
* different proximity domain so keep search for empty
* slot to put it or matching entry whichever comes
* first.
*/
i = (i + 1) % MAX_NODES;
} while (i != start);
/*
* Didn't find empty or matching entry which means have more
* proximity domains than supported nodes (:-(
*/
ASSERT(overflow != B_TRUE);
if (overflow == B_TRUE)
return (-1);
item = (struct srat_item *)((uintptr_t)item + item->len);
}
return (domain_cnt);
}
/*
* Set lgroup latencies for 2 level lgroup topology
*/
static void
lgrp_plat_2level_setup(node_phys_addr_map_t *node_memory,
lgrp_plat_latency_stats_t *lat_stats)
{
int i;
ASSERT(node_memory != NULL && lat_stats != NULL);
if (lgrp_plat_node_cnt >= 4)
cmn_err(CE_NOTE,
"MPO only optimizing for local and remote\n");
for (i = 0; i < lgrp_plat_node_cnt; i++) {
int j;
if (!node_memory[i].exists)
continue;
for (j = 0; j < lgrp_plat_node_cnt; j++) {
if (!node_memory[j].exists)
continue;
if (i == j)
lat_stats->latencies[i][j] = 2;
else
lat_stats->latencies[i][j] = 3;
}
}
lat_stats->latency_min = 2;
lat_stats->latency_max = 3;
lgrp_config(LGRP_CONFIG_FLATTEN, 2, 0);
}
/*
* The following Opteron specific constants, macros, types, and routines define
* PCI configuration space registers and how to read them to determine the NUMA
* configuration of *supported* Opteron processors. They provide the same
* information that may be gotten from the ACPI System Resource Affinity Table
* (SRAT) if it exists on the machine of interest.
*
* The AMD BIOS and Kernel Developer's Guide (BKDG) for the processor family
* of interest describes all of these registers and their contents. The main
* registers used by this code to determine the NUMA configuration of the
* machine are the node ID register for the number of NUMA nodes and the DRAM
* address map registers for the physical address range of each node.
*
* NOTE: The format and how to determine the NUMA configuration using PCI
* config space registers may change or may not be supported in future
* Opteron processor families.
*/
/*
* How many bits to shift Opteron DRAM Address Map base and limit registers
* to get actual value
*/
#define OPT_DRAMADDR_HI_LSHIFT_ADDR 40 /* shift left for address */
#define OPT_DRAMADDR_LO_LSHIFT_ADDR 8 /* shift left for address */
#define OPT_DRAMADDR_HI_MASK_ADDR 0x000000FF /* address bits 47-40 */
#define OPT_DRAMADDR_LO_MASK_ADDR 0xFFFF0000 /* address bits 39-24 */
#define OPT_DRAMADDR_LO_MASK_OFF 0xFFFFFF /* offset for address */
/*
* Macros to derive addresses from Opteron DRAM Address Map registers
*/
#define OPT_DRAMADDR_HI(reg) \
(((u_longlong_t)reg & OPT_DRAMADDR_HI_MASK_ADDR) << \
OPT_DRAMADDR_HI_LSHIFT_ADDR)
#define OPT_DRAMADDR_LO(reg) \
(((u_longlong_t)reg & OPT_DRAMADDR_LO_MASK_ADDR) << \
OPT_DRAMADDR_LO_LSHIFT_ADDR)
#define OPT_DRAMADDR(high, low) \
(OPT_DRAMADDR_HI(high) | OPT_DRAMADDR_LO(low))
/*
* Bit masks defining what's in Opteron DRAM Address Map base register
*/
#define OPT_DRAMBASE_LO_MASK_RE 0x1 /* read enable */
#define OPT_DRAMBASE_LO_MASK_WE 0x2 /* write enable */
#define OPT_DRAMBASE_LO_MASK_INTRLVEN 0x700 /* interleave */
/*
* Bit masks defining what's in Opteron DRAM Address Map limit register
*/
#define OPT_DRAMLIMIT_LO_MASK_DSTNODE 0x7 /* destination node */
#define OPT_DRAMLIMIT_LO_MASK_INTRLVSEL 0x700 /* interleave select */
/*
* Opteron Node ID register in PCI configuration space contains
* number of nodes in system, etc. for Opteron K8. The following
* constants and macros define its contents, structure, and access.
*/
/*
* Bit masks defining what's in Opteron Node ID register
*/
#define OPT_NODE_MASK_ID 0x7 /* node ID */
#define OPT_NODE_MASK_CNT 0x70 /* node count */
#define OPT_NODE_MASK_IONODE 0x700 /* Hypertransport I/O hub node ID */
#define OPT_NODE_MASK_LCKNODE 0x7000 /* lock controller node ID */
#define OPT_NODE_MASK_CPUCNT 0xF0000 /* CPUs in system (0 means 1 CPU) */
/*
* How many bits in Opteron Node ID register to shift right to get actual value
*/
#define OPT_NODE_RSHIFT_CNT 0x4 /* shift right for node count value */
/*
* Macros to get values from Opteron Node ID register
*/
#define OPT_NODE_CNT(reg) \
((reg & OPT_NODE_MASK_CNT) >> OPT_NODE_RSHIFT_CNT)
/*
* Macro to setup PCI Extended Configuration Space (ECS) address to give to
* "in/out" instructions
*
* NOTE: Should only be used in lgrp_plat_init() before MMIO setup because any
* other uses should just do MMIO to access PCI ECS.
* Must enable special bit in Northbridge Configuration Register on
* Greyhound for extended CF8 space access to be able to access PCI ECS
* using "in/out" instructions and restore special bit after done
* accessing PCI ECS.
*/
#define OPT_PCI_ECS_ADDR(bus, device, function, reg) \
(PCI_CONE | (((bus) & 0xff) << 16) | (((device & 0x1f)) << 11) | \
(((function) & 0x7) << 8) | ((reg) & 0xfc) | \
((((reg) >> 8) & 0xf) << 24))
/*
* PCI configuration space registers accessed by specifying
* a bus, device, function, and offset. The following constants
* define the values needed to access Opteron K8 configuration
* info to determine its node topology
*/
#define OPT_PCS_BUS_CONFIG 0 /* Hypertransport config space bus */
/*
* Opteron PCI configuration space register function values
*/
#define OPT_PCS_FUNC_HT 0 /* Hypertransport configuration */
#define OPT_PCS_FUNC_ADDRMAP 1 /* Address map configuration */
#define OPT_PCS_FUNC_DRAM 2 /* DRAM configuration */
#define OPT_PCS_FUNC_MISC 3 /* Miscellaneous configuration */
/*
* PCI Configuration Space register offsets
*/
#define OPT_PCS_OFF_VENDOR 0x0 /* device/vendor ID register */
#define OPT_PCS_OFF_DRAMBASE_HI 0x140 /* DRAM Base register (node 0) */
#define OPT_PCS_OFF_DRAMBASE_LO 0x40 /* DRAM Base register (node 0) */
#define OPT_PCS_OFF_NODEID 0x60 /* Node ID register */
/*
* Opteron PCI Configuration Space device IDs for nodes
*/
#define OPT_PCS_DEV_NODE0 24 /* device number for node 0 */
/*
* Opteron DRAM address map gives base and limit for physical memory in a node
*/
typedef struct opt_dram_addr_map {
uint32_t base_hi;
uint32_t base_lo;
uint32_t limit_hi;
uint32_t limit_lo;
} opt_dram_addr_map_t;
/*
* Supported AMD processor families
*/
#define AMD_FAMILY_HAMMER 15
#define AMD_FAMILY_GREYHOUND 16
/*
* Whether to have is_opteron() return 1 even when processor isn't supported
*/
uint_t is_opteron_override = 0;
/*
* AMD processor family for current CPU
*/
uint_t opt_family = 0;
/*
* Determine whether we're running on a supported AMD Opteron since reading
* node count and DRAM address map registers may have different format or
* may not be supported across processor families
*/
static int
is_opteron(void)
{
if (x86_vendor != X86_VENDOR_AMD)
return (0);
opt_family = cpuid_getfamily(CPU);
if (opt_family == AMD_FAMILY_HAMMER ||
opt_family == AMD_FAMILY_GREYHOUND || is_opteron_override)
return (1);
else
return (0);
}
/*
* Determine NUMA configuration for Opteron from registers that live in PCI
* configuration space
*/
static void
opt_get_numa_config(uint_t *node_cnt, int *mem_intrlv,
node_phys_addr_map_t *node_memory)
{
uint_t bus;
uint_t dev;
struct opt_dram_addr_map dram_map[MAX_NODES];
uint_t node;
uint_t node_info[MAX_NODES];
uint_t off_hi;
uint_t off_lo;
uint64_t nb_cfg_reg;
/*
* Read configuration registers from PCI configuration space to
* determine node information, which memory is in each node, etc.
*
* Write to PCI configuration space address register to specify
* which configuration register to read and read/write PCI
* configuration space data register to get/set contents
*/
bus = OPT_PCS_BUS_CONFIG;
dev = OPT_PCS_DEV_NODE0;
off_hi = OPT_PCS_OFF_DRAMBASE_HI;
off_lo = OPT_PCS_OFF_DRAMBASE_LO;
/*
* Read node ID register for node 0 to get node count
*/
node_info[0] = pci_getl_func(bus, dev, OPT_PCS_FUNC_HT,
OPT_PCS_OFF_NODEID);
*node_cnt = OPT_NODE_CNT(node_info[0]) + 1;
/*
* If number of nodes is more than maximum supported, then set node
* count to 1 and treat system as UMA instead of NUMA.
*/
if (*node_cnt > MAX_NODES) {
*node_cnt = 1;
return;
}
/*
* For Greyhound, PCI Extended Configuration Space must be enabled to
* read high DRAM address map base and limit registers
*/
if (opt_family == AMD_FAMILY_GREYHOUND) {
nb_cfg_reg = rdmsr(MSR_AMD_NB_CFG);
if ((nb_cfg_reg & AMD_GH_NB_CFG_EN_ECS) == 0)
wrmsr(MSR_AMD_NB_CFG,
nb_cfg_reg | AMD_GH_NB_CFG_EN_ECS);
}
for (node = 0; node < *node_cnt; node++) {
uint32_t base_hi;
uint32_t base_lo;
uint32_t limit_hi;
uint32_t limit_lo;
/*
* Read node ID register (except for node 0 which we just read)
*/
if (node > 0) {
node_info[node] = pci_getl_func(bus, dev,
OPT_PCS_FUNC_HT, OPT_PCS_OFF_NODEID);
}
/*
* Read DRAM base and limit registers which specify
* physical memory range of each node
*/
if (opt_family != AMD_FAMILY_GREYHOUND)
base_hi = 0;
else {
outl(PCI_CONFADD, OPT_PCI_ECS_ADDR(bus, dev,
OPT_PCS_FUNC_ADDRMAP, off_hi));
base_hi = dram_map[node].base_hi =
inl(PCI_CONFDATA);
}
base_lo = dram_map[node].base_lo = pci_getl_func(bus, dev,
OPT_PCS_FUNC_ADDRMAP, off_lo);
if ((dram_map[node].base_lo & OPT_DRAMBASE_LO_MASK_INTRLVEN) &&
mem_intrlv)
*mem_intrlv = *mem_intrlv + 1;
off_hi += 4; /* high limit register offset */
if (opt_family != AMD_FAMILY_GREYHOUND)
limit_hi = 0;
else {
outl(PCI_CONFADD, OPT_PCI_ECS_ADDR(bus, dev,
OPT_PCS_FUNC_ADDRMAP, off_hi));
limit_hi = dram_map[node].limit_hi =
inl(PCI_CONFDATA);
}
off_lo += 4; /* low limit register offset */
limit_lo = dram_map[node].limit_lo = pci_getl_func(bus,
dev, OPT_PCS_FUNC_ADDRMAP, off_lo);
/*
* Increment device number to next node and register offsets
* for DRAM base register of next node
*/
off_hi += 4;
off_lo += 4;
dev++;
/*
* Both read and write enable bits must be enabled in DRAM
* address map base register for physical memory to exist in
* node
*/
if ((base_lo & OPT_DRAMBASE_LO_MASK_RE) == 0 ||
(base_lo & OPT_DRAMBASE_LO_MASK_WE) == 0) {
/*
* Mark node memory as non-existent and set start and
* end addresses to be same in node_memory[]
*/
node_memory[node].exists = 0;
node_memory[node].start = node_memory[node].end =
(pfn_t)-1;
continue;
}
/*
* Mark node memory as existing and remember physical address
* range of each node for use later
*/
node_memory[node].exists = 1;
node_memory[node].start = btop(OPT_DRAMADDR(base_hi, base_lo));
node_memory[node].end = btop(OPT_DRAMADDR(limit_hi, limit_lo) |
OPT_DRAMADDR_LO_MASK_OFF);
}
/*
* Restore PCI Extended Configuration Space enable bit
*/
if (opt_family == AMD_FAMILY_GREYHOUND) {
if ((nb_cfg_reg & AMD_GH_NB_CFG_EN_ECS) == 0)
wrmsr(MSR_AMD_NB_CFG, nb_cfg_reg);
}
}
/*
* Return average amount of time to read vendor ID register on Northbridge
* N times on specified destination node from current CPU
*/
static hrtime_t
opt_probe_vendor(int dest_node, int nreads)
{
int cnt;
uint_t dev;
/* LINTED: set but not used in function */
volatile uint_t dev_vendor;
hrtime_t elapsed;
hrtime_t end;
int ipl;
hrtime_t start;
dev = OPT_PCS_DEV_NODE0 + dest_node;
kpreempt_disable();
ipl = spl8();
outl(PCI_CONFADD, PCI_CADDR1(0, dev, OPT_PCS_FUNC_DRAM,
OPT_PCS_OFF_VENDOR));
start = gethrtime();
for (cnt = 0; cnt < nreads; cnt++)
dev_vendor = inl(PCI_CONFDATA);
end = gethrtime();
elapsed = (end - start) / nreads;
splx(ipl);
kpreempt_enable();
return (elapsed);
}