OpenSolaris_b135/uts/common/os/pg.c
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or http://www.opensolaris.org/os/licensing.
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2009 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
#include <sys/systm.h>
#include <sys/types.h>
#include <sys/param.h>
#include <sys/thread.h>
#include <sys/cpuvar.h>
#include <sys/cpupart.h>
#include <sys/kmem.h>
#include <sys/cmn_err.h>
#include <sys/kstat.h>
#include <sys/processor.h>
#include <sys/disp.h>
#include <sys/group.h>
#include <sys/pg.h>
/*
* Processor groups
*
* With the introduction of Chip Multi-Threaded (CMT) processor architectures,
* it is no longer necessarily true that a given physical processor module
* will present itself as a single schedulable entity (cpu_t). Rather, each
* chip and/or processor core may present itself as one or more "logical" CPUs.
*
* The logical CPUs presented may share physical components such as caches,
* data pipes, execution pipelines, FPUs, etc. It is advantageous to have the
* kernel be aware of the relationships existing between logical CPUs so that
* the appropriate optmizations may be employed.
*
* The processor group abstraction represents a set of logical CPUs that
* generally share some sort of physical or characteristic relationship.
*
* In the case of a physical sharing relationship, the CPUs in the group may
* share a pipeline, cache or floating point unit. In the case of a logical
* relationship, a PG may represent the set of CPUs in a processor set, or the
* set of CPUs running at a particular clock speed.
*
* The generic processor group structure, pg_t, contains the elements generic
* to a group of CPUs. Depending on the nature of the CPU relationship
* (LOGICAL or PHYSICAL), a pointer to a pg may be recast to a "view" of that
* PG where more specific data is represented.
*
* As an example, a PG representing a PHYSICAL relationship, may be recast to
* a pghw_t, where data further describing the hardware sharing relationship
* is maintained. See pghw.c and pghw.h for details on physical PGs.
*
* At this time a more specialized casting of a PG representing a LOGICAL
* relationship has not been implemented, but the architecture allows for this
* in the future.
*
* Processor Group Classes
*
* Processor group consumers may wish to maintain and associate specific
* data with the PGs they create. For this reason, a mechanism for creating
* class specific PGs exists. Classes may overload the default functions for
* creating, destroying, and associating CPUs with PGs, and may also register
* class specific callbacks to be invoked when the CPU related system
* configuration changes. Class specific data is stored/associated with
* PGs by incorporating the pg_t (or pghw_t, as appropriate), as the first
* element of a class specific PG object. In memory, such a structure may look
* like:
*
* ----------------------- - - -
* | common | | | | <--(pg_t *)
* ----------------------- | | -
* | HW specific | | | <-----(pghw_t *)
* ----------------------- | -
* | class specific | | <-------(pg_cmt_t *)
* ----------------------- -
*
* Access to the PG class specific data can be had by casting a pointer to
* it's class specific view.
*/
static pg_t *pg_alloc_default(pg_class_t);
static void pg_free_default(pg_t *);
static void pg_null_op();
/*
* Bootstrap CPU specific PG data
* See pg_cpu_bootstrap()
*/
static cpu_pg_t bootstrap_pg_data;
/*
* Bitset of allocated PG ids (they are sequential)
* and the next free id in the set.
*/
static bitset_t pg_id_set;
/*
* ID space starts from 1 to assume that root has ID 0;
*/
static pgid_t pg_id_next = 1;
/*
* Default and externed PG ops vectors
*/
static struct pg_ops pg_ops_default = {
pg_alloc_default, /* alloc */
pg_free_default, /* free */
NULL, /* cpu_init */
NULL, /* cpu_fini */
NULL, /* cpu_active */
NULL, /* cpu_inactive */
NULL, /* cpupart_in */
NULL, /* cpupart_out */
NULL, /* cpupart_move */
NULL, /* cpu_belongs */
NULL, /* policy_name */
};
static struct pg_cb_ops pg_cb_ops_default = {
pg_null_op, /* thread_swtch */
pg_null_op, /* thread_remain */
};
/*
* Class specific PG allocation callbacks
*/
#define PG_ALLOC(class) \
(pg_classes[class].pgc_ops->alloc ? \
pg_classes[class].pgc_ops->alloc() : \
pg_classes[pg_default_cid].pgc_ops->alloc())
#define PG_FREE(pg) \
((pg)->pg_class->pgc_ops->free ? \
(pg)->pg_class->pgc_ops->free(pg) : \
pg_classes[pg_default_cid].pgc_ops->free(pg)) \
/*
* Class specific PG policy name
*/
#define PG_POLICY_NAME(pg) \
((pg)->pg_class->pgc_ops->policy_name ? \
(pg)->pg_class->pgc_ops->policy_name(pg) : NULL) \
/*
* Class specific membership test callback
*/
#define PG_CPU_BELONGS(pg, cp) \
((pg)->pg_class->pgc_ops->cpu_belongs ? \
(pg)->pg_class->pgc_ops->cpu_belongs(pg, cp) : 0) \
/*
* CPU configuration callbacks
*/
#define PG_CPU_INIT(class, cp, cpu_pg) \
{ \
if (pg_classes[class].pgc_ops->cpu_init) \
pg_classes[class].pgc_ops->cpu_init(cp, cpu_pg); \
}
#define PG_CPU_FINI(class, cp, cpu_pg) \
{ \
if (pg_classes[class].pgc_ops->cpu_fini) \
pg_classes[class].pgc_ops->cpu_fini(cp, cpu_pg); \
}
#define PG_CPU_ACTIVE(class, cp) \
{ \
if (pg_classes[class].pgc_ops->cpu_active) \
pg_classes[class].pgc_ops->cpu_active(cp); \
}
#define PG_CPU_INACTIVE(class, cp) \
{ \
if (pg_classes[class].pgc_ops->cpu_inactive) \
pg_classes[class].pgc_ops->cpu_inactive(cp); \
}
/*
* CPU / cpupart configuration callbacks
*/
#define PG_CPUPART_IN(class, cp, pp) \
{ \
if (pg_classes[class].pgc_ops->cpupart_in) \
pg_classes[class].pgc_ops->cpupart_in(cp, pp); \
}
#define PG_CPUPART_OUT(class, cp, pp) \
{ \
if (pg_classes[class].pgc_ops->cpupart_out) \
pg_classes[class].pgc_ops->cpupart_out(cp, pp); \
}
#define PG_CPUPART_MOVE(class, cp, old, new) \
{ \
if (pg_classes[class].pgc_ops->cpupart_move) \
pg_classes[class].pgc_ops->cpupart_move(cp, old, new); \
}
static pg_class_t *pg_classes;
static int pg_nclasses;
static pg_cid_t pg_default_cid;
/*
* Initialze common PG subsystem.
*/
void
pg_init(void)
{
extern void pg_cmt_class_init();
extern void pg_cmt_cpu_startup();
pg_default_cid =
pg_class_register("default", &pg_ops_default, PGR_LOGICAL);
/*
* Initialize classes to allow them to register with the framework
*/
pg_cmt_class_init();
pg_cpu0_init();
pg_cmt_cpu_startup(CPU);
}
/*
* Perform CPU 0 initialization
*/
void
pg_cpu0_init(void)
{
extern void pghw_physid_create();
/*
* Create the physical ID cache for the boot CPU
*/
pghw_physid_create(CPU);
/*
* pg_cpu_* require that cpu_lock be held
*/
mutex_enter(&cpu_lock);
(void) pg_cpu_init(CPU, B_FALSE);
pg_cpupart_in(CPU, &cp_default);
pg_cpu_active(CPU);
mutex_exit(&cpu_lock);
}
/*
* Invoked when topology for CPU0 changes
* post pg_cpu0_init().
*
* Currently happens as a result of null_proc_lpa
* on Starcat.
*/
void
pg_cpu0_reinit(void)
{
mutex_enter(&cpu_lock);
pg_cpu_inactive(CPU);
pg_cpupart_out(CPU, &cp_default);
pg_cpu_fini(CPU, NULL);
(void) pg_cpu_init(CPU, B_FALSE);
pg_cpupart_in(CPU, &cp_default);
pg_cpu_active(CPU);
mutex_exit(&cpu_lock);
}
/*
* Register a new PG class
*/
pg_cid_t
pg_class_register(char *name, struct pg_ops *ops, pg_relation_t relation)
{
pg_class_t *newclass;
pg_class_t *classes_old;
id_t cid;
mutex_enter(&cpu_lock);
/*
* Allocate a new pg_class_t in the pg_classes array
*/
if (pg_nclasses == 0) {
pg_classes = kmem_zalloc(sizeof (pg_class_t), KM_SLEEP);
} else {
classes_old = pg_classes;
pg_classes =
kmem_zalloc(sizeof (pg_class_t) * (pg_nclasses + 1),
KM_SLEEP);
(void) kcopy(classes_old, pg_classes,
sizeof (pg_class_t) * pg_nclasses);
kmem_free(classes_old, sizeof (pg_class_t) * pg_nclasses);
}
cid = pg_nclasses++;
newclass = &pg_classes[cid];
(void) strncpy(newclass->pgc_name, name, PG_CLASS_NAME_MAX);
newclass->pgc_id = cid;
newclass->pgc_ops = ops;
newclass->pgc_relation = relation;
mutex_exit(&cpu_lock);
return (cid);
}
/*
* Try to find an existing pg in set in which to place cp.
* Returns the pg if found, and NULL otherwise.
* In the event that the CPU could belong to multiple
* PGs in the set, the first matching PG will be returned.
*/
pg_t *
pg_cpu_find_pg(cpu_t *cp, group_t *set)
{
pg_t *pg;
group_iter_t i;
group_iter_init(&i);
while ((pg = group_iterate(set, &i)) != NULL) {
/*
* Ask the class if the CPU belongs here
*/
if (PG_CPU_BELONGS(pg, cp))
return (pg);
}
return (NULL);
}
/*
* Iterate over the CPUs in a PG after initializing
* the iterator with PG_CPU_ITR_INIT()
*/
cpu_t *
pg_cpu_next(pg_cpu_itr_t *itr)
{
cpu_t *cpu;
pg_t *pg = itr->pg;
cpu = group_iterate(&pg->pg_cpus, &itr->position);
return (cpu);
}
/*
* Test if a given PG contains a given CPU
*/
boolean_t
pg_cpu_find(pg_t *pg, cpu_t *cp)
{
if (group_find(&pg->pg_cpus, cp) == (uint_t)-1)
return (B_FALSE);
return (B_TRUE);
}
/*
* Set the PGs callbacks to the default
*/
void
pg_callback_set_defaults(pg_t *pg)
{
bcopy(&pg_cb_ops_default, &pg->pg_cb, sizeof (struct pg_cb_ops));
}
/*
* Create a PG of a given class.
* This routine may block.
*/
pg_t *
pg_create(pg_cid_t cid)
{
pg_t *pg;
pgid_t id;
ASSERT(MUTEX_HELD(&cpu_lock));
/*
* Call the class specific PG allocation routine
*/
pg = PG_ALLOC(cid);
pg->pg_class = &pg_classes[cid];
pg->pg_relation = pg->pg_class->pgc_relation;
/*
* Find the next free sequential pg id
*/
do {
if (pg_id_next >= bitset_capacity(&pg_id_set))
bitset_resize(&pg_id_set, pg_id_next + 1);
id = pg_id_next++;
} while (bitset_in_set(&pg_id_set, id));
pg->pg_id = id;
bitset_add(&pg_id_set, pg->pg_id);
/*
* Create the PG's CPU group
*/
group_create(&pg->pg_cpus);
/*
* Initialize the events ops vector
*/
pg_callback_set_defaults(pg);
return (pg);
}
/*
* Destroy a PG.
* This routine may block.
*/
void
pg_destroy(pg_t *pg)
{
ASSERT(MUTEX_HELD(&cpu_lock));
group_destroy(&pg->pg_cpus);
/*
* Unassign the pg_id
*/
if (pg_id_next > pg->pg_id)
pg_id_next = pg->pg_id;
bitset_del(&pg_id_set, pg->pg_id);
/*
* Invoke the class specific de-allocation routine
*/
PG_FREE(pg);
}
/*
* Add the CPU "cp" to processor group "pg"
* This routine may block.
*/
void
pg_cpu_add(pg_t *pg, cpu_t *cp, cpu_pg_t *cpu_pg)
{
int err;
ASSERT(MUTEX_HELD(&cpu_lock));
/* This adds the CPU to the PG's CPU group */
err = group_add(&pg->pg_cpus, cp, GRP_RESIZE);
ASSERT(err == 0);
/*
* The CPU should be referencing the bootstrap PG data still
* at this point, since this routine may block causing us to
* enter the dispatcher.
*/
ASSERT(pg_cpu_is_bootstrapped(cp));
/* This adds the PG to the CPUs PG group */
err = group_add(&cpu_pg->pgs, pg, GRP_RESIZE);
ASSERT(err == 0);
}
/*
* Remove "cp" from "pg".
* This routine may block.
*/
void
pg_cpu_delete(pg_t *pg, cpu_t *cp, cpu_pg_t *cpu_pg)
{
int err;
ASSERT(MUTEX_HELD(&cpu_lock));
/* Remove the CPU from the PG */
err = group_remove(&pg->pg_cpus, cp, GRP_RESIZE);
ASSERT(err == 0);
/*
* The CPU should be referencing the bootstrap PG data still
* at this point, since this routine may block causing us to
* enter the dispatcher.
*/
ASSERT(pg_cpu_is_bootstrapped(cp));
/* Remove the PG from the CPU's PG group */
err = group_remove(&cpu_pg->pgs, pg, GRP_RESIZE);
ASSERT(err == 0);
}
/*
* Allocate a CPU's PG data. This hangs off struct cpu at cpu_pg
*/
static cpu_pg_t *
pg_cpu_data_alloc(void)
{
cpu_pg_t *pgd;
pgd = kmem_zalloc(sizeof (cpu_pg_t), KM_SLEEP);
group_create(&pgd->pgs);
group_create(&pgd->cmt_pgs);
return (pgd);
}
/*
* Free the CPU's PG data.
*/
static void
pg_cpu_data_free(cpu_pg_t *pgd)
{
group_destroy(&pgd->pgs);
group_destroy(&pgd->cmt_pgs);
kmem_free(pgd, sizeof (cpu_pg_t));
}
/*
* Called when either a new CPU is coming into the system (either
* via booting or DR) or when the CPU's PG data is being recalculated.
* Allocate its PG data, and notify all registered classes about
* the new CPU.
*
* If "deferred_init" is B_TRUE, the CPU's PG data will be allocated
* and returned, but the "bootstrap" structure will be left in place.
* The deferred_init option is used when all CPUs in the system are
* using the bootstrap structure as part of the process of recalculating
* all PG data. The caller must replace the bootstrap structure with the
* allocated PG data before pg_cpu_active is called.
*
* This routine may block.
*/
cpu_pg_t *
pg_cpu_init(cpu_t *cp, boolean_t deferred_init)
{
pg_cid_t i;
cpu_pg_t *cpu_pg;
ASSERT(MUTEX_HELD(&cpu_lock));
/*
* Allocate and size the per CPU pg data
*
* The CPU's PG data will be populated by the various
* PG classes during the invocation of the PG_CPU_INIT()
* callback below.
*
* Since the we could block and enter the dispatcher during
* this process, the CPU will continue to reference the bootstrap
* PG data until all the initialization completes.
*/
ASSERT(pg_cpu_is_bootstrapped(cp));
cpu_pg = pg_cpu_data_alloc();
/*
* Notify all registered classes about the new CPU
*/
for (i = 0; i < pg_nclasses; i++)
PG_CPU_INIT(i, cp, cpu_pg);
/*
* The CPU's PG data is now ready to use.
*/
if (deferred_init == B_FALSE)
cp->cpu_pg = cpu_pg;
return (cpu_pg);
}
/*
* Either this CPU is being deleted from the system or its PG data is
* being recalculated. Notify the classes and free up the CPU's PG data.
*
* If "cpu_pg_deferred" is non-NULL, it points to the CPU's PG data and
* serves to indicate that this CPU is already using the bootstrap
* stucture. Used as part of the process to recalculate the PG data for
* all CPUs in the system.
*/
void
pg_cpu_fini(cpu_t *cp, cpu_pg_t *cpu_pg_deferred)
{
pg_cid_t i;
cpu_pg_t *cpu_pg;
ASSERT(MUTEX_HELD(&cpu_lock));
if (cpu_pg_deferred == NULL) {
cpu_pg = cp->cpu_pg;
/*
* This can happen if the CPU coming into the system
* failed to power on.
*/
if (cpu_pg == NULL || pg_cpu_is_bootstrapped(cp))
return;
/*
* Have the CPU reference the bootstrap PG data to survive
* the dispatcher should it block from here on out.
*/
pg_cpu_bootstrap(cp);
} else {
ASSERT(pg_cpu_is_bootstrapped(cp));
cpu_pg = cpu_pg_deferred;
}
for (i = 0; i < pg_nclasses; i++)
PG_CPU_FINI(i, cp, cpu_pg);
pg_cpu_data_free(cpu_pg);
}
/*
* This CPU is becoming active (online)
* This routine may not block as it is called from paused CPUs
* context.
*/
void
pg_cpu_active(cpu_t *cp)
{
pg_cid_t i;
ASSERT(MUTEX_HELD(&cpu_lock));
/*
* Notify all registered classes about the new CPU
*/
for (i = 0; i < pg_nclasses; i++)
PG_CPU_ACTIVE(i, cp);
}
/*
* This CPU is going inactive (offline)
* This routine may not block, as it is called from paused
* CPUs context.
*/
void
pg_cpu_inactive(cpu_t *cp)
{
pg_cid_t i;
ASSERT(MUTEX_HELD(&cpu_lock));
/*
* Notify all registered classes about the new CPU
*/
for (i = 0; i < pg_nclasses; i++)
PG_CPU_INACTIVE(i, cp);
}
/*
* Invoked when the CPU is about to move into the partition
* This routine may block.
*/
void
pg_cpupart_in(cpu_t *cp, cpupart_t *pp)
{
int i;
ASSERT(MUTEX_HELD(&cpu_lock));
/*
* Notify all registered classes that the
* CPU is about to enter the CPU partition
*/
for (i = 0; i < pg_nclasses; i++)
PG_CPUPART_IN(i, cp, pp);
}
/*
* Invoked when the CPU is about to move out of the partition
* This routine may block.
*/
/*ARGSUSED*/
void
pg_cpupart_out(cpu_t *cp, cpupart_t *pp)
{
int i;
ASSERT(MUTEX_HELD(&cpu_lock));
/*
* Notify all registered classes that the
* CPU is about to leave the CPU partition
*/
for (i = 0; i < pg_nclasses; i++)
PG_CPUPART_OUT(i, cp, pp);
}
/*
* Invoked when the CPU is *moving* partitions.
*
* This routine may not block, as it is called from paused CPUs
* context.
*/
void
pg_cpupart_move(cpu_t *cp, cpupart_t *oldpp, cpupart_t *newpp)
{
int i;
ASSERT(MUTEX_HELD(&cpu_lock));
/*
* Notify all registered classes that the
* CPU is about to leave the CPU partition
*/
for (i = 0; i < pg_nclasses; i++)
PG_CPUPART_MOVE(i, cp, oldpp, newpp);
}
/*
* Return a class specific string describing a policy implemented
* across this PG
*/
char *
pg_policy_name(pg_t *pg)
{
char *str;
if ((str = PG_POLICY_NAME(pg)) != NULL)
return (str);
return ("N/A");
}
/*
* Provide the specified CPU a bootstrap pg
* This is needed to allow sane behaviour if any PG consuming
* code needs to deal with a partially initialized CPU
*/
void
pg_cpu_bootstrap(cpu_t *cp)
{
cp->cpu_pg = &bootstrap_pg_data;
}
/*
* Return non-zero if the specified CPU is bootstrapped,
* which means it's CPU specific PG data has not yet been
* fully constructed.
*/
int
pg_cpu_is_bootstrapped(cpu_t *cp)
{
return (cp->cpu_pg == &bootstrap_pg_data);
}
/*ARGSUSED*/
static pg_t *
pg_alloc_default(pg_class_t class)
{
return (kmem_zalloc(sizeof (pg_t), KM_SLEEP));
}
/*ARGSUSED*/
static void
pg_free_default(struct pg *pg)
{
kmem_free(pg, sizeof (pg_t));
}
static void
pg_null_op()
{
}
/*
* Invoke the "thread switch" callback for each of the CPU's PGs
* This is invoked from the dispatcher swtch() routine, which is called
* when a thread running an a CPU should switch to another thread.
* "cp" is the CPU on which the thread switch is happening
* "now" is an unscaled hrtime_t timestamp taken in swtch()
* "old" and "new" are the outgoing and incoming threads, respectively.
*/
void
pg_ev_thread_swtch(struct cpu *cp, hrtime_t now, kthread_t *old, kthread_t *new)
{
int i, sz;
group_t *grp;
pg_t *pg;
grp = &cp->cpu_pg->pgs;
sz = GROUP_SIZE(grp);
for (i = 0; i < sz; i++) {
pg = GROUP_ACCESS(grp, i);
pg->pg_cb.thread_swtch(pg, cp, now, old, new);
}
}
/*
* Invoke the "thread remain" callback for each of the CPU's PGs.
* This is called from the dispatcher's swtch() routine when a thread
* running on the CPU "cp" is switching to itself, which can happen as an
* artifact of the thread's timeslice expiring.
*/
void
pg_ev_thread_remain(struct cpu *cp, kthread_t *t)
{
int i, sz;
group_t *grp;
pg_t *pg;
grp = &cp->cpu_pg->pgs;
sz = GROUP_SIZE(grp);
for (i = 0; i < sz; i++) {
pg = GROUP_ACCESS(grp, i);
pg->pg_cb.thread_remain(pg, cp, t);
}
}