OpenSolaris_b135/uts/common/disp/cpucaps.c
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or http://www.opensolaris.org/os/licensing.
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2009 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
#include <sys/disp.h>
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/sysmacros.h>
#include <sys/atomic.h>
#include <sys/cpucaps_impl.h>
#include <sys/dtrace.h>
#include <sys/sdt.h>
#include <sys/debug.h>
#include <sys/rctl.h>
#include <sys/errno.h>
/*
* CPU Caps implementation
* =======================
*
* A CPU cap can be set on any project or any zone. Zone CPU cap limits the CPU
* usage for all projects running inside the zone. If the zone CPU cap is set
* below the project CPU cap, the latter will have no effect.
*
* When CPU usage of projects and/or zones reaches specified caps, threads in
* them do not get scheduled and instead are placed on wait queues associated
* with a cap. Such threads will start running again only when CPU usage drops
* below the cap level. Each zone and each project has its own wait queue.
*
* When CPU cap is set, the kernel continously keeps track of CPU time used by
* capped zones and/or projects over a short time interval and calculates their
* current CPU usage as a percentage. When the accumulated usage reaches the CPU
* cap, LWPs running in the user-land (when they are not holding any critical
* kernel locks) are placed on special wait queues until their project's or
* zone's CPU usage drops below the cap.
*
* The system maintains a list of all capped projects and all capped zones. On
* every clock tick every active thread belonging to a capped project adds its
* CPU usage to its project. Usage from all projects belonging to a capped zone
* is aggregated to get the zone usage.
*
* When the current CPU usage is above the cap, a project or zone is considered
* over-capped. Every user thread caught running in an over-capped project or
* zone is marked by setting TS_PROJWAITQ flag in thread's t_schedflag field and
* is requested to surrender its CPU. This causes scheduling class specific
* CL_PREEMPT() callback to be invoked. The callback function places threads
* marked as TS_PROJWAIT on a wait queue and calls switch().
*
* Threads are only placed on wait queues after trapping from user-land
* (they could be holding some user locks, but no kernel locks) and while
* returning from the trap back to the user-land when no kernel locks are held.
* Putting threads on wait queues in random places while running in the
* kernel might lead to all kinds of locking problems.
*
* Accounting
* ==========
*
* Accounting of CPU usage is based on per-thread micro-state accounting data.
* On every clock tick clock() adds new on-CPU time for every thread found on
* CPU. Scheduling classes also add new on-CPU time for any thread leaving CPU.
* New times means time since it was last accounted for. On-CPU times greater
* than 1 tick are truncated to 1 tick.
*
* Project CPU usage is aggregated from all threads within the project.
* Zone CPU usage is the sum of usages for all projects within the zone. Zone
* CPU usage is calculated on every clock tick by walking list of projects and
* adding their usage together.
*
* Decay
* =====
*
* CPU usage is decayed by the caps_update() routine which is called once per
* every clock tick. It walks lists of project caps and decays their usages by
* one per cent. If CPU usage drops below cap levels, threads on the wait queue
* are made runnable again, one thread per clock tick.
*
* Interfaces
* ==========
*
* The CPU Caps facility provides the following interfaces to the rest of the
* system:
*
* cpucaps_project_add(kproject_t *)
*
* Notifies the framework of a new project. It should be put on the
* capped_projects list if its zone has a cap.
*
* cpucaps_project_remove(kproject_t *)
*
* Remove the association between the specified project and its cap.
* Called right before the project is destroyed.
*
* cpucaps_project_set(kproject_t *, rctl_qty_t)
*
* Set project cap of the specified project to the specified value. Setting the
* value to NOCAP is equivalent to removing the cap.
*
* cpucaps_zone_set(zone_t *, rctl_qty_t)
*
* Set zone cap of the specified zone to the specified value. Setting the value
* to NOCAP is equivalent to removing the cap.
*
* cpucaps_zone_remove(zone_t *)
*
* Remove the association between the zone and its cap.
*
* cpucaps_charge(kthread_id_t, caps_sc_t *, cpucaps_charge_t)
*
* Charges specified thread's project the amount of on-CPU time that it used.
* If the third argument is CPUCAPS_CHARGE_ONLY returns False.
* Otherwise returns True if project or zone should be penalized because its
* project or zone is exceeding its cap. Also sets TS_PROJWAITQ or TS_ZONEWAITQ
* bits in t_schedflag in this case.
*
* CPUCAPS_ENFORCE(kthread_id_t *)
*
* Enforces CPU caps for a specified thread. Places LWPs running in LWP_USER
* state on project or zone wait queues, as requested by TS_PROJWAITQ or
* TS_ZONEWAITQ bits in t_schedflag. Returns True if the thread was placed on a
* wait queue or False otherwise.
*
* cpucaps_sc_init(caps_sc_t *)
*
* Initializes the scheduling-class specific CPU Caps data for a thread.
*
* LOCKS
* =====
*
* all the individual caps structures and their lists are protected by a global
* caps_lock mutex. The lock is grabbed either by clock() or by events modifying
* caps, so it is usually uncontended. We avoid all blocking memory allocations
* while holding caps_lock to prevent clock() from blocking.
*
* Thread state is protected by the thread lock. It protects the association
* between a thread and its project and, as a consequence, to its zone. The
* association can not break while thread lock is held, so the project or zone
* cap are not going to disappear while thread lock is held.
*
* Cap usage field is protected by high-pil spin-lock cap_usagelock. It is
* grabbed by scheduling classes already holding thread lock at high PIL and by
* clock thread performing usage decay. We should do as little work as possible
* while holding the lock since it may be very hot. All threads in the project
* contend for the same cache line doing cap usage updates.
*/
/*
* caps_lock protects list of capped projects and zones, changes in the cap
* state and changes of the global cpucaps_enabled flag.
*
* Changing zone caps also sets cpucaps_busy to avoid races when a zone cap is
* modified in parallel. This can be per-zone cap flag, but we don't keep any
* cap state for now.
*/
static kmutex_t caps_lock; /* lock to protect: */
static list_t capped_zones; /* - list of zones with caps */
static list_t capped_projects; /* - list of projects with caps */
boolean_t cpucaps_enabled; /* - are there any caps defined? */
boolean_t cpucaps_busy; /* - is framework busy? */
/*
* The accounting is based on the number of nanoseconds threads spend running
* during a tick which is kept in the cap_tick_cost variable.
*/
static hrtime_t cap_tick_cost;
/*
* How much of the usage value is decayed every clock tick
* Decay one per cent of value per tick
*/
#define CAP_DECAY_FACTOR 100
/*
* Scale the value and round it to the closest integer value
*/
#define ROUND_SCALE(x, y) (((x) + (y) / 2) / (y))
static void caps_update();
/*
* CAP kstats.
*/
struct cap_kstat {
kstat_named_t cap_value;
kstat_named_t cap_usage;
kstat_named_t cap_nwait;
kstat_named_t cap_below;
kstat_named_t cap_above;
kstat_named_t cap_maxusage;
kstat_named_t cap_zonename;
} cap_kstat = {
{ "value", KSTAT_DATA_UINT64 },
{ "usage", KSTAT_DATA_UINT64 },
{ "nwait", KSTAT_DATA_UINT64 },
{ "below_sec", KSTAT_DATA_UINT64 },
{ "above_sec", KSTAT_DATA_UINT64 },
{ "maxusage", KSTAT_DATA_UINT64 },
{ "zonename", KSTAT_DATA_STRING },
};
static kmutex_t cap_kstat_lock;
static int cap_kstat_update(kstat_t *, int);
/*
* Initialize CPU caps infrastructure.
* - Initialize lists of capped zones and capped projects
* - Set cpucaps_clock_callout to NULL
*/
void
cpucaps_init()
{
/*
* Initialize global variables
*/
cap_tick_cost = TICK_TO_NSEC((hrtime_t)1);
list_create(&capped_zones, sizeof (cpucap_t),
offsetof(cpucap_t, cap_link));
list_create(&capped_projects, sizeof (cpucap_t),
offsetof(cpucap_t, cap_link));
cpucaps_enabled = B_FALSE;
cpucaps_busy = B_FALSE;
cpucaps_clock_callout = NULL;
}
/*
* Initialize scheduling-class specific CPU Caps data.
*/
void
cpucaps_sc_init(caps_sc_t *csc)
{
csc->csc_cputime = 0;
}
/*
* Allocate and initialize cpucap structure
*/
static cpucap_t *
cap_alloc(void)
{
cpucap_t *cap = kmem_zalloc(sizeof (cpucap_t), KM_SLEEP);
DISP_LOCK_INIT(&cap->cap_usagelock);
waitq_init(&cap->cap_waitq);
return (cap);
}
/*
* Free cpucap structure
*/
static void
cap_free(cpucap_t *cap)
{
if (cap == NULL)
return;
/*
* This cap should not be active
*/
ASSERT(!list_link_active(&cap->cap_link));
ASSERT(cap->cap_value == 0);
ASSERT(!DISP_LOCK_HELD(&cap->cap_usagelock));
waitq_fini(&cap->cap_waitq);
DISP_LOCK_DESTROY(&cap->cap_usagelock);
kmem_free(cap, sizeof (cpucap_t));
}
/*
* Activate cap - insert into active list and unblock its
* wait queue. Should be called with caps_lock held.
* The cap_value field is set to the value supplied.
*/
static void
cap_enable(list_t *l, cpucap_t *cap, hrtime_t value)
{
ASSERT(MUTEX_HELD(&caps_lock));
/*
* Cap can not be already enabled
*/
ASSERT(!CAP_ENABLED(cap));
ASSERT(!list_link_active(&cap->cap_link));
list_insert_tail(l, cap);
cap->cap_below = cap->cap_above = 0;
cap->cap_maxusage = 0;
cap->cap_usage = 0;
cap->cap_value = value;
waitq_unblock(&cap->cap_waitq);
if (CPUCAPS_OFF()) {
cpucaps_enabled = B_TRUE;
cpucaps_clock_callout = caps_update;
}
}
/*
* Deactivate cap
* - Block its wait queue. This prevents any new threads from being
* enqueued there and moves all enqueued threads to the run queue.
* - Remove cap from list l.
* - Disable CPU caps globally if there are no capped projects or zones
*
* Should be called with caps_lock held.
*/
static void
cap_disable(list_t *l, cpucap_t *cap)
{
ASSERT(MUTEX_HELD(&caps_lock));
/*
* Cap should be currently active
*/
ASSERT(CPUCAPS_ON());
ASSERT(list_link_active(&cap->cap_link));
ASSERT(CAP_ENABLED(cap));
waitq_block(&cap->cap_waitq);
list_remove(l, cap);
if (list_is_empty(&capped_projects) && list_is_empty(&capped_zones)) {
cpucaps_enabled = B_FALSE;
cpucaps_clock_callout = NULL;
}
cap->cap_value = 0;
cap->cap_project = NULL;
cap->cap_zone = NULL;
if (cap->cap_kstat != NULL) {
kstat_delete(cap->cap_kstat);
cap->cap_kstat = NULL;
}
}
/*
* Enable cap for a project kpj
* It is safe to enable already enabled project cap.
* Should be called with caps_lock held.
*/
static void
cap_project_enable(kproject_t *kpj, hrtime_t value)
{
cpucap_t *cap = kpj->kpj_cpucap;
ASSERT(MUTEX_HELD(&caps_lock));
ASSERT(cap != NULL);
if (CAP_DISABLED(cap)) {
ASSERT(cap->cap_kstat == NULL);
cap_enable(&capped_projects, cap, value);
cap->cap_project = kpj;
cap->cap_zone = kpj->kpj_zone;
/*
* Create cap kstats
*/
if ((cap->cap_kstat = rctl_kstat_create_project(kpj, "cpucaps",
KSTAT_TYPE_NAMED,
sizeof (cap_kstat) / sizeof (kstat_named_t),
KSTAT_FLAG_VIRTUAL)) != NULL) {
cap->cap_kstat->ks_data_size +=
strlen(cap->cap_zone->zone_name) + 1;
cap->cap_kstat->ks_lock = &cap_kstat_lock;
cap->cap_kstat->ks_data = &cap_kstat;
cap->cap_kstat->ks_update = cap_kstat_update;
cap->cap_kstat->ks_private = cap;
kstat_install(cap->cap_kstat);
}
}
}
/*
* Disable project cap.
* It is safe to disable already disabled project cap.
* Should be called with caps_lock held.
*/
static void
cap_project_disable(kproject_t *kpj)
{
cpucap_t *cap = kpj->kpj_cpucap;
ASSERT(MUTEX_HELD(&caps_lock));
ASSERT(cap != NULL);
ASSERT(cap->cap_project == kpj);
if (CAP_ENABLED(cap))
cap_disable(&capped_projects, cap);
}
/*
* Enable cap for a zone
* It is safe to enable already enabled zone cap.
* Should be called with caps_lock held.
*/
static void
cap_zone_enable(zone_t *zone, hrtime_t value)
{
cpucap_t *cap = zone->zone_cpucap;
ASSERT(MUTEX_HELD(&caps_lock));
ASSERT(cap != NULL);
if (CAP_DISABLED(cap)) {
ASSERT(cap->cap_kstat == NULL);
cap_enable(&capped_zones, cap, value);
cap->cap_zone = zone;
/*
* Create cap kstats
*/
if ((cap->cap_kstat = rctl_kstat_create_zone(zone, "cpucaps",
KSTAT_TYPE_NAMED,
sizeof (cap_kstat) / sizeof (kstat_named_t),
KSTAT_FLAG_VIRTUAL)) != NULL) {
cap->cap_kstat->ks_data_size +=
strlen(cap->cap_zone->zone_name) + 1;
cap->cap_kstat->ks_lock = &cap_kstat_lock;
cap->cap_kstat->ks_data = &cap_kstat;
cap->cap_kstat->ks_update = cap_kstat_update;
cap->cap_kstat->ks_private = cap;
kstat_install(cap->cap_kstat);
}
}
}
/*
* Disable zone cap.
* It is safe to disable already disabled zone cap.
* Should be called with caps_lock held.
*/
static void
cap_zone_disable(zone_t *zone)
{
cpucap_t *cap = zone->zone_cpucap;
ASSERT(MUTEX_HELD(&caps_lock));
ASSERT(cap != NULL);
ASSERT(cap->cap_zone == zone);
if (CAP_ENABLED(cap))
cap_disable(&capped_zones, cap);
}
/*
* Apply specified callback to all caps contained in the list `l'.
*/
static void
cap_walk(list_t *l, void (*cb)(cpucap_t *, int64_t))
{
static uint64_t cpucap_walk_gen;
cpucap_t *cap;
ASSERT(MUTEX_HELD(&caps_lock));
for (cap = list_head(l); cap != NULL; cap = list_next(l, cap)) {
(*cb)(cap, cpucap_walk_gen);
}
atomic_inc_64(&cpucap_walk_gen);
}
/*
* If cap limit is not reached, make one thread from wait queue runnable.
* The waitq_isempty check is performed without the waitq lock. If a new thread
* is placed on the waitq right after the check, it will be picked up during the
* next invocation of cap_poke_waitq().
*/
/* ARGSUSED */
static void
cap_poke_waitq(cpucap_t *cap, int64_t gen)
{
ASSERT(MUTEX_HELD(&caps_lock));
if (cap->cap_usage >= cap->cap_value) {
cap->cap_above++;
} else {
waitq_t *wq = &cap->cap_waitq;
cap->cap_below++;
if (!waitq_isempty(wq))
waitq_runone(wq);
}
}
/*
* The callback function called for every cap on capped_projects list.
* Decay cap usage by CAP_DECAY_FACTOR
* Add this cap project usage to its zone usage.
* Kick off a thread from the cap waitq if cap is not reached.
*/
static void
cap_project_usage_walker(cpucap_t *cap, int64_t gen)
{
zone_t *zone = cap->cap_zone;
hrtime_t cap_usage = cap->cap_usage;
ASSERT(MUTEX_HELD(&caps_lock));
ASSERT(cap->cap_project->kpj_cpucap == cap);
ASSERT(zone == cap->cap_project->kpj_zone);
ASSERT(CAP_ENABLED(cap));
/*
* Set or clear the CAP_REACHED flag based on the current usage.
* Only projects having their own caps are ever marked as CAP_REACHED.
*/
cap_poke_waitq(cap, 0);
/*
* Add project's CPU usage to our zone's CPU usage.
*/
if (ZONE_IS_CAPPED(zone)) {
cpucap_t *zcap = zone->zone_cpucap;
ASSERT(zcap->cap_zone == zone);
/*
* If we haven't reset this zone's usage during this clock tick
* yet, then do it now. The cap_gen field is used to check
* whether this is the first zone's project we see during this
* tick or a subsequent one.
*/
if (zcap->cap_gen != gen) {
if (zcap->cap_usage > zcap->cap_maxusage)
zcap->cap_maxusage = zcap->cap_usage;
zcap->cap_usage = 0;
zcap->cap_gen = gen;
}
DTRACE_PROBE2(cpucaps__zusage, cpucap_t *, zcap,
hrtime_t, cap_usage);
zcap->cap_usage += cap_usage;
/* Check for overflows */
if (zcap->cap_usage < 0)
zcap->cap_usage = MAX_USAGE - 1;
}
/*
* Decay project usage.
*/
disp_lock_enter(&cap->cap_usagelock);
cap->cap_usage -= ROUND_SCALE(cap_usage, CAP_DECAY_FACTOR);
disp_lock_exit(&cap->cap_usagelock);
}
/*
* On every clock tick walk the list of project caps and update the CPU usage.
* Also walk the list of zone caps checking whether any threads should
* transition from wait queue to run queue.
*
* This function gets called by the clock thread directly when there are any
* defined caps. The only lock that it grabs is caps_lock. Nothing else grabs
* caps_lock for long periods of time, so there should be almost no contention
* for it.
*/
static void
caps_update()
{
mutex_enter(&caps_lock);
cap_walk(&capped_projects, cap_project_usage_walker);
cap_walk(&capped_zones, cap_poke_waitq);
mutex_exit(&caps_lock);
}
/*
* The function is called for each project in a zone when the zone cap is
* modified. It enables project caps if zone cap is enabled and disables if the
* zone cap is disabled and project doesn't have its own cap.
*
* For each project that does not have cpucap structure allocated it allocates a
* new structure and assigns to kpj->cpu_cap. The allocation is performed
* without holding caps_lock to avoid using KM_SLEEP allocation with caps_lock
* held.
*/
static int
cap_project_zone_modify_walker(kproject_t *kpj, void *arg)
{
cpucap_t *project_cap = NULL;
cpucap_t *zone_cap = (cpucap_t *)arg;
ASSERT(zone_cap != NULL);
if (kpj->kpj_cpucap == NULL) {
/*
* This is the first time any cap was established for this
* project. Allocate a new cpucap structure for it.
*/
project_cap = cap_alloc();
}
mutex_enter(&caps_lock);
/*
* Double-check that kpj_cpucap is still NULL - now with caps_lock held
* and assign the newly allocated cpucap structure to it.
*/
if (kpj->kpj_cpucap == NULL) {
kpj->kpj_cpucap = project_cap;
} else if (project_cap != NULL) {
cap_free(project_cap);
}
project_cap = kpj->kpj_cpucap;
if (CAP_DISABLED(zone_cap)) {
/*
* Remove all projects in this zone without caps
* from the capped_projects list.
*/
if (project_cap->cap_value == MAX_USAGE) {
cap_project_disable(kpj);
}
} else if (CAP_DISABLED(project_cap)) {
/*
* Add the project to capped_projects list.
*/
ASSERT(project_cap->cap_value == 0);
cap_project_enable(kpj, MAX_USAGE);
}
mutex_exit(&caps_lock);
return (0);
}
/*
* Set zone cap to cap_val
* If cap_val is equal to NOCAP, disable zone cap.
*
* If this is the first time a cap is set on a zone, allocate cpucap structure
* without holding caps_lock to avoid KM_SLEEP allocation with caps_lock held.
*/
int
cpucaps_zone_set(zone_t *zone, rctl_qty_t cap_val)
{
cpucap_t *cap = NULL;
hrtime_t value;
if (cap_val == 0)
return (EINVAL);
ASSERT(cap_val <= MAXCAP);
if (cap_val > MAXCAP)
cap_val = MAXCAP;
/*
* Nothing to do if trying to disable a cap on a zone when caps are off
* or a zone which does not have a cap yet.
*/
if ((CPUCAPS_OFF() || !ZONE_IS_CAPPED(zone)) && (cap_val == NOCAP))
return (0);
if (zone->zone_cpucap == NULL)
cap = cap_alloc();
mutex_enter(&caps_lock);
if (cpucaps_busy) {
mutex_exit(&caps_lock);
return (EBUSY);
}
/*
* Double-check whether zone->zone_cpucap is NULL, now with caps_lock
* held. If it is still NULL, assign a newly allocated cpucap to it.
*/
if (zone->zone_cpucap == NULL) {
zone->zone_cpucap = cap;
} else if (cap != NULL) {
cap_free(cap);
}
cap = zone->zone_cpucap;
value = cap_val * cap_tick_cost;
if (value < 0)
value = MAX_USAGE;
/* Nothing to do if the value is staying the same */
if (value == cap->cap_value) {
mutex_exit(&caps_lock);
return (0);
}
/*
* Clear cap statistics since the cap value itself changes.
*/
cap->cap_above = cap->cap_below = 0;
if (cap_val == NOCAP) {
if (CAP_ENABLED(cap)) {
/*
* Remove cap for the zone
*/
cap_zone_disable(zone);
cpucaps_busy = B_TRUE;
mutex_exit(&caps_lock);
/*
* Disable caps for all project belonging to this zone
* unless they have their own cap.
*/
(void) project_walk_all(zone->zone_id,
cap_project_zone_modify_walker, cap);
mutex_enter(&caps_lock);
cpucaps_busy = B_FALSE;
}
} else if (CAP_DISABLED(cap)) {
/*
* Set a cap on a zone which previously was not capped.
*/
cap_zone_enable(zone, value);
cpucaps_busy = B_TRUE;
mutex_exit(&caps_lock);
/*
* Enable cap for all projects belonging to this zone.
*/
(void) project_walk_all(zone->zone_id,
cap_project_zone_modify_walker, cap);
mutex_enter(&caps_lock);
cpucaps_busy = B_FALSE;
} else {
/*
* No state transitions, just change the value
*/
cap->cap_value = value;
}
ASSERT(MUTEX_HELD(&caps_lock));
ASSERT(!cpucaps_busy);
mutex_exit(&caps_lock);
return (0);
}
/*
* The project is going away so disable its cap.
*/
void
cpucaps_project_remove(kproject_t *kpj)
{
mutex_enter(&caps_lock);
if (PROJECT_IS_CAPPED(kpj))
cap_project_disable(kpj);
if (kpj->kpj_cpucap != NULL) {
cap_free(kpj->kpj_cpucap);
kpj->kpj_cpucap = NULL;
}
mutex_exit(&caps_lock);
}
/*
* The zone is going away, so disable its cap.
*/
void
cpucaps_zone_remove(zone_t *zone)
{
mutex_enter(&caps_lock);
while (ZONE_IS_CAPPED(zone)) {
mutex_exit(&caps_lock);
(void) cpucaps_zone_set(zone, NOCAP);
mutex_enter(&caps_lock);
}
if (zone->zone_cpucap != NULL) {
cap_free(zone->zone_cpucap);
zone->zone_cpucap = NULL;
}
mutex_exit(&caps_lock);
}
/*
* New project was created. It should be put on the capped_projects list if
* its zone has a cap.
*/
void
cpucaps_project_add(kproject_t *kpj)
{
cpucap_t *cap = NULL;
if (CPUCAPS_OFF() || !ZONE_IS_CAPPED(kpj->kpj_zone))
return;
/*
* This project was never capped before, so allocate its cap structure.
*/
if (kpj->kpj_cpucap == NULL)
cap = cap_alloc();
mutex_enter(&caps_lock);
/*
* Double-check with caps_lock held
*/
if (kpj->kpj_cpucap == NULL) {
kpj->kpj_cpucap = cap;
} else if (cap != NULL) {
cap_free(cap);
}
if (ZONE_IS_CAPPED(kpj->kpj_zone))
cap_project_enable(kpj, MAX_USAGE);
mutex_exit(&caps_lock);
}
/*
* Set project cap to cap_val
* If cap_val is equal to NOCAP, disable project cap.
*
* If this is the first time a cap is set on a project, allocate cpucap
* structure without holding caps_lock to avoid KM_SLEEP allocation with
* caps_lock held.
*/
int
cpucaps_project_set(kproject_t *kpj, rctl_qty_t cap_val)
{
cpucap_t *cap = NULL;
hrtime_t value;
if (cap_val == 0)
return (EINVAL);
ASSERT(cap_val <= MAXCAP);
if (cap_val > MAXCAP)
cap_val = MAXCAP;
/*
* Nothing to do if trying to disable project cap and caps are not
* enabled or if trying to disable cap on a project that does not have
* cap enabled.
*/
if ((cap_val == NOCAP) && (CPUCAPS_OFF() || !PROJECT_IS_CAPPED(kpj)))
return (0);
if (kpj->kpj_cpucap == NULL) {
/*
* This project was never capped before, so allocate its cap
* structure.
*/
cap = cap_alloc();
}
mutex_enter(&caps_lock);
/*
* Double-check with caps_lock held.
*/
if (kpj->kpj_cpucap == NULL) {
kpj->kpj_cpucap = cap;
} else if (cap != NULL) {
cap_free(cap);
}
/*
* Get the actual pointer to the project cap.
*/
cap = kpj->kpj_cpucap;
value = cap_val * cap_tick_cost;
if (value < 0)
value = MAX_USAGE;
/*
* Nothing to do if the value is not changing
*/
if (value == cap->cap_value) {
mutex_exit(&caps_lock);
return (0);
}
/*
* Clear cap statistics since the cap value itself changes.
*/
cap->cap_above = cap->cap_below = 0;
cap->cap_maxusage = 0;
if (cap_val != NOCAP) {
/*
* Enable this cap if it is not already enabled.
*/
if (CAP_DISABLED(cap))
cap_project_enable(kpj, value);
else
cap->cap_value = value;
} else if (CAP_ENABLED(cap)) {
/*
* User requested to drop a cap on the project. If it is part of
* capped zone, keep the cap and set the value to MAX_USAGE,
* otherwise disable the cap.
*/
if (ZONE_IS_CAPPED(kpj->kpj_zone)) {
cap->cap_value = MAX_USAGE;
} else {
cap_project_disable(kpj);
}
}
mutex_exit(&caps_lock);
return (0);
}
/*
* Get cap usage.
*/
static rctl_qty_t
cap_get(cpucap_t *cap)
{
return (cap != NULL ? (rctl_qty_t)(cap->cap_usage / cap_tick_cost) : 0);
}
/*
* Get current project usage.
*/
rctl_qty_t
cpucaps_project_get(kproject_t *kpj)
{
return (cap_get(kpj->kpj_cpucap));
}
/*
* Get current zone usage.
*/
rctl_qty_t
cpucaps_zone_get(zone_t *zone)
{
return (cap_get(zone->zone_cpucap));
}
/*
* Charge project of thread t the time thread t spent on CPU since previously
* adjusted.
*
* Record the current on-CPU time in the csc structure.
*
* Do not adjust for more than one tick worth of time.
*
* It is possible that the project cap is being disabled while this routine is
* executed. This should not cause any issues since the association between the
* thread and its project is protected by thread lock.
*/
static void
caps_charge_adjust(kthread_id_t t, caps_sc_t *csc)
{
kproject_t *kpj = ttoproj(t);
hrtime_t new_usage;
hrtime_t usage_delta;
ASSERT(THREAD_LOCK_HELD(t));
ASSERT(kpj->kpj_cpucap != NULL);
/* Get on-CPU time since birth of a thread */
new_usage = mstate_thread_onproc_time(t);
/* Time spent on CPU since last checked */
usage_delta = new_usage - csc->csc_cputime;
/* Save the accumulated on-CPU time */
csc->csc_cputime = new_usage;
/* Charge at most one tick worth of on-CPU time */
if (usage_delta > cap_tick_cost)
usage_delta = cap_tick_cost;
/* Add usage_delta to the project usage value. */
if (usage_delta > 0) {
cpucap_t *cap = kpj->kpj_cpucap;
DTRACE_PROBE2(cpucaps__project__charge,
kthread_id_t, t, hrtime_t, usage_delta);
disp_lock_enter_high(&cap->cap_usagelock);
cap->cap_usage += usage_delta;
/* Check for overflows */
if (cap->cap_usage < 0)
cap->cap_usage = MAX_USAGE - 1;
disp_lock_exit_high(&cap->cap_usagelock);
/*
* cap_maxusage is only kept for observability. Move it outside
* the lock to reduce the time spent while holding the lock.
*/
if (cap->cap_usage > cap->cap_maxusage)
cap->cap_maxusage = cap->cap_usage;
}
}
/*
* Charge thread's project and return True if project or zone should be
* penalized because its project or zone is exceeding its cap. Also sets
* TS_PROJWAITQ or TS_ZONEWAITQ in this case.
*
* It is possible that the project cap is being disabled while this routine is
* executed. This should not cause any issues since the association between the
* thread and its project is protected by thread lock. It will still set
* TS_PROJECTWAITQ/TS_ZONEWAITQ in this case but cpucaps_enforce will not place
* anything on the blocked wait queue.
*
*/
boolean_t
cpucaps_charge(kthread_id_t t, caps_sc_t *csc, cpucaps_charge_t charge_type)
{
kproject_t *kpj = ttoproj(t);
klwp_t *lwp = t->t_lwp;
zone_t *zone;
cpucap_t *project_cap;
boolean_t rc = B_FALSE;
ASSERT(THREAD_LOCK_HELD(t));
/* Nothing to do for projects that are not capped. */
if (lwp == NULL || !PROJECT_IS_CAPPED(kpj))
return (B_FALSE);
caps_charge_adjust(t, csc);
/*
* The caller only requested to charge the project usage, no enforcement
* part.
*/
if (charge_type == CPUCAPS_CHARGE_ONLY)
return (B_FALSE);
project_cap = kpj->kpj_cpucap;
if (project_cap->cap_usage >= project_cap->cap_value) {
t->t_schedflag |= TS_PROJWAITQ;
rc = B_TRUE;
} else if (t->t_schedflag & TS_PROJWAITQ) {
t->t_schedflag &= ~TS_PROJWAITQ;
}
zone = ttozone(t);
if (!ZONE_IS_CAPPED(zone)) {
if (t->t_schedflag & TS_ZONEWAITQ)
t->t_schedflag &= ~TS_ZONEWAITQ;
} else {
cpucap_t *zone_cap = zone->zone_cpucap;
if (zone_cap->cap_usage >= zone_cap->cap_value) {
t->t_schedflag |= TS_ZONEWAITQ;
rc = B_TRUE;
} else if (t->t_schedflag & TS_ZONEWAITQ) {
t->t_schedflag &= ~TS_ZONEWAITQ;
}
}
return (rc);
}
/*
* Enforce CPU caps. If got preempted in the user-land, we know that thread does
* not hold any kernel locks, so enqueue ourselves on the waitq, if needed.
*
* CPU Caps are only enforced for user threads.
*
* Threads flagged with TS_PROJWAITQ are placed on their project wait queues and
* threads marked with TS_ZONEWAITQ are placed on their zone wait queue.
*
* It is possible that by the time we enter cpucaps_enforce() the cap is already
* disabled. In this case waitq_enqueue() fails and doesn't enqueue anything. We
* still clear TS_PROJWAITQ/TS_ZONEWAITQ flags in this case since they no longer
* apply.
*/
boolean_t
cpucaps_enforce(kthread_t *t)
{
klwp_t *lwp = t->t_lwp;
ASSERT(THREAD_LOCK_HELD(t));
if (lwp != NULL && lwp->lwp_state == LWP_USER) {
if (t->t_schedflag & TS_PROJWAITQ) {
ASSERT(ttoproj(t)->kpj_cpucap != NULL);
t->t_schedflag &= ~TS_ANYWAITQ;
if (waitq_enqueue(&(ttoproj(t)->kpj_cpucap->cap_waitq),
t)) {
return (B_TRUE);
}
}
if (t->t_schedflag & TS_ZONEWAITQ) {
ASSERT(ttozone(t)->zone_cpucap != NULL);
t->t_schedflag &= ~TS_ZONEWAITQ;
if (waitq_enqueue(&(ttozone(t)->zone_cpucap->cap_waitq),
t)) {
return (B_TRUE);
}
}
}
/*
* The thread is not enqueued on the wait queue.
*/
return (B_FALSE);
}
/*
* Convert internal cap statistics into values exported by cap kstat.
*/
static int
cap_kstat_update(kstat_t *ksp, int rw)
{
struct cap_kstat *capsp = &cap_kstat;
cpucap_t *cap = ksp->ks_private;
clock_t tick_sec = SEC_TO_TICK(1);
char *zonename = cap->cap_zone->zone_name;
if (rw == KSTAT_WRITE)
return (EACCES);
capsp->cap_value.value.ui64 =
ROUND_SCALE(cap->cap_value, cap_tick_cost);
capsp->cap_usage.value.ui64 =
ROUND_SCALE(cap->cap_usage, cap_tick_cost);
capsp->cap_maxusage.value.ui64 =
ROUND_SCALE(cap->cap_maxusage, cap_tick_cost);
capsp->cap_nwait.value.ui64 = cap->cap_waitq.wq_count;
capsp->cap_below.value.ui64 = ROUND_SCALE(cap->cap_below, tick_sec);
capsp->cap_above.value.ui64 = ROUND_SCALE(cap->cap_above, tick_sec);
kstat_named_setstr(&capsp->cap_zonename, zonename);
return (0);
}