NetBSD-5.0.2/sys/kern/vfs_cache.c
/* $NetBSD: vfs_cache.c,v 1.80 2008/10/25 14:20:17 yamt Exp $ */
/*-
* Copyright (c) 2008 The NetBSD Foundation, Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
* TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
/*
* Copyright (c) 1989, 1993
* The Regents of the University of California. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* @(#)vfs_cache.c 8.3 (Berkeley) 8/22/94
*/
#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: vfs_cache.c,v 1.80 2008/10/25 14:20:17 yamt Exp $");
#include "opt_ddb.h"
#include "opt_revcache.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/time.h>
#include <sys/mount.h>
#include <sys/vnode.h>
#include <sys/namei.h>
#include <sys/errno.h>
#include <sys/pool.h>
#include <sys/mutex.h>
#include <sys/atomic.h>
#include <sys/kthread.h>
#include <sys/kernel.h>
#include <sys/cpu.h>
#include <sys/evcnt.h>
#define NAMECACHE_ENTER_REVERSE
/*
* Name caching works as follows:
*
* Names found by directory scans are retained in a cache
* for future reference. It is managed LRU, so frequently
* used names will hang around. Cache is indexed by hash value
* obtained from (dvp, name) where dvp refers to the directory
* containing name.
*
* For simplicity (and economy of storage), names longer than
* a maximum length of NCHNAMLEN are not cached; they occur
* infrequently in any case, and are almost never of interest.
*
* Upon reaching the last segment of a path, if the reference
* is for DELETE, or NOCACHE is set (rewrite), and the
* name is located in the cache, it will be dropped.
* The entry is dropped also when it was not possible to lock
* the cached vnode, either because vget() failed or the generation
* number has changed while waiting for the lock.
*/
/*
* Per-cpu namecache data.
*/
struct nchcpu {
kmutex_t cpu_lock;
struct nchstats cpu_stats;
};
/*
* Structures associated with name cacheing.
*/
LIST_HEAD(nchashhead, namecache) *nchashtbl;
u_long nchash; /* size of hash table - 1 */
#define NCHASH(cnp, dvp) \
(((cnp)->cn_hash ^ ((uintptr_t)(dvp) >> 3)) & nchash)
LIST_HEAD(ncvhashhead, namecache) *ncvhashtbl;
u_long ncvhash; /* size of hash table - 1 */
#define NCVHASH(vp) (((uintptr_t)(vp) >> 3) & ncvhash)
long numcache; /* number of cache entries allocated */
static u_int cache_gcpend; /* number of entries pending GC */
static void *cache_gcqueue; /* garbage collection queue */
TAILQ_HEAD(, namecache) nclruhead = /* LRU chain */
TAILQ_HEAD_INITIALIZER(nclruhead);
#define COUNT(c,x) (c.x++)
struct nchstats nchstats; /* cache effectiveness statistics */
static pool_cache_t namecache_cache;
MALLOC_DEFINE(M_CACHE, "namecache", "Dynamically allocated cache entries");
int cache_lowat = 95;
int cache_hiwat = 98;
int cache_hottime = 5; /* number of seconds */
int doingcache = 1; /* 1 => enable the cache */
static struct evcnt cache_ev_scan;
static struct evcnt cache_ev_gc;
static struct evcnt cache_ev_over;
static struct evcnt cache_ev_under;
static struct evcnt cache_ev_forced;
/* A single lock to serialize modifications. */
static kmutex_t *namecache_lock;
static void cache_invalidate(struct namecache *);
static inline struct namecache *cache_lookup_entry(
const struct vnode *, const struct componentname *);
static void cache_thread(void *);
static void cache_invalidate(struct namecache *);
static void cache_disassociate(struct namecache *);
static void cache_reclaim(void);
static int cache_ctor(void *, void *, int);
static void cache_dtor(void *, void *);
/*
* Invalidate a cache entry and enqueue it for garbage collection.
*/
static void
cache_invalidate(struct namecache *ncp)
{
void *head;
KASSERT(mutex_owned(&ncp->nc_lock));
if (ncp->nc_dvp != NULL) {
ncp->nc_vp = NULL;
ncp->nc_dvp = NULL;
do {
head = cache_gcqueue;
ncp->nc_gcqueue = head;
} while (atomic_cas_ptr(&cache_gcqueue, head, ncp) != head);
atomic_inc_uint(&cache_gcpend);
}
}
/*
* Disassociate a namecache entry from any vnodes it is attached to,
* and remove from the global LRU list.
*/
static void
cache_disassociate(struct namecache *ncp)
{
KASSERT(mutex_owned(namecache_lock));
KASSERT(ncp->nc_dvp == NULL);
if (ncp->nc_lru.tqe_prev != NULL) {
TAILQ_REMOVE(&nclruhead, ncp, nc_lru);
ncp->nc_lru.tqe_prev = NULL;
}
if (ncp->nc_vhash.le_prev != NULL) {
LIST_REMOVE(ncp, nc_vhash);
ncp->nc_vhash.le_prev = NULL;
}
if (ncp->nc_vlist.le_prev != NULL) {
LIST_REMOVE(ncp, nc_vlist);
ncp->nc_vlist.le_prev = NULL;
}
if (ncp->nc_dvlist.le_prev != NULL) {
LIST_REMOVE(ncp, nc_dvlist);
ncp->nc_dvlist.le_prev = NULL;
}
}
/*
* Lock all CPUs to prevent any cache lookup activity. Conceptually,
* this locks out all "readers".
*/
static void
cache_lock_cpus(void)
{
CPU_INFO_ITERATOR cii;
struct cpu_info *ci;
struct nchcpu *cpup;
long *s, *d, *m;
for (CPU_INFO_FOREACH(cii, ci)) {
cpup = ci->ci_data.cpu_nch;
mutex_enter(&cpup->cpu_lock);
/* Collate statistics. */
d = (long *)&nchstats;
s = (long *)&cpup->cpu_stats;
m = s + sizeof(nchstats) / sizeof(long);
for (; s < m; s++, d++) {
*d += *s;
*s = 0;
}
}
}
/*
* Release all CPU locks.
*/
static void
cache_unlock_cpus(void)
{
CPU_INFO_ITERATOR cii;
struct cpu_info *ci;
struct nchcpu *cpup;
for (CPU_INFO_FOREACH(cii, ci)) {
cpup = ci->ci_data.cpu_nch;
mutex_exit(&cpup->cpu_lock);
}
}
/*
* Find a single cache entry and return it locked. 'namecache_lock' or
* at least one of the per-CPU locks must be held.
*/
static struct namecache *
cache_lookup_entry(const struct vnode *dvp, const struct componentname *cnp)
{
struct nchashhead *ncpp;
struct namecache *ncp;
ncpp = &nchashtbl[NCHASH(cnp, dvp)];
LIST_FOREACH(ncp, ncpp, nc_hash) {
if (ncp->nc_dvp != dvp ||
ncp->nc_nlen != cnp->cn_namelen ||
memcmp(ncp->nc_name, cnp->cn_nameptr, (u_int)ncp->nc_nlen))
continue;
mutex_enter(&ncp->nc_lock);
if (__predict_true(ncp->nc_dvp == dvp)) {
ncp->nc_hittime = hardclock_ticks;
return ncp;
}
/* Raced: entry has been nullified. */
mutex_exit(&ncp->nc_lock);
}
return NULL;
}
/*
* Look for a the name in the cache. We don't do this
* if the segment name is long, simply so the cache can avoid
* holding long names (which would either waste space, or
* add greatly to the complexity).
*
* Lookup is called with ni_dvp pointing to the directory to search,
* ni_ptr pointing to the name of the entry being sought, ni_namelen
* tells the length of the name, and ni_hash contains a hash of
* the name. If the lookup succeeds, the vnode is locked, stored in ni_vp
* and a status of zero is returned. If the locking fails for whatever
* reason, the vnode is unlocked and the error is returned to caller.
* If the lookup determines that the name does not exist (negative cacheing),
* a status of ENOENT is returned. If the lookup fails, a status of -1
* is returned.
*/
int
cache_lookup(struct vnode *dvp, struct vnode **vpp, struct componentname *cnp)
{
struct namecache *ncp;
struct vnode *vp;
struct nchcpu *cpup;
int error;
if (__predict_false(!doingcache)) {
cnp->cn_flags &= ~MAKEENTRY;
*vpp = NULL;
return -1;
}
cpup = curcpu()->ci_data.cpu_nch;
mutex_enter(&cpup->cpu_lock);
if (__predict_false(cnp->cn_namelen > NCHNAMLEN)) {
COUNT(cpup->cpu_stats, ncs_long);
cnp->cn_flags &= ~MAKEENTRY;
mutex_exit(&cpup->cpu_lock);
*vpp = NULL;
return -1;
}
ncp = cache_lookup_entry(dvp, cnp);
if (__predict_false(ncp == NULL)) {
COUNT(cpup->cpu_stats, ncs_miss);
mutex_exit(&cpup->cpu_lock);
*vpp = NULL;
return -1;
}
if ((cnp->cn_flags & MAKEENTRY) == 0) {
COUNT(cpup->cpu_stats, ncs_badhits);
/*
* Last component and we are renaming or deleting,
* the cache entry is invalid, or otherwise don't
* want cache entry to exist.
*/
cache_invalidate(ncp);
mutex_exit(&ncp->nc_lock);
mutex_exit(&cpup->cpu_lock);
*vpp = NULL;
return -1;
} else if (ncp->nc_vp == NULL) {
/*
* Restore the ISWHITEOUT flag saved earlier.
*/
KASSERT((ncp->nc_flags & ~ISWHITEOUT) == 0);
cnp->cn_flags |= ncp->nc_flags;
if (__predict_true(cnp->cn_nameiop != CREATE ||
(cnp->cn_flags & ISLASTCN) == 0)) {
COUNT(cpup->cpu_stats, ncs_neghits);
mutex_exit(&ncp->nc_lock);
mutex_exit(&cpup->cpu_lock);
return ENOENT;
} else {
COUNT(cpup->cpu_stats, ncs_badhits);
/*
* Last component and we are renaming or
* deleting, the cache entry is invalid,
* or otherwise don't want cache entry to
* exist.
*/
cache_invalidate(ncp);
mutex_exit(&ncp->nc_lock);
mutex_exit(&cpup->cpu_lock);
*vpp = NULL;
return -1;
}
}
vp = ncp->nc_vp;
if (vtryget(vp)) {
mutex_exit(&ncp->nc_lock);
mutex_exit(&cpup->cpu_lock);
} else {
mutex_enter(&vp->v_interlock);
mutex_exit(&ncp->nc_lock);
mutex_exit(&cpup->cpu_lock);
error = vget(vp, LK_NOWAIT | LK_INTERLOCK);
if (error) {
KASSERT(error == EBUSY);
/*
* This vnode is being cleaned out.
* XXX badhits?
*/
COUNT(cpup->cpu_stats, ncs_falsehits);
*vpp = NULL;
return -1;
}
}
#ifdef DEBUG
/*
* since we released nb->nb_lock,
* we can't use this pointer any more.
*/
ncp = NULL;
#endif /* DEBUG */
if (vp == dvp) { /* lookup on "." */
error = 0;
} else if (cnp->cn_flags & ISDOTDOT) {
VOP_UNLOCK(dvp, 0);
error = vn_lock(vp, LK_EXCLUSIVE);
vn_lock(dvp, LK_EXCLUSIVE | LK_RETRY);
} else {
error = vn_lock(vp, LK_EXCLUSIVE);
}
/*
* Check that the lock succeeded.
*/
if (error) {
/* Unlocked, but only for stats. */
COUNT(cpup->cpu_stats, ncs_badhits);
vrele(vp);
*vpp = NULL;
return -1;
}
/* Unlocked, but only for stats. */
COUNT(cpup->cpu_stats, ncs_goodhits);
*vpp = vp;
return 0;
}
int
cache_lookup_raw(struct vnode *dvp, struct vnode **vpp,
struct componentname *cnp)
{
struct namecache *ncp;
struct vnode *vp;
struct nchcpu *cpup;
int error;
if (__predict_false(!doingcache)) {
cnp->cn_flags &= ~MAKEENTRY;
*vpp = NULL;
return (-1);
}
cpup = curcpu()->ci_data.cpu_nch;
mutex_enter(&cpup->cpu_lock);
if (__predict_false(cnp->cn_namelen > NCHNAMLEN)) {
COUNT(cpup->cpu_stats, ncs_long);
cnp->cn_flags &= ~MAKEENTRY;
mutex_exit(&cpup->cpu_lock);
*vpp = NULL;
return -1;
}
ncp = cache_lookup_entry(dvp, cnp);
if (__predict_false(ncp == NULL)) {
COUNT(cpup->cpu_stats, ncs_miss);
mutex_exit(&cpup->cpu_lock);
*vpp = NULL;
return -1;
}
vp = ncp->nc_vp;
if (vp == NULL) {
/*
* Restore the ISWHITEOUT flag saved earlier.
*/
KASSERT((ncp->nc_flags & ~ISWHITEOUT) == 0);
cnp->cn_flags |= ncp->nc_flags;
COUNT(cpup->cpu_stats, ncs_neghits);
mutex_exit(&ncp->nc_lock);
mutex_exit(&cpup->cpu_lock);
return ENOENT;
}
if (vtryget(vp)) {
mutex_exit(&ncp->nc_lock);
mutex_exit(&cpup->cpu_lock);
} else {
mutex_enter(&vp->v_interlock);
mutex_exit(&ncp->nc_lock);
mutex_exit(&cpup->cpu_lock);
error = vget(vp, LK_NOWAIT | LK_INTERLOCK);
if (error) {
KASSERT(error == EBUSY);
/*
* This vnode is being cleaned out.
* XXX badhits?
*/
COUNT(cpup->cpu_stats, ncs_falsehits);
*vpp = NULL;
return -1;
}
}
/* Unlocked, but only for stats. */
COUNT(cpup->cpu_stats, ncs_goodhits); /* XXX can be "badhits" */
*vpp = vp;
return 0;
}
/*
* Scan cache looking for name of directory entry pointing at vp.
*
* Fill in dvpp.
*
* If bufp is non-NULL, also place the name in the buffer which starts
* at bufp, immediately before *bpp, and move bpp backwards to point
* at the start of it. (Yes, this is a little baroque, but it's done
* this way to cater to the whims of getcwd).
*
* Returns 0 on success, -1 on cache miss, positive errno on failure.
*/
int
cache_revlookup(struct vnode *vp, struct vnode **dvpp, char **bpp, char *bufp)
{
struct namecache *ncp;
struct vnode *dvp;
struct ncvhashhead *nvcpp;
char *bp;
if (!doingcache)
goto out;
nvcpp = &ncvhashtbl[NCVHASH(vp)];
mutex_enter(namecache_lock);
LIST_FOREACH(ncp, nvcpp, nc_vhash) {
mutex_enter(&ncp->nc_lock);
if (ncp->nc_vp == vp &&
(dvp = ncp->nc_dvp) != NULL &&
dvp != vp) { /* avoid pesky . entries.. */
#ifdef DIAGNOSTIC
if (ncp->nc_nlen == 1 &&
ncp->nc_name[0] == '.')
panic("cache_revlookup: found entry for .");
if (ncp->nc_nlen == 2 &&
ncp->nc_name[0] == '.' &&
ncp->nc_name[1] == '.')
panic("cache_revlookup: found entry for ..");
#endif
COUNT(nchstats, ncs_revhits);
if (bufp) {
bp = *bpp;
bp -= ncp->nc_nlen;
if (bp <= bufp) {
*dvpp = NULL;
mutex_exit(&ncp->nc_lock);
mutex_exit(namecache_lock);
return (ERANGE);
}
memcpy(bp, ncp->nc_name, ncp->nc_nlen);
*bpp = bp;
}
/* XXX MP: how do we know dvp won't evaporate? */
*dvpp = dvp;
mutex_exit(&ncp->nc_lock);
mutex_exit(namecache_lock);
return (0);
}
mutex_exit(&ncp->nc_lock);
}
COUNT(nchstats, ncs_revmiss);
mutex_exit(namecache_lock);
out:
*dvpp = NULL;
return (-1);
}
/*
* Add an entry to the cache
*/
void
cache_enter(struct vnode *dvp, struct vnode *vp, struct componentname *cnp)
{
struct namecache *ncp;
struct namecache *oncp;
struct nchashhead *ncpp;
struct ncvhashhead *nvcpp;
#ifdef DIAGNOSTIC
if (cnp->cn_namelen > NCHNAMLEN)
panic("cache_enter: name too long");
#endif
if (!doingcache)
return;
if (numcache > desiredvnodes) {
mutex_enter(namecache_lock);
cache_ev_forced.ev_count++;
cache_reclaim();
mutex_exit(namecache_lock);
}
ncp = pool_cache_get(namecache_cache, PR_WAITOK);
mutex_enter(namecache_lock);
numcache++;
/*
* Concurrent lookups in the same directory may race for a
* cache entry. if there's a duplicated entry, free it.
*/
oncp = cache_lookup_entry(dvp, cnp);
if (oncp) {
cache_invalidate(oncp);
mutex_exit(&oncp->nc_lock);
}
/* Grab the vnode we just found. */
mutex_enter(&ncp->nc_lock);
ncp->nc_vp = vp;
ncp->nc_flags = 0;
ncp->nc_hittime = 0;
ncp->nc_gcqueue = NULL;
if (vp == NULL) {
/*
* For negative hits, save the ISWHITEOUT flag so we can
* restore it later when the cache entry is used again.
*/
ncp->nc_flags = cnp->cn_flags & ISWHITEOUT;
}
/* Fill in cache info. */
ncp->nc_dvp = dvp;
LIST_INSERT_HEAD(&dvp->v_dnclist, ncp, nc_dvlist);
if (vp)
LIST_INSERT_HEAD(&vp->v_nclist, ncp, nc_vlist);
else {
ncp->nc_vlist.le_prev = NULL;
ncp->nc_vlist.le_next = NULL;
}
ncp->nc_nlen = cnp->cn_namelen;
TAILQ_INSERT_TAIL(&nclruhead, ncp, nc_lru);
memcpy(ncp->nc_name, cnp->cn_nameptr, (unsigned)ncp->nc_nlen);
ncpp = &nchashtbl[NCHASH(cnp, dvp)];
/*
* Flush updates before making visible in table. No need for a
* memory barrier on the other side: to see modifications the
* list must be followed, meaning a dependent pointer load.
* The below is LIST_INSERT_HEAD() inlined, with the memory
* barrier included in the correct place.
*/
if ((ncp->nc_hash.le_next = ncpp->lh_first) != NULL)
ncpp->lh_first->nc_hash.le_prev = &ncp->nc_hash.le_next;
ncp->nc_hash.le_prev = &ncpp->lh_first;
membar_producer();
ncpp->lh_first = ncp;
ncp->nc_vhash.le_prev = NULL;
ncp->nc_vhash.le_next = NULL;
/*
* Create reverse-cache entries (used in getcwd) for directories.
* (and in linux procfs exe node)
*/
if (vp != NULL &&
vp != dvp &&
#ifndef NAMECACHE_ENTER_REVERSE
vp->v_type == VDIR &&
#endif
(ncp->nc_nlen > 2 ||
(ncp->nc_nlen > 1 && ncp->nc_name[1] != '.') ||
(/* ncp->nc_nlen > 0 && */ ncp->nc_name[0] != '.'))) {
nvcpp = &ncvhashtbl[NCVHASH(vp)];
LIST_INSERT_HEAD(nvcpp, ncp, nc_vhash);
}
mutex_exit(&ncp->nc_lock);
mutex_exit(namecache_lock);
}
/*
* Name cache initialization, from vfs_init() when we are booting
*/
void
nchinit(void)
{
int error;
namecache_cache = pool_cache_init(sizeof(struct namecache),
coherency_unit, 0, 0, "ncache", NULL, IPL_NONE, cache_ctor,
cache_dtor, NULL);
KASSERT(namecache_cache != NULL);
namecache_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE);
nchashtbl = hashinit(desiredvnodes, HASH_LIST, true, &nchash);
ncvhashtbl =
#ifdef NAMECACHE_ENTER_REVERSE
hashinit(desiredvnodes, HASH_LIST, true, &ncvhash);
#else
hashinit(desiredvnodes/8, HASH_LIST, true, &ncvhash);
#endif
error = kthread_create(PRI_VM, KTHREAD_MPSAFE, NULL, cache_thread,
NULL, NULL, "cachegc");
if (error != 0)
panic("nchinit %d", error);
evcnt_attach_dynamic(&cache_ev_scan, EVCNT_TYPE_MISC, NULL,
"namecache", "entries scanned");
evcnt_attach_dynamic(&cache_ev_gc, EVCNT_TYPE_MISC, NULL,
"namecache", "entries collected");
evcnt_attach_dynamic(&cache_ev_over, EVCNT_TYPE_MISC, NULL,
"namecache", "over scan target");
evcnt_attach_dynamic(&cache_ev_under, EVCNT_TYPE_MISC, NULL,
"namecache", "under scan target");
evcnt_attach_dynamic(&cache_ev_forced, EVCNT_TYPE_MISC, NULL,
"namecache", "forced reclaims");
}
static int
cache_ctor(void *arg, void *obj, int flag)
{
struct namecache *ncp;
ncp = obj;
mutex_init(&ncp->nc_lock, MUTEX_DEFAULT, IPL_NONE);
return 0;
}
static void
cache_dtor(void *arg, void *obj)
{
struct namecache *ncp;
ncp = obj;
mutex_destroy(&ncp->nc_lock);
}
/*
* Called once for each CPU in the system as attached.
*/
void
cache_cpu_init(struct cpu_info *ci)
{
struct nchcpu *cpup;
size_t sz;
sz = roundup2(sizeof(*cpup), coherency_unit) + coherency_unit;
cpup = kmem_zalloc(sz, KM_SLEEP);
cpup = (void *)roundup2((uintptr_t)cpup, coherency_unit);
mutex_init(&cpup->cpu_lock, MUTEX_DEFAULT, IPL_NONE);
ci->ci_data.cpu_nch = cpup;
}
/*
* Name cache reinitialization, for when the maximum number of vnodes increases.
*/
void
nchreinit(void)
{
struct namecache *ncp;
struct nchashhead *oldhash1, *hash1;
struct ncvhashhead *oldhash2, *hash2;
u_long i, oldmask1, oldmask2, mask1, mask2;
hash1 = hashinit(desiredvnodes, HASH_LIST, true, &mask1);
hash2 =
#ifdef NAMECACHE_ENTER_REVERSE
hashinit(desiredvnodes, HASH_LIST, true, &mask2);
#else
hashinit(desiredvnodes/8, HASH_LIST, true, &mask2);
#endif
mutex_enter(namecache_lock);
cache_lock_cpus();
oldhash1 = nchashtbl;
oldmask1 = nchash;
nchashtbl = hash1;
nchash = mask1;
oldhash2 = ncvhashtbl;
oldmask2 = ncvhash;
ncvhashtbl = hash2;
ncvhash = mask2;
for (i = 0; i <= oldmask1; i++) {
while ((ncp = LIST_FIRST(&oldhash1[i])) != NULL) {
LIST_REMOVE(ncp, nc_hash);
ncp->nc_hash.le_prev = NULL;
}
}
for (i = 0; i <= oldmask2; i++) {
while ((ncp = LIST_FIRST(&oldhash2[i])) != NULL) {
LIST_REMOVE(ncp, nc_vhash);
ncp->nc_vhash.le_prev = NULL;
}
}
cache_unlock_cpus();
mutex_exit(namecache_lock);
hashdone(oldhash1, HASH_LIST, oldmask1);
hashdone(oldhash2, HASH_LIST, oldmask2);
}
/*
* Cache flush, a particular vnode; called when a vnode is renamed to
* hide entries that would now be invalid
*/
void
cache_purge1(struct vnode *vp, const struct componentname *cnp, int flags)
{
struct namecache *ncp, *ncnext;
mutex_enter(namecache_lock);
if (flags & PURGE_PARENTS) {
for (ncp = LIST_FIRST(&vp->v_nclist); ncp != NULL;
ncp = ncnext) {
ncnext = LIST_NEXT(ncp, nc_vlist);
mutex_enter(&ncp->nc_lock);
cache_invalidate(ncp);
mutex_exit(&ncp->nc_lock);
cache_disassociate(ncp);
}
}
if (flags & PURGE_CHILDREN) {
for (ncp = LIST_FIRST(&vp->v_dnclist); ncp != NULL;
ncp = ncnext) {
ncnext = LIST_NEXT(ncp, nc_dvlist);
mutex_enter(&ncp->nc_lock);
cache_invalidate(ncp);
mutex_exit(&ncp->nc_lock);
cache_disassociate(ncp);
}
}
if (cnp != NULL) {
ncp = cache_lookup_entry(vp, cnp);
if (ncp) {
cache_invalidate(ncp);
cache_disassociate(ncp);
mutex_exit(&ncp->nc_lock);
}
}
mutex_exit(namecache_lock);
}
/*
* Cache flush, a whole filesystem; called when filesys is umounted to
* remove entries that would now be invalid.
*/
void
cache_purgevfs(struct mount *mp)
{
struct namecache *ncp, *nxtcp;
mutex_enter(namecache_lock);
for (ncp = TAILQ_FIRST(&nclruhead); ncp != NULL; ncp = nxtcp) {
nxtcp = TAILQ_NEXT(ncp, nc_lru);
mutex_enter(&ncp->nc_lock);
if (ncp->nc_dvp != NULL && ncp->nc_dvp->v_mount == mp) {
/* Free the resources we had. */
cache_invalidate(ncp);
cache_disassociate(ncp);
}
mutex_exit(&ncp->nc_lock);
}
cache_reclaim();
mutex_exit(namecache_lock);
}
/*
* Scan global list invalidating entries until we meet a preset target.
* Prefer to invalidate entries that have not scored a hit within
* cache_hottime seconds. We sort the LRU list only for this routine's
* benefit.
*/
static void
cache_prune(int incache, int target)
{
struct namecache *ncp, *nxtcp, *sentinel;
int items, recent, tryharder;
KASSERT(mutex_owned(namecache_lock));
items = 0;
tryharder = 0;
recent = hardclock_ticks - hz * cache_hottime;
sentinel = NULL;
for (ncp = TAILQ_FIRST(&nclruhead); ncp != NULL; ncp = nxtcp) {
if (incache <= target)
break;
items++;
nxtcp = TAILQ_NEXT(ncp, nc_lru);
if (ncp->nc_dvp == NULL)
continue;
if (ncp == sentinel) {
/*
* If we looped back on ourself, then ignore
* recent entries and purge whatever we find.
*/
tryharder = 1;
}
if (!tryharder && ncp->nc_hittime > recent) {
if (sentinel == NULL)
sentinel = ncp;
TAILQ_REMOVE(&nclruhead, ncp, nc_lru);
TAILQ_INSERT_TAIL(&nclruhead, ncp, nc_lru);
continue;
}
mutex_enter(&ncp->nc_lock);
if (ncp->nc_dvp != NULL) {
cache_invalidate(ncp);
cache_disassociate(ncp);
incache--;
}
mutex_exit(&ncp->nc_lock);
}
cache_ev_scan.ev_count += items;
}
/*
* Collect dead cache entries from all CPUs and garbage collect.
*/
static void
cache_reclaim(void)
{
struct namecache *ncp, *next;
int items;
KASSERT(mutex_owned(namecache_lock));
/*
* If the number of extant entries not awaiting garbage collection
* exceeds the high water mark, then reclaim stale entries until we
* reach our low water mark.
*/
items = numcache - cache_gcpend;
if (items > (uint64_t)desiredvnodes * cache_hiwat / 100) {
cache_prune(items, (int)((uint64_t)desiredvnodes *
cache_lowat / 100));
cache_ev_over.ev_count++;
} else
cache_ev_under.ev_count++;
/*
* Stop forward lookup activity on all CPUs and garbage collect dead
* entries.
*/
cache_lock_cpus();
ncp = cache_gcqueue;
cache_gcqueue = NULL;
items = cache_gcpend;
cache_gcpend = 0;
while (ncp != NULL) {
next = ncp->nc_gcqueue;
cache_disassociate(ncp);
KASSERT(ncp->nc_dvp == NULL);
if (ncp->nc_hash.le_prev != NULL) {
LIST_REMOVE(ncp, nc_hash);
ncp->nc_hash.le_prev = NULL;
}
pool_cache_put(namecache_cache, ncp);
ncp = next;
}
cache_unlock_cpus();
numcache -= items;
cache_ev_gc.ev_count += items;
}
/*
* Cache maintainence thread, awakening once per second to:
*
* => keep number of entries below the high water mark
* => sort pseudo-LRU list
* => garbage collect dead entries
*/
static void
cache_thread(void *arg)
{
mutex_enter(namecache_lock);
for (;;) {
cache_reclaim();
kpause("cachegc", false, hz, namecache_lock);
}
}
#ifdef DDB
void
namecache_print(struct vnode *vp, void (*pr)(const char *, ...))
{
struct vnode *dvp = NULL;
struct namecache *ncp;
TAILQ_FOREACH(ncp, &nclruhead, nc_lru) {
if (ncp->nc_vp == vp && ncp->nc_dvp != NULL) {
(*pr)("name %.*s\n", ncp->nc_nlen, ncp->nc_name);
dvp = ncp->nc_dvp;
}
}
if (dvp == NULL) {
(*pr)("name not found\n");
return;
}
vp = dvp;
TAILQ_FOREACH(ncp, &nclruhead, nc_lru) {
if (ncp->nc_vp == vp) {
(*pr)("parent %.*s\n", ncp->nc_nlen, ncp->nc_name);
}
}
}
#endif