OpenSolaris_b135/uts/common/os/mmapobj.c
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or http://www.opensolaris.org/os/licensing.
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2009 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
#include <sys/types.h>
#include <sys/sysmacros.h>
#include <sys/kmem.h>
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/errno.h>
#include <sys/mman.h>
#include <sys/cmn_err.h>
#include <sys/cred.h>
#include <sys/vmsystm.h>
#include <sys/machsystm.h>
#include <sys/debug.h>
#include <vm/as.h>
#include <vm/seg.h>
#include <sys/vmparam.h>
#include <sys/vfs.h>
#include <sys/elf.h>
#include <sys/machelf.h>
#include <sys/corectl.h>
#include <sys/exec.h>
#include <sys/exechdr.h>
#include <sys/autoconf.h>
#include <sys/mem.h>
#include <vm/seg_dev.h>
#include <sys/vmparam.h>
#include <sys/mmapobj.h>
#include <sys/atomic.h>
/*
* Theory statement:
*
* The main driving force behind mmapobj is to interpret and map ELF files
* inside of the kernel instead of having the linker be responsible for this.
*
* mmapobj also supports the AOUT 4.x binary format as well as flat files in
* a read only manner.
*
* When interpreting and mapping an ELF file, mmapobj will map each PT_LOAD
* or PT_SUNWBSS segment according to the ELF standard. Refer to the "Linker
* and Libraries Guide" for more information about the standard and mapping
* rules.
*
* Having mmapobj interpret and map objects will allow the kernel to make the
* best decision for where to place the mappings for said objects. Thus, we
* can make optimizations inside of the kernel for specific platforms or
* cache mapping information to make mapping objects faster.
*
* The lib_va_hash will be one such optimization. For each ELF object that
* mmapobj is asked to interpret, we will attempt to cache the information
* about the PT_LOAD and PT_SUNWBSS sections to speed up future mappings of
* the same objects. We will cache up to LIBVA_CACHED_SEGS (see below) program
* headers which should cover a majority of the libraries out there without
* wasting space. In order to make sure that the cached information is valid,
* we check the passed in vnode's mtime and ctime to make sure the vnode
* has not been modified since the last time we used it.
*
* In addition, the lib_va_hash may contain a preferred starting VA for the
* object which can be useful for platforms which support a shared context.
* This will increase the likelyhood that library text can be shared among
* many different processes. We limit the reserved VA space for 32 bit objects
* in order to minimize fragmenting the processes address space.
*
* In addition to the above, the mmapobj interface allows for padding to be
* requested before the first mapping and after the last mapping created.
* When padding is requested, no additional optimizations will be made for
* that request.
*/
/*
* Threshold to prevent allocating too much kernel memory to read in the
* program headers for an object. If it requires more than below,
* we will use a KM_NOSLEEP allocation to allocate memory to hold all of the
* program headers which could possibly fail. If less memory than below is
* needed, then we use a KM_SLEEP allocation and are willing to wait for the
* memory if we need to.
*/
size_t mmapobj_alloc_threshold = 65536;
/* Debug stats for test coverage */
#ifdef DEBUG
struct mobj_stats {
uint_t mobjs_unmap_called;
uint_t mobjs_remap_devnull;
uint_t mobjs_lookup_start;
uint_t mobjs_alloc_start;
uint_t mobjs_alloc_vmem;
uint_t mobjs_add_collision;
uint_t mobjs_get_addr;
uint_t mobjs_map_flat_no_padding;
uint_t mobjs_map_flat_padding;
uint_t mobjs_map_ptload_text;
uint_t mobjs_map_ptload_initdata;
uint_t mobjs_map_ptload_preread;
uint_t mobjs_map_ptload_unaligned_text;
uint_t mobjs_map_ptload_unaligned_map_fail;
uint_t mobjs_map_ptload_unaligned_read_fail;
uint_t mobjs_zfoddiff;
uint_t mobjs_zfoddiff_nowrite;
uint_t mobjs_zfodextra;
uint_t mobjs_ptload_failed;
uint_t mobjs_map_elf_no_holes;
uint_t mobjs_unmap_hole;
uint_t mobjs_nomem_header;
uint_t mobjs_inval_header;
uint_t mobjs_overlap_header;
uint_t mobjs_np2_align;
uint_t mobjs_np2_align_overflow;
uint_t mobjs_exec_padding;
uint_t mobjs_exec_addr_mapped;
uint_t mobjs_exec_addr_devnull;
uint_t mobjs_exec_addr_in_use;
uint_t mobjs_lvp_found;
uint_t mobjs_no_loadable_yet;
uint_t mobjs_nothing_to_map;
uint_t mobjs_e2big;
uint_t mobjs_dyn_pad_align;
uint_t mobjs_dyn_pad_noalign;
uint_t mobjs_alloc_start_fail;
uint_t mobjs_lvp_nocache;
uint_t mobjs_extra_padding;
uint_t mobjs_lvp_not_needed;
uint_t mobjs_no_mem_map_sz;
uint_t mobjs_check_exec_failed;
uint_t mobjs_lvp_used;
uint_t mobjs_wrong_model;
uint_t mobjs_noexec_fs;
uint_t mobjs_e2big_et_rel;
uint_t mobjs_et_rel_mapped;
uint_t mobjs_unknown_elf_type;
uint_t mobjs_phent32_too_small;
uint_t mobjs_phent64_too_small;
uint_t mobjs_inval_elf_class;
uint_t mobjs_too_many_phdrs;
uint_t mobjs_no_phsize;
uint_t mobjs_phsize_large;
uint_t mobjs_phsize_xtralarge;
uint_t mobjs_fast_wrong_model;
uint_t mobjs_fast_e2big;
uint_t mobjs_fast;
uint_t mobjs_fast_success;
uint_t mobjs_fast_not_now;
uint_t mobjs_small_file;
uint_t mobjs_read_error;
uint_t mobjs_unsupported;
uint_t mobjs_flat_e2big;
uint_t mobjs_phent_align32;
uint_t mobjs_phent_align64;
uint_t mobjs_lib_va_find_hit;
uint_t mobjs_lib_va_find_delay_delete;
uint_t mobjs_lib_va_find_delete;
uint_t mobjs_lib_va_add_delay_delete;
uint_t mobjs_lib_va_add_delete;
uint_t mobjs_lib_va_create_failure;
uint_t mobjs_min_align;
#if defined(__sparc)
uint_t mobjs_aout_uzero_fault;
uint_t mobjs_aout_64bit_try;
uint_t mobjs_aout_noexec;
uint_t mobjs_aout_e2big;
uint_t mobjs_aout_lib;
uint_t mobjs_aout_fixed;
uint_t mobjs_aout_zfoddiff;
uint_t mobjs_aout_map_bss;
uint_t mobjs_aout_bss_fail;
uint_t mobjs_aout_nlist;
uint_t mobjs_aout_addr_in_use;
#endif
} mobj_stats;
#define MOBJ_STAT_ADD(stat) ((mobj_stats.mobjs_##stat)++)
#else
#define MOBJ_STAT_ADD(stat)
#endif
/*
* Check if addr is at or above the address space reserved for the stack.
* The stack is at the top of the address space for all sparc processes
* and 64 bit x86 processes. For 32 bit x86, the stack is not at the top
* of the address space and thus this check wil always return false for
* 32 bit x86 processes.
*/
#if defined(__sparc)
#define OVERLAPS_STACK(addr, p) \
(addr >= (p->p_usrstack - ((p->p_stk_ctl + PAGEOFFSET) & PAGEMASK)))
#elif defined(__amd64)
#define OVERLAPS_STACK(addr, p) \
((p->p_model == DATAMODEL_LP64) && \
(addr >= (p->p_usrstack - ((p->p_stk_ctl + PAGEOFFSET) & PAGEMASK))))
#elif defined(__i386)
#define OVERLAPS_STACK(addr, p) 0
#endif
/* lv_flags values - bitmap */
#define LV_ELF32 0x1 /* 32 bit ELF file */
#define LV_ELF64 0x2 /* 64 bit ELF file */
#define LV_DEL 0x4 /* delete when lv_refcnt hits zero */
/*
* Note: lv_num_segs will denote how many segments this file has and will
* only be set after the lv_mps array has been filled out.
* lv_mps can only be valid if lv_num_segs is non-zero.
*/
struct lib_va {
struct lib_va *lv_next;
caddr_t lv_base_va; /* start va for library */
ssize_t lv_len; /* total va span of library */
size_t lv_align; /* minimum alignment */
uint64_t lv_nodeid; /* filesystem node id */
uint64_t lv_fsid; /* filesystem id */
timestruc_t lv_ctime; /* last time file was changed */
timestruc_t lv_mtime; /* or modified */
mmapobj_result_t lv_mps[LIBVA_CACHED_SEGS]; /* cached pheaders */
int lv_num_segs; /* # segs for this file */
int lv_flags;
uint_t lv_refcnt; /* number of holds on struct */
};
#define LIB_VA_SIZE 1024
#define LIB_VA_MASK (LIB_VA_SIZE - 1)
#define LIB_VA_MUTEX_SHIFT 3
#if (LIB_VA_SIZE & (LIB_VA_SIZE - 1))
#error "LIB_VA_SIZE is not a power of 2"
#endif
static struct lib_va *lib_va_hash[LIB_VA_SIZE];
static kmutex_t lib_va_hash_mutex[LIB_VA_SIZE >> LIB_VA_MUTEX_SHIFT];
#define LIB_VA_HASH_MUTEX(index) \
(&lib_va_hash_mutex[index >> LIB_VA_MUTEX_SHIFT])
#define LIB_VA_HASH(nodeid) \
(((nodeid) ^ ((nodeid) << 7) ^ ((nodeid) << 13)) & LIB_VA_MASK)
#define LIB_VA_MATCH_ID(arg1, arg2) \
((arg1)->lv_nodeid == (arg2)->va_nodeid && \
(arg1)->lv_fsid == (arg2)->va_fsid)
#define LIB_VA_MATCH_TIME(arg1, arg2) \
((arg1)->lv_ctime.tv_sec == (arg2)->va_ctime.tv_sec && \
(arg1)->lv_mtime.tv_sec == (arg2)->va_mtime.tv_sec && \
(arg1)->lv_ctime.tv_nsec == (arg2)->va_ctime.tv_nsec && \
(arg1)->lv_mtime.tv_nsec == (arg2)->va_mtime.tv_nsec)
#define LIB_VA_MATCH(arg1, arg2) \
(LIB_VA_MATCH_ID(arg1, arg2) && LIB_VA_MATCH_TIME(arg1, arg2))
/*
* lib_va will be used for optimized allocation of address ranges for
* libraries, such that subsequent mappings of the same library will attempt
* to use the same VA as previous mappings of that library.
* In order to map libraries at the same VA in many processes, we need to carve
* out our own address space for them which is unique across many processes.
* We use different arenas for 32 bit and 64 bit libraries.
*
* Since the 32 bit address space is relatively small, we limit the number of
* libraries which try to use consistent virtual addresses to lib_threshold.
* For 64 bit libraries there is no such limit since the address space is large.
*/
static vmem_t *lib_va_32_arena;
static vmem_t *lib_va_64_arena;
uint_t lib_threshold = 20; /* modifiable via /etc/system */
static kmutex_t lib_va_init_mutex; /* no need to initialize */
/*
* Number of 32 bit and 64 bit libraries in lib_va hash.
*/
static uint_t libs_mapped_32 = 0;
static uint_t libs_mapped_64 = 0;
/*
* Free up the resources associated with lvp as well as lvp itself.
* We also decrement the number of libraries mapped via a lib_va
* cached virtual address.
*/
void
lib_va_free(struct lib_va *lvp)
{
int is_64bit = lvp->lv_flags & LV_ELF64;
ASSERT(lvp->lv_refcnt == 0);
if (lvp->lv_base_va != NULL) {
vmem_xfree(is_64bit ? lib_va_64_arena : lib_va_32_arena,
lvp->lv_base_va, lvp->lv_len);
if (is_64bit) {
atomic_add_32(&libs_mapped_64, -1);
} else {
atomic_add_32(&libs_mapped_32, -1);
}
}
kmem_free(lvp, sizeof (struct lib_va));
}
/*
* See if the file associated with the vap passed in is in the lib_va hash.
* If it is and the file has not been modified since last use, then
* return a pointer to that data. Otherwise, return NULL if the file has
* changed or the file was not found in the hash.
*/
static struct lib_va *
lib_va_find(vattr_t *vap)
{
struct lib_va *lvp;
struct lib_va *del = NULL;
struct lib_va **tmp;
uint_t index;
index = LIB_VA_HASH(vap->va_nodeid);
mutex_enter(LIB_VA_HASH_MUTEX(index));
tmp = &lib_va_hash[index];
while (*tmp != NULL) {
lvp = *tmp;
if (LIB_VA_MATCH_ID(lvp, vap)) {
if (LIB_VA_MATCH_TIME(lvp, vap)) {
ASSERT((lvp->lv_flags & LV_DEL) == 0);
lvp->lv_refcnt++;
MOBJ_STAT_ADD(lib_va_find_hit);
} else {
/*
* file was updated since last use.
* need to remove it from list.
*/
del = lvp;
*tmp = del->lv_next;
del->lv_next = NULL;
/*
* If we can't delete it now, mark it for later
*/
if (del->lv_refcnt) {
MOBJ_STAT_ADD(lib_va_find_delay_delete);
del->lv_flags |= LV_DEL;
del = NULL;
}
lvp = NULL;
}
mutex_exit(LIB_VA_HASH_MUTEX(index));
if (del) {
ASSERT(del->lv_refcnt == 0);
MOBJ_STAT_ADD(lib_va_find_delete);
lib_va_free(del);
}
return (lvp);
}
tmp = &lvp->lv_next;
}
mutex_exit(LIB_VA_HASH_MUTEX(index));
return (NULL);
}
/*
* Add a new entry to the lib_va hash.
* Search the hash while holding the appropriate mutex to make sure that the
* data is not already in the cache. If we find data that is in the cache
* already and has not been modified since last use, we return NULL. If it
* has been modified since last use, we will remove that entry from
* the hash and it will be deleted once it's reference count reaches zero.
* If there is no current entry in the hash we will add the new entry and
* return it to the caller who is responsible for calling lib_va_release to
* drop their reference count on it.
*
* lv_num_segs will be set to zero since the caller needs to add that
* information to the data structure.
*/
static struct lib_va *
lib_va_add_hash(caddr_t base_va, ssize_t len, size_t align, vattr_t *vap)
{
struct lib_va *lvp;
uint_t index;
model_t model;
struct lib_va **tmp;
struct lib_va *del = NULL;
model = get_udatamodel();
index = LIB_VA_HASH(vap->va_nodeid);
lvp = kmem_alloc(sizeof (struct lib_va), KM_SLEEP);
mutex_enter(LIB_VA_HASH_MUTEX(index));
/*
* Make sure not adding same data a second time.
* The hash chains should be relatively short and adding
* is a relatively rare event, so it's worth the check.
*/
tmp = &lib_va_hash[index];
while (*tmp != NULL) {
if (LIB_VA_MATCH_ID(*tmp, vap)) {
if (LIB_VA_MATCH_TIME(*tmp, vap)) {
mutex_exit(LIB_VA_HASH_MUTEX(index));
kmem_free(lvp, sizeof (struct lib_va));
return (NULL);
}
/*
* We have the same nodeid and fsid but the file has
* been modified since we last saw it.
* Need to remove the old node and add this new
* one.
* Could probably use a callback mechanism to make
* this cleaner.
*/
ASSERT(del == NULL);
del = *tmp;
*tmp = del->lv_next;
del->lv_next = NULL;
/*
* Check to see if we can free it. If lv_refcnt
* is greater than zero, than some other thread
* has a reference to the one we want to delete
* and we can not delete it. All of this is done
* under the lib_va_hash_mutex lock so it is atomic.
*/
if (del->lv_refcnt) {
MOBJ_STAT_ADD(lib_va_add_delay_delete);
del->lv_flags |= LV_DEL;
del = NULL;
}
/* tmp is already advanced */
continue;
}
tmp = &((*tmp)->lv_next);
}
lvp->lv_base_va = base_va;
lvp->lv_len = len;
lvp->lv_align = align;
lvp->lv_nodeid = vap->va_nodeid;
lvp->lv_fsid = vap->va_fsid;
lvp->lv_ctime.tv_sec = vap->va_ctime.tv_sec;
lvp->lv_ctime.tv_nsec = vap->va_ctime.tv_nsec;
lvp->lv_mtime.tv_sec = vap->va_mtime.tv_sec;
lvp->lv_mtime.tv_nsec = vap->va_mtime.tv_nsec;
lvp->lv_next = NULL;
lvp->lv_refcnt = 1;
/* Caller responsible for filling this and lv_mps out */
lvp->lv_num_segs = 0;
if (model == DATAMODEL_LP64) {
lvp->lv_flags = LV_ELF64;
} else {
ASSERT(model == DATAMODEL_ILP32);
lvp->lv_flags = LV_ELF32;
}
if (base_va != NULL) {
if (model == DATAMODEL_LP64) {
atomic_add_32(&libs_mapped_64, 1);
} else {
ASSERT(model == DATAMODEL_ILP32);
atomic_add_32(&libs_mapped_32, 1);
}
}
ASSERT(*tmp == NULL);
*tmp = lvp;
mutex_exit(LIB_VA_HASH_MUTEX(index));
if (del) {
ASSERT(del->lv_refcnt == 0);
MOBJ_STAT_ADD(lib_va_add_delete);
lib_va_free(del);
}
return (lvp);
}
/*
* Release the hold on lvp which was acquired by lib_va_find or lib_va_add_hash.
* In addition, if this is the last hold and lvp is marked for deletion,
* free up it's reserved address space and free the structure.
*/
static void
lib_va_release(struct lib_va *lvp)
{
uint_t index;
int to_del = 0;
ASSERT(lvp->lv_refcnt > 0);
index = LIB_VA_HASH(lvp->lv_nodeid);
mutex_enter(LIB_VA_HASH_MUTEX(index));
if (--lvp->lv_refcnt == 0 && (lvp->lv_flags & LV_DEL)) {
to_del = 1;
}
mutex_exit(LIB_VA_HASH_MUTEX(index));
if (to_del) {
ASSERT(lvp->lv_next == 0);
lib_va_free(lvp);
}
}
/*
* Dummy function for mapping through /dev/null
* Normally I would have used mmmmap in common/io/mem.c
* but that is a static function, and for /dev/null, it
* just returns -1.
*/
/* ARGSUSED */
static int
mmapobj_dummy(dev_t dev, off_t off, int prot)
{
return (-1);
}
/*
* Called when an error occurred which requires mmapobj to return failure.
* All mapped objects will be unmapped and /dev/null mappings will be
* reclaimed if necessary.
* num_mapped is the number of elements of mrp which have been mapped, and
* num_segs is the total number of elements in mrp.
* For e_type ET_EXEC, we need to unmap all of the elements in mrp since
* we had already made reservations for them.
* If num_mapped equals num_segs, then we know that we had fully mapped
* the file and only need to clean up the segments described.
* If they are not equal, then for ET_DYN we will unmap the range from the
* end of the last mapped segment to the end of the last segment in mrp
* since we would have made a reservation for that memory earlier.
* If e_type is passed in as zero, num_mapped must equal num_segs.
*/
void
mmapobj_unmap(mmapobj_result_t *mrp, int num_mapped, int num_segs,
ushort_t e_type)
{
int i;
struct as *as = curproc->p_as;
caddr_t addr;
size_t size;
if (e_type == ET_EXEC) {
num_mapped = num_segs;
}
#ifdef DEBUG
if (e_type == 0) {
ASSERT(num_mapped == num_segs);
}
#endif
MOBJ_STAT_ADD(unmap_called);
for (i = 0; i < num_mapped; i++) {
/*
* If we are going to have to create a mapping we need to
* make sure that no one else will use the address we
* need to remap between the time it is unmapped and
* mapped below.
*/
if (mrp[i].mr_flags & MR_RESV) {
as_rangelock(as);
}
/* Always need to unmap what we mapped */
(void) as_unmap(as, mrp[i].mr_addr, mrp[i].mr_msize);
/* Need to reclaim /dev/null reservation from earlier */
if (mrp[i].mr_flags & MR_RESV) {
struct segdev_crargs dev_a;
ASSERT(e_type != ET_DYN);
/*
* Use seg_dev segment driver for /dev/null mapping.
*/
dev_a.mapfunc = mmapobj_dummy;
dev_a.dev = makedevice(mm_major, M_NULL);
dev_a.offset = 0;
dev_a.type = 0; /* neither PRIVATE nor SHARED */
dev_a.prot = dev_a.maxprot = (uchar_t)PROT_NONE;
dev_a.hat_attr = 0;
dev_a.hat_flags = 0;
(void) as_map(as, mrp[i].mr_addr, mrp[i].mr_msize,
segdev_create, &dev_a);
MOBJ_STAT_ADD(remap_devnull);
as_rangeunlock(as);
}
}
if (num_mapped != num_segs) {
ASSERT(e_type == ET_DYN);
/* Need to unmap any reservation made after last mapped seg */
if (num_mapped == 0) {
addr = mrp[0].mr_addr;
} else {
addr = mrp[num_mapped - 1].mr_addr +
mrp[num_mapped - 1].mr_msize;
}
size = (size_t)mrp[num_segs - 1].mr_addr +
mrp[num_segs - 1].mr_msize - (size_t)addr;
(void) as_unmap(as, addr, size);
/*
* Now we need to unmap the holes between mapped segs.
* Note that we have not mapped all of the segments and thus
* the holes between segments would not have been unmapped
* yet. If num_mapped == num_segs, then all of the holes
* between segments would have already been unmapped.
*/
for (i = 1; i < num_mapped; i++) {
addr = mrp[i - 1].mr_addr + mrp[i - 1].mr_msize;
size = mrp[i].mr_addr - addr;
(void) as_unmap(as, addr, size);
}
}
}
/*
* We need to add the start address into mrp so that the unmap function
* has absolute addresses to use.
*/
static void
mmapobj_unmap_exec(mmapobj_result_t *mrp, int num_mapped, caddr_t start_addr)
{
int i;
for (i = 0; i < num_mapped; i++) {
mrp[i].mr_addr += (size_t)start_addr;
}
mmapobj_unmap(mrp, num_mapped, num_mapped, ET_EXEC);
}
static caddr_t
mmapobj_lookup_start_addr(struct lib_va *lvp)
{
proc_t *p = curproc;
struct as *as = p->p_as;
struct segvn_crargs crargs = SEGVN_ZFOD_ARGS(PROT_USER, PROT_ALL);
int error;
uint_t ma_flags = _MAP_LOW32;
caddr_t base = NULL;
size_t len;
size_t align;
ASSERT(lvp != NULL);
MOBJ_STAT_ADD(lookup_start);
as_rangelock(as);
base = lvp->lv_base_va;
len = lvp->lv_len;
/*
* If we don't have an expected base address, or the one that we want
* to use is not available or acceptable, go get an acceptable
* address range.
*/
if (base == NULL || as_gap(as, len, &base, &len, 0, NULL) ||
valid_usr_range(base, len, PROT_ALL, as, as->a_userlimit) !=
RANGE_OKAY || OVERLAPS_STACK(base + len, p)) {
if (lvp->lv_flags & LV_ELF64) {
ma_flags = 0;
}
align = lvp->lv_align;
if (align > 1) {
ma_flags |= MAP_ALIGN;
}
base = (caddr_t)align;
map_addr(&base, len, 0, 1, ma_flags);
}
/*
* Need to reserve the address space we're going to use.
* Don't reserve swap space since we'll be mapping over this.
*/
if (base != NULL) {
crargs.flags |= MAP_NORESERVE;
error = as_map(as, base, len, segvn_create, &crargs);
if (error) {
base = NULL;
}
}
as_rangeunlock(as);
return (base);
}
/*
* Get the starting address for a given file to be mapped and return it
* to the caller. If we're using lib_va and we need to allocate an address,
* we will attempt to allocate it from the global reserved pool such that the
* same address can be used in the future for this file. If we can't use the
* reserved address then we just get one that will fit in our address space.
*
* Returns the starting virtual address for the range to be mapped or NULL
* if an error is encountered. If we successfully insert the requested info
* into the lib_va hash, then *lvpp will be set to point to this lib_va
* structure. The structure will have a hold on it and thus lib_va_release
* needs to be called on it by the caller. This function will not fill out
* lv_mps or lv_num_segs since it does not have enough information to do so.
* The caller is responsible for doing this making sure that any modifications
* to lv_mps are visible before setting lv_num_segs.
*/
static caddr_t
mmapobj_alloc_start_addr(struct lib_va **lvpp, size_t len, int use_lib_va,
size_t align, vattr_t *vap)
{
proc_t *p = curproc;
struct as *as = p->p_as;
struct segvn_crargs crargs = SEGVN_ZFOD_ARGS(PROT_USER, PROT_ALL);
int error;
model_t model;
uint_t ma_flags = _MAP_LOW32;
caddr_t base = NULL;
vmem_t *model_vmem;
size_t lib_va_start;
size_t lib_va_end;
size_t lib_va_len;
ASSERT(lvpp != NULL);
MOBJ_STAT_ADD(alloc_start);
model = get_udatamodel();
if (model == DATAMODEL_LP64) {
ma_flags = 0;
model_vmem = lib_va_64_arena;
} else {
ASSERT(model == DATAMODEL_ILP32);
model_vmem = lib_va_32_arena;
}
if (align > 1) {
ma_flags |= MAP_ALIGN;
}
if (use_lib_va) {
/*
* The first time through, we need to setup the lib_va arenas.
* We call map_addr to find a suitable range of memory to map
* the given library, and we will set the highest address
* in our vmem arena to the end of this adddress range.
* We allow up to half of the address space to be used
* for lib_va addresses but we do not prevent any allocations
* in this range from other allocation paths.
*/
if (lib_va_64_arena == NULL && model == DATAMODEL_LP64) {
mutex_enter(&lib_va_init_mutex);
if (lib_va_64_arena == NULL) {
base = (caddr_t)align;
as_rangelock(as);
map_addr(&base, len, 0, 1, ma_flags);
as_rangeunlock(as);
if (base == NULL) {
mutex_exit(&lib_va_init_mutex);
MOBJ_STAT_ADD(lib_va_create_failure);
goto nolibva;
}
lib_va_end = (size_t)base + len;
lib_va_len = lib_va_end >> 1;
lib_va_len = P2ROUNDUP(lib_va_len, PAGESIZE);
lib_va_start = lib_va_end - lib_va_len;
/*
* Need to make sure we avoid the address hole.
* We know lib_va_end is valid but we need to
* make sure lib_va_start is as well.
*/
if ((lib_va_end > (size_t)hole_end) &&
(lib_va_start < (size_t)hole_end)) {
lib_va_start = P2ROUNDUP(
(size_t)hole_end, PAGESIZE);
lib_va_len = lib_va_end - lib_va_start;
}
lib_va_64_arena = vmem_create("lib_va_64",
(void *)lib_va_start, lib_va_len, PAGESIZE,
NULL, NULL, NULL, 0,
VM_NOSLEEP | VMC_IDENTIFIER);
if (lib_va_64_arena == NULL) {
mutex_exit(&lib_va_init_mutex);
goto nolibva;
}
}
model_vmem = lib_va_64_arena;
mutex_exit(&lib_va_init_mutex);
} else if (lib_va_32_arena == NULL &&
model == DATAMODEL_ILP32) {
mutex_enter(&lib_va_init_mutex);
if (lib_va_32_arena == NULL) {
base = (caddr_t)align;
as_rangelock(as);
map_addr(&base, len, 0, 1, ma_flags);
as_rangeunlock(as);
if (base == NULL) {
mutex_exit(&lib_va_init_mutex);
MOBJ_STAT_ADD(lib_va_create_failure);
goto nolibva;
}
lib_va_end = (size_t)base + len;
lib_va_len = lib_va_end >> 1;
lib_va_len = P2ROUNDUP(lib_va_len, PAGESIZE);
lib_va_start = lib_va_end - lib_va_len;
lib_va_32_arena = vmem_create("lib_va_32",
(void *)lib_va_start, lib_va_len, PAGESIZE,
NULL, NULL, NULL, 0,
VM_NOSLEEP | VMC_IDENTIFIER);
if (lib_va_32_arena == NULL) {
mutex_exit(&lib_va_init_mutex);
goto nolibva;
}
}
model_vmem = lib_va_32_arena;
mutex_exit(&lib_va_init_mutex);
}
if (model == DATAMODEL_LP64 || libs_mapped_32 < lib_threshold) {
base = vmem_xalloc(model_vmem, len, align, 0, 0, NULL,
NULL, VM_NOSLEEP | VM_ENDALLOC);
MOBJ_STAT_ADD(alloc_vmem);
}
/*
* Even if the address fails to fit in our address space,
* or we can't use a reserved address,
* we should still save it off in lib_va_hash.
*/
*lvpp = lib_va_add_hash(base, len, align, vap);
/*
* Check for collision on insertion and free up our VA space.
* This is expected to be rare, so we'll just reset base to
* NULL instead of looking it up in the lib_va hash.
*/
if (*lvpp == NULL) {
if (base != NULL) {
vmem_xfree(model_vmem, base, len);
base = NULL;
MOBJ_STAT_ADD(add_collision);
}
}
}
nolibva:
as_rangelock(as);
/*
* If we don't have an expected base address, or the one that we want
* to use is not available or acceptable, go get an acceptable
* address range.
*/
if (base == NULL || as_gap(as, len, &base, &len, 0, NULL) ||
valid_usr_range(base, len, PROT_ALL, as, as->a_userlimit) !=
RANGE_OKAY || OVERLAPS_STACK(base + len, p)) {
MOBJ_STAT_ADD(get_addr);
base = (caddr_t)align;
map_addr(&base, len, 0, 1, ma_flags);
}
/*
* Need to reserve the address space we're going to use.
* Don't reserve swap space since we'll be mapping over this.
*/
if (base != NULL) {
/* Don't reserve swap space since we'll be mapping over this */
crargs.flags |= MAP_NORESERVE;
error = as_map(as, base, len, segvn_create, &crargs);
if (error) {
base = NULL;
}
}
as_rangeunlock(as);
return (base);
}
/*
* Map the file associated with vp into the address space as a single
* read only private mapping.
* Returns 0 for success, and non-zero for failure to map the file.
*/
static int
mmapobj_map_flat(vnode_t *vp, mmapobj_result_t *mrp, size_t padding,
cred_t *fcred)
{
int error = 0;
struct as *as = curproc->p_as;
caddr_t addr = NULL;
caddr_t start_addr;
size_t len;
size_t pad_len;
int prot = PROT_USER | PROT_READ;
uint_t ma_flags = _MAP_LOW32;
vattr_t vattr;
struct segvn_crargs crargs = SEGVN_ZFOD_ARGS(PROT_USER, PROT_ALL);
if (get_udatamodel() == DATAMODEL_LP64) {
ma_flags = 0;
}
vattr.va_mask = AT_SIZE;
error = VOP_GETATTR(vp, &vattr, 0, fcred, NULL);
if (error) {
return (error);
}
len = vattr.va_size;
ma_flags |= MAP_PRIVATE;
if (padding == 0) {
MOBJ_STAT_ADD(map_flat_no_padding);
error = VOP_MAP(vp, 0, as, &addr, len, prot, PROT_ALL,
ma_flags, fcred, NULL);
if (error == 0) {
mrp[0].mr_addr = addr;
mrp[0].mr_msize = len;
mrp[0].mr_fsize = len;
mrp[0].mr_offset = 0;
mrp[0].mr_prot = prot;
mrp[0].mr_flags = 0;
}
return (error);
}
/* padding was requested so there's more work to be done */
MOBJ_STAT_ADD(map_flat_padding);
/* No need to reserve swap space now since it will be reserved later */
crargs.flags |= MAP_NORESERVE;
/* Need to setup padding which can only be in PAGESIZE increments. */
ASSERT((padding & PAGEOFFSET) == 0);
pad_len = len + (2 * padding);
as_rangelock(as);
map_addr(&addr, pad_len, 0, 1, ma_flags);
error = as_map(as, addr, pad_len, segvn_create, &crargs);
as_rangeunlock(as);
if (error) {
return (error);
}
start_addr = addr;
addr += padding;
ma_flags |= MAP_FIXED;
error = VOP_MAP(vp, 0, as, &addr, len, prot, PROT_ALL, ma_flags,
fcred, NULL);
if (error == 0) {
mrp[0].mr_addr = start_addr;
mrp[0].mr_msize = padding;
mrp[0].mr_fsize = 0;
mrp[0].mr_offset = 0;
mrp[0].mr_prot = 0;
mrp[0].mr_flags = MR_PADDING;
mrp[1].mr_addr = addr;
mrp[1].mr_msize = len;
mrp[1].mr_fsize = len;
mrp[1].mr_offset = 0;
mrp[1].mr_prot = prot;
mrp[1].mr_flags = 0;
mrp[2].mr_addr = addr + P2ROUNDUP(len, PAGESIZE);
mrp[2].mr_msize = padding;
mrp[2].mr_fsize = 0;
mrp[2].mr_offset = 0;
mrp[2].mr_prot = 0;
mrp[2].mr_flags = MR_PADDING;
} else {
/* Need to cleanup the as_map from earlier */
(void) as_unmap(as, start_addr, pad_len);
}
return (error);
}
/*
* Map a PT_LOAD or PT_SUNWBSS section of an executable file into the user's
* address space.
* vp - vnode to be mapped in
* addr - start address
* len - length of vp to be mapped
* zfodlen - length of zero filled memory after len above
* offset - offset into file where mapping should start
* prot - protections for this mapping
* fcred - credentials for the file associated with vp at open time.
*/
static int
mmapobj_map_ptload(struct vnode *vp, caddr_t addr, size_t len, size_t zfodlen,
off_t offset, int prot, cred_t *fcred)
{
int error = 0;
caddr_t zfodbase, oldaddr;
size_t oldlen;
size_t end;
size_t zfoddiff;
label_t ljb;
struct as *as = curproc->p_as;
model_t model;
int full_page;
/*
* See if addr and offset are aligned such that we can map in
* full pages instead of partial pages.
*/
full_page = (((uintptr_t)addr & PAGEOFFSET) ==
((uintptr_t)offset & PAGEOFFSET));
model = get_udatamodel();
oldaddr = addr;
addr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
if (len) {
spgcnt_t availm, npages;
int preread;
uint_t mflag = MAP_PRIVATE | MAP_FIXED;
if (model == DATAMODEL_ILP32) {
mflag |= _MAP_LOW32;
}
/* We may need to map in extra bytes */
oldlen = len;
len += ((size_t)oldaddr & PAGEOFFSET);
if (full_page) {
offset = (off_t)((uintptr_t)offset & PAGEMASK);
if ((prot & (PROT_WRITE | PROT_EXEC)) == PROT_EXEC) {
mflag |= MAP_TEXT;
MOBJ_STAT_ADD(map_ptload_text);
} else {
mflag |= MAP_INITDATA;
MOBJ_STAT_ADD(map_ptload_initdata);
}
/*
* maxprot is passed as PROT_ALL so that mdb can
* write to this segment.
*/
if (error = VOP_MAP(vp, (offset_t)offset, as, &addr,
len, prot, PROT_ALL, mflag, fcred, NULL)) {
return (error);
}
/*
* If the segment can fit and is relatively small, then
* we prefault the entire segment in. This is based
* on the model that says the best working set of a
* small program is all of its pages.
* We only do this if freemem will not drop below
* lotsfree since we don't want to induce paging.
*/
npages = (spgcnt_t)btopr(len);
availm = freemem - lotsfree;
preread = (npages < availm && len < PGTHRESH) ? 1 : 0;
/*
* If we aren't prefaulting the segment,
* increment "deficit", if necessary to ensure
* that pages will become available when this
* process starts executing.
*/
if (preread == 0 && npages > availm &&
deficit < lotsfree) {
deficit += MIN((pgcnt_t)(npages - availm),
lotsfree - deficit);
}
if (preread) {
(void) as_faulta(as, addr, len);
MOBJ_STAT_ADD(map_ptload_preread);
}
} else {
/*
* addr and offset were not aligned such that we could
* use VOP_MAP, thus we need to as_map the memory we
* need and then read the data in from disk.
* This code path is a corner case which should never
* be taken, but hand crafted binaries could trigger
* this logic and it needs to work correctly.
*/
MOBJ_STAT_ADD(map_ptload_unaligned_text);
as_rangelock(as);
(void) as_unmap(as, addr, len);
/*
* We use zfod_argsp because we need to be able to
* write to the mapping and then we'll change the
* protections later if they are incorrect.
*/
error = as_map(as, addr, len, segvn_create, zfod_argsp);
as_rangeunlock(as);
if (error) {
MOBJ_STAT_ADD(map_ptload_unaligned_map_fail);
return (error);
}
/* Now read in the data from disk */
error = vn_rdwr(UIO_READ, vp, oldaddr, oldlen, offset,
UIO_USERSPACE, 0, (rlim64_t)0, fcred, NULL);
if (error) {
MOBJ_STAT_ADD(map_ptload_unaligned_read_fail);
return (error);
}
/*
* Now set protections.
*/
if (prot != PROT_ZFOD) {
(void) as_setprot(as, addr, len, prot);
}
}
}
if (zfodlen) {
end = (size_t)addr + len;
zfodbase = (caddr_t)P2ROUNDUP(end, PAGESIZE);
zfoddiff = (uintptr_t)zfodbase - end;
if (zfoddiff) {
MOBJ_STAT_ADD(zfoddiff);
if ((prot & PROT_WRITE) == 0) {
(void) as_setprot(as, (caddr_t)end,
zfoddiff, prot | PROT_WRITE);
MOBJ_STAT_ADD(zfoddiff_nowrite);
}
if (on_fault(&ljb)) {
no_fault();
if ((prot & PROT_WRITE) == 0) {
(void) as_setprot(as, (caddr_t)end,
zfoddiff, prot);
}
return (EFAULT);
}
uzero((void *)end, zfoddiff);
no_fault();
/*
* Remove write protection to return to original state
*/
if ((prot & PROT_WRITE) == 0) {
(void) as_setprot(as, (caddr_t)end,
zfoddiff, prot);
}
}
if (zfodlen > zfoddiff) {
struct segvn_crargs crargs =
SEGVN_ZFOD_ARGS(prot, PROT_ALL);
MOBJ_STAT_ADD(zfodextra);
zfodlen -= zfoddiff;
crargs.szc = AS_MAP_NO_LPOOB;
as_rangelock(as);
(void) as_unmap(as, (caddr_t)zfodbase, zfodlen);
error = as_map(as, (caddr_t)zfodbase,
zfodlen, segvn_create, &crargs);
as_rangeunlock(as);
if (error) {
return (error);
}
}
}
return (0);
}
/*
* Map the ELF file represented by vp into the users address space. The
* first mapping will start at start_addr and there will be num_elements
* mappings. The mappings are described by the data in mrp which may be
* modified upon returning from this function.
* Returns 0 for success or errno for failure.
*/
static int
mmapobj_map_elf(struct vnode *vp, caddr_t start_addr, mmapobj_result_t *mrp,
int num_elements, cred_t *fcred, ushort_t e_type)
{
int i;
int ret;
caddr_t lo;
caddr_t hi;
struct as *as = curproc->p_as;
for (i = 0; i < num_elements; i++) {
caddr_t addr;
size_t p_memsz;
size_t p_filesz;
size_t zfodlen;
offset_t p_offset;
size_t dif;
int prot;
/* Always need to adjust mr_addr */
addr = start_addr + (size_t)(mrp[i].mr_addr);
mrp[i].mr_addr =
(caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
/* Padding has already been mapped */
if (MR_GET_TYPE(mrp[i].mr_flags) == MR_PADDING) {
continue;
}
p_memsz = mrp[i].mr_msize;
p_filesz = mrp[i].mr_fsize;
zfodlen = p_memsz - p_filesz;
p_offset = mrp[i].mr_offset;
dif = (uintptr_t)(addr) & PAGEOFFSET;
prot = mrp[i].mr_prot | PROT_USER;
ret = mmapobj_map_ptload(vp, addr, p_filesz, zfodlen,
p_offset, prot, fcred);
if (ret != 0) {
MOBJ_STAT_ADD(ptload_failed);
mmapobj_unmap(mrp, i, num_elements, e_type);
return (ret);
}
/* Need to cleanup mrp to reflect the actual values used */
mrp[i].mr_msize += dif;
mrp[i].mr_offset = (size_t)addr & PAGEOFFSET;
}
/* Also need to unmap any holes created above */
if (num_elements == 1) {
MOBJ_STAT_ADD(map_elf_no_holes);
return (0);
}
if (e_type == ET_EXEC) {
return (0);
}
as_rangelock(as);
lo = start_addr;
hi = mrp[0].mr_addr;
/* Remove holes made by the rest of the segments */
for (i = 0; i < num_elements - 1; i++) {
lo = (caddr_t)P2ROUNDUP((size_t)(mrp[i].mr_addr) +
mrp[i].mr_msize, PAGESIZE);
hi = mrp[i + 1].mr_addr;
if (lo < hi) {
/*
* If as_unmap fails we just use up a bit of extra
* space
*/
(void) as_unmap(as, (caddr_t)lo,
(size_t)hi - (size_t)lo);
MOBJ_STAT_ADD(unmap_hole);
}
}
as_rangeunlock(as);
return (0);
}
/* Ugly hack to get STRUCT_* macros to work below */
struct myphdr {
Phdr x; /* native version */
};
struct myphdr32 {
Elf32_Phdr x;
};
/*
* Calculate and return the number of loadable segments in the ELF Phdr
* represented by phdrbase as well as the len of the total mapping and
* the max alignment that is needed for a given segment. On success,
* 0 is returned, and *len, *loadable and *align have been filled out.
* On failure, errno will be returned, which in this case is ENOTSUP
* if we were passed an ELF file with overlapping segments.
*/
static int
calc_loadable(Ehdr *ehdrp, caddr_t phdrbase, int nphdrs, size_t *len,
int *loadable, size_t *align)
{
int i;
int hsize;
model_t model;
ushort_t e_type = ehdrp->e_type; /* same offset 32 and 64 bit */
uint_t p_type;
offset_t p_offset;
size_t p_memsz;
size_t p_align;
caddr_t vaddr;
int num_segs = 0;
caddr_t start_addr = NULL;
caddr_t p_end = NULL;
size_t max_align = 0;
size_t min_align = PAGESIZE; /* needed for vmem_xalloc */
STRUCT_HANDLE(myphdr, mph);
#if defined(__sparc)
extern int vac_size;
/*
* Want to prevent aliasing by making the start address at least be
* aligned to vac_size.
*/
min_align = MAX(PAGESIZE, vac_size);
#endif
model = get_udatamodel();
STRUCT_SET_HANDLE(mph, model, (struct myphdr *)phdrbase);
/* hsize alignment should have been checked before calling this func */
if (model == DATAMODEL_LP64) {
hsize = ehdrp->e_phentsize;
if (hsize & 7) {
return (ENOTSUP);
}
} else {
ASSERT(model == DATAMODEL_ILP32);
hsize = ((Elf32_Ehdr *)ehdrp)->e_phentsize;
if (hsize & 3) {
return (ENOTSUP);
}
}
/*
* Determine the span of all loadable segments and calculate the
* number of loadable segments.
*/
for (i = 0; i < nphdrs; i++) {
p_type = STRUCT_FGET(mph, x.p_type);
if (p_type == PT_LOAD || p_type == PT_SUNWBSS) {
vaddr = (caddr_t)(uintptr_t)STRUCT_FGET(mph, x.p_vaddr);
p_memsz = STRUCT_FGET(mph, x.p_memsz);
/*
* Skip this header if it requests no memory to be
* mapped.
*/
if (p_memsz == 0) {
STRUCT_SET_HANDLE(mph, model,
(struct myphdr *)((size_t)STRUCT_BUF(mph) +
hsize));
MOBJ_STAT_ADD(nomem_header);
continue;
}
if (num_segs++ == 0) {
/*
* The p_vaddr of the first PT_LOAD segment
* must either be NULL or within the first
* page in order to be interpreted.
* Otherwise, its an invalid file.
*/
if (e_type == ET_DYN &&
((caddr_t)((uintptr_t)vaddr &
(uintptr_t)PAGEMASK) != NULL)) {
MOBJ_STAT_ADD(inval_header);
return (ENOTSUP);
}
start_addr = vaddr;
/*
* For the first segment, we need to map from
* the beginning of the file, so we will
* adjust the size of the mapping to include
* this memory.
*/
p_offset = STRUCT_FGET(mph, x.p_offset);
} else {
p_offset = 0;
}
/*
* Check to make sure that this mapping wouldn't
* overlap a previous mapping.
*/
if (vaddr < p_end) {
MOBJ_STAT_ADD(overlap_header);
return (ENOTSUP);
}
p_end = vaddr + p_memsz + p_offset;
p_end = (caddr_t)P2ROUNDUP((size_t)p_end, PAGESIZE);
p_align = STRUCT_FGET(mph, x.p_align);
if (p_align > 1 && p_align > max_align) {
max_align = p_align;
if (max_align < min_align) {
max_align = min_align;
MOBJ_STAT_ADD(min_align);
}
}
}
STRUCT_SET_HANDLE(mph, model,
(struct myphdr *)((size_t)STRUCT_BUF(mph) + hsize));
}
/*
* The alignment should be a power of 2, if it isn't we forgive it
* and round up. On overflow, we'll set the alignment to max_align
* rounded down to the nearest power of 2.
*/
if (max_align > 0 && !ISP2(max_align)) {
MOBJ_STAT_ADD(np2_align);
*align = 2 * (1L << (highbit(max_align) - 1));
if (*align < max_align ||
(*align > UINT_MAX && model == DATAMODEL_ILP32)) {
MOBJ_STAT_ADD(np2_align_overflow);
*align = 1L << (highbit(max_align) - 1);
}
} else {
*align = max_align;
}
ASSERT(*align >= PAGESIZE || *align == 0);
*loadable = num_segs;
*len = p_end - start_addr;
return (0);
}
/*
* Check the address space to see if the virtual addresses to be used are
* available. If they are not, return errno for failure. On success, 0
* will be returned, and the virtual addresses for each mmapobj_result_t
* will be reserved. Note that a reservation could have earlier been made
* for a given segment via a /dev/null mapping. If that is the case, then
* we can use that VA space for our mappings.
* Note: this function will only be used for ET_EXEC binaries.
*/
int
check_exec_addrs(int loadable, mmapobj_result_t *mrp, caddr_t start_addr)
{
int i;
struct as *as = curproc->p_as;
struct segvn_crargs crargs = SEGVN_ZFOD_ARGS(PROT_ZFOD, PROT_ALL);
int ret;
caddr_t myaddr;
size_t mylen;
struct seg *seg;
/* No need to reserve swap space now since it will be reserved later */
crargs.flags |= MAP_NORESERVE;
as_rangelock(as);
for (i = 0; i < loadable; i++) {
myaddr = start_addr + (size_t)mrp[i].mr_addr;
mylen = mrp[i].mr_msize;
/* See if there is a hole in the as for this range */
if (as_gap(as, mylen, &myaddr, &mylen, 0, NULL) == 0) {
ASSERT(myaddr == start_addr + (size_t)mrp[i].mr_addr);
ASSERT(mylen == mrp[i].mr_msize);
#ifdef DEBUG
if (MR_GET_TYPE(mrp[i].mr_flags) == MR_PADDING) {
MOBJ_STAT_ADD(exec_padding);
}
#endif
ret = as_map(as, myaddr, mylen, segvn_create, &crargs);
if (ret) {
as_rangeunlock(as);
mmapobj_unmap_exec(mrp, i, start_addr);
return (ret);
}
} else {
/*
* There is a mapping that exists in the range
* so check to see if it was a "reservation"
* from /dev/null. The mapping is from
* /dev/null if the mapping comes from
* segdev and the type is neither MAP_SHARED
* nor MAP_PRIVATE.
*/
AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
seg = as_findseg(as, myaddr, 0);
MOBJ_STAT_ADD(exec_addr_mapped);
if (seg && seg->s_ops == &segdev_ops &&
((SEGOP_GETTYPE(seg, myaddr) &
(MAP_SHARED | MAP_PRIVATE)) == 0) &&
myaddr >= seg->s_base &&
myaddr + mylen <=
seg->s_base + seg->s_size) {
MOBJ_STAT_ADD(exec_addr_devnull);
AS_LOCK_EXIT(as, &as->a_lock);
(void) as_unmap(as, myaddr, mylen);
ret = as_map(as, myaddr, mylen, segvn_create,
&crargs);
mrp[i].mr_flags |= MR_RESV;
if (ret) {
as_rangeunlock(as);
/* Need to remap what we unmapped */
mmapobj_unmap_exec(mrp, i + 1,
start_addr);
return (ret);
}
} else {
AS_LOCK_EXIT(as, &as->a_lock);
as_rangeunlock(as);
mmapobj_unmap_exec(mrp, i, start_addr);
MOBJ_STAT_ADD(exec_addr_in_use);
return (EADDRINUSE);
}
}
}
as_rangeunlock(as);
return (0);
}
/*
* Walk through the ELF program headers and extract all useful information
* for PT_LOAD and PT_SUNWBSS segments into mrp.
* Return 0 on success or error on failure.
*/
static int
process_phdr(Ehdr *ehdrp, caddr_t phdrbase, int nphdrs, mmapobj_result_t *mrp,
vnode_t *vp, uint_t *num_mapped, size_t padding, cred_t *fcred)
{
int i;
caddr_t start_addr = NULL;
caddr_t vaddr;
size_t len = 0;
size_t lib_len = 0;
int ret;
int prot;
struct lib_va *lvp = NULL;
vattr_t vattr;
struct as *as = curproc->p_as;
int error;
int loadable = 0;
int current = 0;
int use_lib_va = 1;
size_t align = 0;
size_t add_pad = 0;
int hdr_seen = 0;
ushort_t e_type = ehdrp->e_type; /* same offset 32 and 64 bit */
uint_t p_type;
offset_t p_offset;
size_t p_memsz;
size_t p_filesz;
uint_t p_flags;
int hsize;
model_t model;
STRUCT_HANDLE(myphdr, mph);
model = get_udatamodel();
STRUCT_SET_HANDLE(mph, model, (struct myphdr *)phdrbase);
/*
* Need to make sure that hsize is aligned properly.
* For 32bit processes, 4 byte alignment is required.
* For 64bit processes, 8 byte alignment is required.
* If the alignment isn't correct, we need to return failure
* since it could cause an alignment error panic while walking
* the phdr array.
*/
if (model == DATAMODEL_LP64) {
hsize = ehdrp->e_phentsize;
if (hsize & 7) {
MOBJ_STAT_ADD(phent_align64);
return (ENOTSUP);
}
} else {
ASSERT(model == DATAMODEL_ILP32);
hsize = ((Elf32_Ehdr *)ehdrp)->e_phentsize;
if (hsize & 3) {
MOBJ_STAT_ADD(phent_align32);
return (ENOTSUP);
}
}
if (padding != 0) {
use_lib_va = 0;
}
if (e_type == ET_DYN) {
vattr.va_mask = AT_FSID | AT_NODEID | AT_CTIME | AT_MTIME;
error = VOP_GETATTR(vp, &vattr, 0, fcred, NULL);
if (error) {
return (error);
}
/* Check to see if we already have a description for this lib */
lvp = lib_va_find(&vattr);
if (lvp != NULL) {
MOBJ_STAT_ADD(lvp_found);
if (use_lib_va) {
start_addr = mmapobj_lookup_start_addr(lvp);
if (start_addr == NULL) {
lib_va_release(lvp);
return (ENOMEM);
}
}
/*
* loadable may be zero if the original allocator
* of lvp hasn't finished setting it up but the rest
* of the fields will be accurate.
*/
loadable = lvp->lv_num_segs;
len = lvp->lv_len;
align = lvp->lv_align;
}
}
/*
* Determine the span of all loadable segments and calculate the
* number of loadable segments, the total len spanned by the mappings
* and the max alignment, if we didn't get them above.
*/
if (loadable == 0) {
MOBJ_STAT_ADD(no_loadable_yet);
ret = calc_loadable(ehdrp, phdrbase, nphdrs, &len,
&loadable, &align);
if (ret != 0) {
/*
* Since it'd be an invalid file, we shouldn't have
* cached it previously.
*/
ASSERT(lvp == NULL);
return (ret);
}
#ifdef DEBUG
if (lvp) {
ASSERT(len == lvp->lv_len);
ASSERT(align == lvp->lv_align);
}
#endif
}
/* Make sure there's something to map. */
if (len == 0 || loadable == 0) {
/*
* Since it'd be an invalid file, we shouldn't have
* cached it previously.
*/
ASSERT(lvp == NULL);
MOBJ_STAT_ADD(nothing_to_map);
return (ENOTSUP);
}
lib_len = len;
if (padding != 0) {
loadable += 2;
}
if (loadable > *num_mapped) {
*num_mapped = loadable;
/* cleanup previous reservation */
if (start_addr) {
(void) as_unmap(as, start_addr, lib_len);
}
MOBJ_STAT_ADD(e2big);
if (lvp) {
lib_va_release(lvp);
}
return (E2BIG);
}
/*
* We now know the size of the object to map and now we need to
* get the start address to map it at. It's possible we already
* have it if we found all the info we need in the lib_va cache.
*/
if (e_type == ET_DYN && start_addr == NULL) {
/*
* Need to make sure padding does not throw off
* required alignment. We can only specify an
* alignment for the starting address to be mapped,
* so we round padding up to the alignment and map
* from there and then throw out the extra later.
*/
if (padding != 0) {
if (align > 1) {
add_pad = P2ROUNDUP(padding, align);
len += add_pad;
MOBJ_STAT_ADD(dyn_pad_align);
} else {
MOBJ_STAT_ADD(dyn_pad_noalign);
len += padding; /* at beginning */
}
len += padding; /* at end of mapping */
}
/*
* At this point, if lvp is non-NULL, then above we
* already found it in the cache but did not get
* the start address since we were not going to use lib_va.
* Since we know that lib_va will not be used, it's safe
* to call mmapobj_alloc_start_addr and know that lvp
* will not be modified.
*/
ASSERT(lvp ? use_lib_va == 0 : 1);
start_addr = mmapobj_alloc_start_addr(&lvp, len,
use_lib_va, align, &vattr);
if (start_addr == NULL) {
if (lvp) {
lib_va_release(lvp);
}
MOBJ_STAT_ADD(alloc_start_fail);
return (ENOMEM);
}
/*
* If we can't cache it, no need to hang on to it.
* Setting lv_num_segs to non-zero will make that
* field active and since there are too many segments
* to cache, all future users will not try to use lv_mps.
*/
if (lvp != NULL && loadable > LIBVA_CACHED_SEGS && use_lib_va) {
lvp->lv_num_segs = loadable;
lib_va_release(lvp);
lvp = NULL;
MOBJ_STAT_ADD(lvp_nocache);
}
/*
* Free the beginning of the mapping if the padding
* was not aligned correctly.
*/
if (padding != 0 && add_pad != padding) {
(void) as_unmap(as, start_addr,
add_pad - padding);
start_addr += (add_pad - padding);
MOBJ_STAT_ADD(extra_padding);
}
}
/*
* At this point, we have reserved the virtual address space
* for our mappings. Now we need to start filling out the mrp
* array to describe all of the individual mappings we are going
* to return.
* For ET_EXEC there has been no memory reservation since we are
* using fixed addresses. While filling in the mrp array below,
* we will have the first segment biased to start at addr 0
* and the rest will be biased by this same amount. Thus if there
* is padding, the first padding will start at addr 0, and the next
* segment will start at the value of padding.
*/
/* We'll fill out padding later, so start filling in mrp at index 1 */
if (padding != 0) {
current = 1;
}
/* If we have no more need for lvp let it go now */
if (lvp != NULL && use_lib_va == 0) {
lib_va_release(lvp);
MOBJ_STAT_ADD(lvp_not_needed);
lvp = NULL;
}
/* Now fill out the mrp structs from the program headers */
STRUCT_SET_HANDLE(mph, model, (struct myphdr *)phdrbase);
for (i = 0; i < nphdrs; i++) {
p_type = STRUCT_FGET(mph, x.p_type);
if (p_type == PT_LOAD || p_type == PT_SUNWBSS) {
vaddr = (caddr_t)(uintptr_t)STRUCT_FGET(mph, x.p_vaddr);
p_memsz = STRUCT_FGET(mph, x.p_memsz);
p_filesz = STRUCT_FGET(mph, x.p_filesz);
p_offset = STRUCT_FGET(mph, x.p_offset);
p_flags = STRUCT_FGET(mph, x.p_flags);
/*
* Skip this header if it requests no memory to be
* mapped.
*/
if (p_memsz == 0) {
STRUCT_SET_HANDLE(mph, model,
(struct myphdr *)((size_t)STRUCT_BUF(mph) +
hsize));
MOBJ_STAT_ADD(no_mem_map_sz);
continue;
}
prot = 0;
if (p_flags & PF_R)
prot |= PROT_READ;
if (p_flags & PF_W)
prot |= PROT_WRITE;
if (p_flags & PF_X)
prot |= PROT_EXEC;
ASSERT(current < loadable);
mrp[current].mr_msize = p_memsz;
mrp[current].mr_fsize = p_filesz;
mrp[current].mr_offset = p_offset;
mrp[current].mr_prot = prot;
if (hdr_seen == 0 && p_filesz != 0) {
mrp[current].mr_flags = MR_HDR_ELF;
/*
* We modify mr_offset because we
* need to map the ELF header as well, and if
* we didn't then the header could be left out
* of the mapping that we will create later.
* Since we're removing the offset, we need to
* account for that in the other fields as well
* since we will be mapping the memory from 0
* to p_offset.
*/
if (e_type == ET_DYN) {
mrp[current].mr_offset = 0;
mrp[current].mr_msize += p_offset;
mrp[current].mr_fsize += p_offset;
} else {
ASSERT(e_type == ET_EXEC);
/*
* Save off the start addr which will be
* our bias for the rest of the
* ET_EXEC mappings.
*/
start_addr = vaddr - padding;
}
mrp[current].mr_addr = (caddr_t)padding;
hdr_seen = 1;
} else {
if (e_type == ET_EXEC) {
/* bias mr_addr */
mrp[current].mr_addr =
vaddr - (size_t)start_addr;
} else {
mrp[current].mr_addr = vaddr + padding;
}
mrp[current].mr_flags = 0;
}
current++;
}
/* Move to next phdr */
STRUCT_SET_HANDLE(mph, model,
(struct myphdr *)((size_t)STRUCT_BUF(mph) +
hsize));
}
/* Now fill out the padding segments */
if (padding != 0) {
mrp[0].mr_addr = NULL;
mrp[0].mr_msize = padding;
mrp[0].mr_fsize = 0;
mrp[0].mr_offset = 0;
mrp[0].mr_prot = 0;
mrp[0].mr_flags = MR_PADDING;
/* Setup padding for the last segment */
ASSERT(current == loadable - 1);
mrp[current].mr_addr = (caddr_t)lib_len + padding;
mrp[current].mr_msize = padding;
mrp[current].mr_fsize = 0;
mrp[current].mr_offset = 0;
mrp[current].mr_prot = 0;
mrp[current].mr_flags = MR_PADDING;
}
/*
* Need to make sure address ranges desired are not in use or
* are previously allocated reservations from /dev/null. For
* ET_DYN, we already made sure our address range was free.
*/
if (e_type == ET_EXEC) {
ret = check_exec_addrs(loadable, mrp, start_addr);
if (ret != 0) {
ASSERT(lvp == NULL);
MOBJ_STAT_ADD(check_exec_failed);
return (ret);
}
}
/* Finish up our business with lvp. */
if (lvp) {
ASSERT(e_type == ET_DYN);
if (lvp->lv_num_segs == 0 && loadable <= LIBVA_CACHED_SEGS) {
bcopy(mrp, lvp->lv_mps,
loadable * sizeof (mmapobj_result_t));
membar_producer();
}
/*
* Setting lv_num_segs to a non-zero value indicates that
* lv_mps is now valid and can be used by other threads.
* So, the above stores need to finish before lv_num_segs
* is updated. lv_mps is only valid if lv_num_segs is
* greater than LIBVA_CACHED_SEGS.
*/
lvp->lv_num_segs = loadable;
lib_va_release(lvp);
MOBJ_STAT_ADD(lvp_used);
}
/* Now that we have mrp completely filled out go map it */
ret = mmapobj_map_elf(vp, start_addr, mrp, loadable, fcred, e_type);
if (ret == 0) {
*num_mapped = loadable;
}
return (ret);
}
/*
* Take the ELF file passed in, and do the work of mapping it.
* num_mapped in - # elements in user buffer
* num_mapped out - # sections mapped and length of mrp array if
* no errors.
*/
static int
doelfwork(Ehdr *ehdrp, vnode_t *vp, mmapobj_result_t *mrp,
uint_t *num_mapped, size_t padding, cred_t *fcred)
{
int error;
offset_t phoff;
int nphdrs;
unsigned char ei_class;
unsigned short phentsize;
ssize_t phsizep;
caddr_t phbasep;
int to_map;
model_t model;
ei_class = ehdrp->e_ident[EI_CLASS];
model = get_udatamodel();
if ((model == DATAMODEL_ILP32 && ei_class == ELFCLASS64) ||
(model == DATAMODEL_LP64 && ei_class == ELFCLASS32)) {
MOBJ_STAT_ADD(wrong_model);
return (ENOTSUP);
}
/* Can't execute code from "noexec" mounted filesystem. */
if (ehdrp->e_type == ET_EXEC &&
(vp->v_vfsp->vfs_flag & VFS_NOEXEC) != 0) {
MOBJ_STAT_ADD(noexec_fs);
return (EACCES);
}
/*
* Relocatable and core files are mapped as a single flat file
* since no interpretation is done on them by mmapobj.
*/
if (ehdrp->e_type == ET_REL || ehdrp->e_type == ET_CORE) {
to_map = padding ? 3 : 1;
if (*num_mapped < to_map) {
*num_mapped = to_map;
MOBJ_STAT_ADD(e2big_et_rel);
return (E2BIG);
}
error = mmapobj_map_flat(vp, mrp, padding, fcred);
if (error == 0) {
*num_mapped = to_map;
mrp[padding ? 1 : 0].mr_flags = MR_HDR_ELF;
MOBJ_STAT_ADD(et_rel_mapped);
}
return (error);
}
/* Check for an unknown ELF type */
if (ehdrp->e_type != ET_EXEC && ehdrp->e_type != ET_DYN) {
MOBJ_STAT_ADD(unknown_elf_type);
return (ENOTSUP);
}
if (ei_class == ELFCLASS32) {
Elf32_Ehdr *e32hdr = (Elf32_Ehdr *)ehdrp;
ASSERT(model == DATAMODEL_ILP32);
nphdrs = e32hdr->e_phnum;
phentsize = e32hdr->e_phentsize;
if (phentsize < sizeof (Elf32_Phdr)) {
MOBJ_STAT_ADD(phent32_too_small);
return (ENOTSUP);
}
phoff = e32hdr->e_phoff;
} else if (ei_class == ELFCLASS64) {
Elf64_Ehdr *e64hdr = (Elf64_Ehdr *)ehdrp;
ASSERT(model == DATAMODEL_LP64);
nphdrs = e64hdr->e_phnum;
phentsize = e64hdr->e_phentsize;
if (phentsize < sizeof (Elf64_Phdr)) {
MOBJ_STAT_ADD(phent64_too_small);
return (ENOTSUP);
}
phoff = e64hdr->e_phoff;
} else {
/* fallthrough case for an invalid ELF class */
MOBJ_STAT_ADD(inval_elf_class);
return (ENOTSUP);
}
/*
* nphdrs should only have this value for core files which are handled
* above as a single mapping. If other file types ever use this
* sentinel, then we'll add the support needed to handle this here.
*/
if (nphdrs == PN_XNUM) {
MOBJ_STAT_ADD(too_many_phdrs);
return (ENOTSUP);
}
phsizep = nphdrs * phentsize;
if (phsizep == 0) {
MOBJ_STAT_ADD(no_phsize);
return (ENOTSUP);
}
/* Make sure we only wait for memory if it's a reasonable request */
if (phsizep > mmapobj_alloc_threshold) {
MOBJ_STAT_ADD(phsize_large);
if ((phbasep = kmem_alloc(phsizep, KM_NOSLEEP)) == NULL) {
MOBJ_STAT_ADD(phsize_xtralarge);
return (ENOMEM);
}
} else {
phbasep = kmem_alloc(phsizep, KM_SLEEP);
}
if ((error = vn_rdwr(UIO_READ, vp, phbasep, phsizep,
(offset_t)phoff, UIO_SYSSPACE, 0, (rlim64_t)0,
fcred, NULL)) != 0) {
kmem_free(phbasep, phsizep);
return (error);
}
/* Now process the phdr's */
error = process_phdr(ehdrp, phbasep, nphdrs, mrp, vp, num_mapped,
padding, fcred);
kmem_free(phbasep, phsizep);
return (error);
}
#if defined(__sparc)
/*
* Hack to support 64 bit kernels running AOUT 4.x programs.
* This is the sizeof (struct nlist) for a 32 bit kernel.
* Since AOUT programs are 32 bit only, they will never use the 64 bit
* sizeof (struct nlist) and thus creating a #define is the simplest
* way around this since this is a format which is not being updated.
* This will be used in the place of sizeof (struct nlist) below.
*/
#define NLIST_SIZE (0xC)
static int
doaoutwork(vnode_t *vp, mmapobj_result_t *mrp,
uint_t *num_mapped, struct exec *hdr, cred_t *fcred)
{
int error;
size_t size;
size_t osize;
size_t nsize; /* nlist size */
size_t msize;
size_t zfoddiff;
caddr_t addr;
caddr_t start_addr;
struct as *as = curproc->p_as;
int prot = PROT_USER | PROT_READ | PROT_EXEC;
uint_t mflag = MAP_PRIVATE | _MAP_LOW32;
offset_t off = 0;
int segnum = 0;
uint_t to_map;
int is_library = 0;
struct segvn_crargs crargs = SEGVN_ZFOD_ARGS(PROT_ZFOD, PROT_ALL);
/* Only 32bit apps supported by this file format */
if (get_udatamodel() != DATAMODEL_ILP32) {
MOBJ_STAT_ADD(aout_64bit_try);
return (ENOTSUP);
}
/* Check to see if this is a library */
if (hdr->a_magic == ZMAGIC && hdr->a_entry < PAGESIZE) {
is_library = 1;
}
/* Can't execute code from "noexec" mounted filesystem. */
if (((vp->v_vfsp->vfs_flag & VFS_NOEXEC) != 0) && (is_library == 0)) {
MOBJ_STAT_ADD(aout_noexec);
return (EACCES);
}
/*
* There are 2 ways to calculate the mapped size of executable:
* 1) rounded text size + data size + bss size.
* 2) starting offset for text + text size + data size + text relocation
* size + data relocation size + room for nlist data structure.
*
* The larger of the two sizes will be used to map this binary.
*/
osize = P2ROUNDUP(hdr->a_text, PAGESIZE) + hdr->a_data + hdr->a_bss;
off = hdr->a_magic == ZMAGIC ? 0 : sizeof (struct exec);
nsize = off + hdr->a_text + hdr->a_data + hdr->a_trsize +
hdr->a_drsize + NLIST_SIZE;
size = MAX(osize, nsize);
if (size != nsize) {
nsize = 0;
}
/*
* 1 seg for text and 1 seg for initialized data.
* 1 seg for bss (if can't fit in leftover space of init data)
* 1 seg for nlist if needed.
*/
to_map = 2 + (nsize ? 1 : 0) +
(hdr->a_bss > PAGESIZE - P2PHASE(hdr->a_data, PAGESIZE) ? 1 : 0);
if (*num_mapped < to_map) {
*num_mapped = to_map;
MOBJ_STAT_ADD(aout_e2big);
return (E2BIG);
}
/* Reserve address space for the whole mapping */
if (is_library) {
/* We'll let VOP_MAP below pick our address for us */
addr = NULL;
MOBJ_STAT_ADD(aout_lib);
} else {
/*
* default start address for fixed binaries from AOUT 4.x
* standard.
*/
MOBJ_STAT_ADD(aout_fixed);
mflag |= MAP_FIXED;
addr = (caddr_t)0x2000;
as_rangelock(as);
if (as_gap(as, size, &addr, &size, 0, NULL) != 0) {
as_rangeunlock(as);
MOBJ_STAT_ADD(aout_addr_in_use);
return (EADDRINUSE);
}
crargs.flags |= MAP_NORESERVE;
error = as_map(as, addr, size, segvn_create, &crargs);
ASSERT(addr == (caddr_t)0x2000);
as_rangeunlock(as);
}
start_addr = addr;
osize = size;
/*
* Map as large as we need, backed by file, this will be text, and
* possibly the nlist segment. We map over this mapping for bss and
* initialized data segments.
*/
error = VOP_MAP(vp, off, as, &addr, size, prot, PROT_ALL,
mflag, fcred, NULL);
if (error) {
if (!is_library) {
(void) as_unmap(as, start_addr, osize);
}
return (error);
}
/* pickup the value of start_addr and osize for libraries */
start_addr = addr;
osize = size;
/*
* We have our initial reservation/allocation so we need to use fixed
* addresses from now on.
*/
mflag |= MAP_FIXED;
mrp[0].mr_addr = addr;
mrp[0].mr_msize = hdr->a_text;
mrp[0].mr_fsize = hdr->a_text;
mrp[0].mr_offset = 0;
mrp[0].mr_prot = PROT_READ | PROT_EXEC;
mrp[0].mr_flags = MR_HDR_AOUT;
/*
* Map initialized data. We are mapping over a portion of the
* previous mapping which will be unmapped in VOP_MAP below.
*/
off = P2ROUNDUP((offset_t)(hdr->a_text), PAGESIZE);
msize = off;
addr += off;
size = hdr->a_data;
error = VOP_MAP(vp, off, as, &addr, size, PROT_ALL, PROT_ALL,
mflag, fcred, NULL);
if (error) {
(void) as_unmap(as, start_addr, osize);
return (error);
}
msize += size;
mrp[1].mr_addr = addr;
mrp[1].mr_msize = size;
mrp[1].mr_fsize = size;
mrp[1].mr_offset = 0;
mrp[1].mr_prot = PROT_READ | PROT_WRITE | PROT_EXEC;
mrp[1].mr_flags = 0;
/* Need to zero out remainder of page */
addr += hdr->a_data;
zfoddiff = P2PHASE((size_t)addr, PAGESIZE);
if (zfoddiff) {
label_t ljb;
MOBJ_STAT_ADD(aout_zfoddiff);
zfoddiff = PAGESIZE - zfoddiff;
if (on_fault(&ljb)) {
no_fault();
MOBJ_STAT_ADD(aout_uzero_fault);
(void) as_unmap(as, start_addr, osize);
return (EFAULT);
}
uzero(addr, zfoddiff);
no_fault();
}
msize += zfoddiff;
segnum = 2;
/* Map bss */
if (hdr->a_bss > zfoddiff) {
struct segvn_crargs crargs =
SEGVN_ZFOD_ARGS(PROT_ZFOD, PROT_ALL);
MOBJ_STAT_ADD(aout_map_bss);
addr += zfoddiff;
size = hdr->a_bss - zfoddiff;
as_rangelock(as);
(void) as_unmap(as, addr, size);
error = as_map(as, addr, size, segvn_create, &crargs);
as_rangeunlock(as);
msize += size;
if (error) {
MOBJ_STAT_ADD(aout_bss_fail);
(void) as_unmap(as, start_addr, osize);
return (error);
}
mrp[2].mr_addr = addr;
mrp[2].mr_msize = size;
mrp[2].mr_fsize = 0;
mrp[2].mr_offset = 0;
mrp[2].mr_prot = PROT_READ | PROT_WRITE | PROT_EXEC;
mrp[2].mr_flags = 0;
addr += size;
segnum = 3;
}
/*
* If we have extra bits left over, we need to include that in how
* much we mapped to make sure the nlist logic is correct
*/
msize = P2ROUNDUP(msize, PAGESIZE);
if (nsize && msize < nsize) {
MOBJ_STAT_ADD(aout_nlist);
mrp[segnum].mr_addr = addr;
mrp[segnum].mr_msize = nsize - msize;
mrp[segnum].mr_fsize = 0;
mrp[segnum].mr_offset = 0;
mrp[segnum].mr_prot = PROT_READ | PROT_EXEC;
mrp[segnum].mr_flags = 0;
}
*num_mapped = to_map;
return (0);
}
#endif
/*
* These are the two types of files that we can interpret and we want to read
* in enough info to cover both types when looking at the initial header.
*/
#define MAX_HEADER_SIZE (MAX(sizeof (Ehdr), sizeof (struct exec)))
/*
* Map vp passed in in an interpreted manner. ELF and AOUT files will be
* interpreted and mapped appropriately for execution.
* num_mapped in - # elements in mrp
* num_mapped out - # sections mapped and length of mrp array if
* no errors or E2BIG returned.
*
* Returns 0 on success, errno value on failure.
*/
static int
mmapobj_map_interpret(vnode_t *vp, mmapobj_result_t *mrp,
uint_t *num_mapped, size_t padding, cred_t *fcred)
{
int error = 0;
vattr_t vattr;
struct lib_va *lvp;
caddr_t start_addr;
model_t model;
/*
* header has to be aligned to the native size of ulong_t in order
* to avoid an unaligned access when dereferencing the header as
* a ulong_t. Thus we allocate our array on the stack of type
* ulong_t and then have header, which we dereference later as a char
* array point at lheader.
*/
ulong_t lheader[(MAX_HEADER_SIZE / (sizeof (ulong_t))) + 1];
caddr_t header = (caddr_t)&lheader;
vattr.va_mask = AT_FSID | AT_NODEID | AT_CTIME | AT_MTIME | AT_SIZE;
error = VOP_GETATTR(vp, &vattr, 0, fcred, NULL);
if (error) {
return (error);
}
/*
* Check lib_va to see if we already have a full description
* for this library. This is the fast path and only used for
* ET_DYN ELF files (dynamic libraries).
*/
if (padding == 0 && (lvp = lib_va_find(&vattr)) != NULL) {
int num_segs;
model = get_udatamodel();
if ((model == DATAMODEL_ILP32 &&
lvp->lv_flags & LV_ELF64) ||
(model == DATAMODEL_LP64 &&
lvp->lv_flags & LV_ELF32)) {
lib_va_release(lvp);
MOBJ_STAT_ADD(fast_wrong_model);
return (ENOTSUP);
}
num_segs = lvp->lv_num_segs;
if (*num_mapped < num_segs) {
*num_mapped = num_segs;
lib_va_release(lvp);
MOBJ_STAT_ADD(fast_e2big);
return (E2BIG);
}
/*
* Check to see if we have all the mappable program headers
* cached.
*/
if (num_segs <= LIBVA_CACHED_SEGS && num_segs != 0) {
MOBJ_STAT_ADD(fast);
start_addr = mmapobj_lookup_start_addr(lvp);
if (start_addr == NULL) {
lib_va_release(lvp);
return (ENOMEM);
}
bcopy(lvp->lv_mps, mrp,
num_segs * sizeof (mmapobj_result_t));
error = mmapobj_map_elf(vp, start_addr, mrp,
num_segs, fcred, ET_DYN);
lib_va_release(lvp);
if (error == 0) {
*num_mapped = num_segs;
MOBJ_STAT_ADD(fast_success);
}
return (error);
}
MOBJ_STAT_ADD(fast_not_now);
/* Release it for now since we'll look it up below */
lib_va_release(lvp);
}
/*
* Time to see if this is a file we can interpret. If it's smaller
* than this, then we can't interpret it.
*/
if (vattr.va_size < MAX_HEADER_SIZE) {
MOBJ_STAT_ADD(small_file);
return (ENOTSUP);
}
if ((error = vn_rdwr(UIO_READ, vp, header, MAX_HEADER_SIZE, 0,
UIO_SYSSPACE, 0, (rlim64_t)0, fcred, NULL)) != 0) {
MOBJ_STAT_ADD(read_error);
return (error);
}
/* Verify file type */
if (header[EI_MAG0] == ELFMAG0 && header[EI_MAG1] == ELFMAG1 &&
header[EI_MAG2] == ELFMAG2 && header[EI_MAG3] == ELFMAG3) {
return (doelfwork((Ehdr *)lheader, vp, mrp, num_mapped,
padding, fcred));
}
#if defined(__sparc)
/* On sparc, check for 4.X AOUT format */
switch (((struct exec *)header)->a_magic) {
case OMAGIC:
case ZMAGIC:
case NMAGIC:
return (doaoutwork(vp, mrp, num_mapped,
(struct exec *)lheader, fcred));
}
#endif
/* Unsupported type */
MOBJ_STAT_ADD(unsupported);
return (ENOTSUP);
}
/*
* Given a vnode, map it as either a flat file or interpret it and map
* it according to the rules of the file type.
* *num_mapped will contain the size of the mmapobj_result_t array passed in.
* If padding is non-zero, the mappings will be padded by that amount
* rounded up to the nearest pagesize.
* If the mapping is successful, *num_mapped will contain the number of
* distinct mappings created, and mrp will point to the array of
* mmapobj_result_t's which describe these mappings.
*
* On error, -1 is returned and errno is set appropriately.
* A special error case will set errno to E2BIG when there are more than
* *num_mapped mappings to be created and *num_mapped will be set to the
* number of mappings needed.
*/
int
mmapobj(vnode_t *vp, uint_t flags, mmapobj_result_t *mrp,
uint_t *num_mapped, size_t padding, cred_t *fcred)
{
int to_map;
int error = 0;
ASSERT((padding & PAGEOFFSET) == 0);
ASSERT((flags & ~MMOBJ_ALL_FLAGS) == 0);
ASSERT(num_mapped != NULL);
ASSERT((flags & MMOBJ_PADDING) ? padding != 0 : padding == 0);
if ((flags & MMOBJ_INTERPRET) == 0) {
to_map = padding ? 3 : 1;
if (*num_mapped < to_map) {
*num_mapped = to_map;
MOBJ_STAT_ADD(flat_e2big);
return (E2BIG);
}
error = mmapobj_map_flat(vp, mrp, padding, fcred);
if (error) {
return (error);
}
*num_mapped = to_map;
return (0);
}
error = mmapobj_map_interpret(vp, mrp, num_mapped, padding, fcred);
return (error);
}