OpenSolaris_b135/lib/libsqlite/tool/spaceanal.tcl
#pragma ident "%Z%%M% %I% %E% SMI"
# Run this TCL script using "testfixture" in order get a report that shows
# how much disk space is used by a particular data to actually store data
# versus how much space is unused.
#
# Get the name of the database to analyze
#
if {[llength $argv]!=1} {
puts stderr "Usage: $argv0 database-name"
exit 1
}
set file_to_analyze [lindex $argv 0]
if {![file exists $file_to_analyze]} {
puts stderr "No such file: $file_to_analyze"
exit 1
}
if {![file readable $file_to_analyze]} {
puts stderr "File is not readable: $file_to_analyze"
exit 1
}
if {[file size $file_to_analyze]<2048} {
puts stderr "Empty or malformed database: $file_to_analyze"
exit 1
}
# Open the database
#
sqlite db [lindex $argv 0]
set DB [btree_open [lindex $argv 0]]
# In-memory database for collecting statistics
#
sqlite mem :memory:
set tabledef\
{CREATE TABLE space_used(
name clob, -- Name of a table or index in the database file
tblname clob, -- Name of associated table
is_index boolean, -- TRUE if it is an index, false for a table
nentry int, -- Number of entries in the BTree
payload int, -- Total amount of data stored in this table or index
mx_payload int, -- Maximum payload size
n_ovfl int, -- Number of entries that overflow
pri_pages int, -- Number of primary pages used
ovfl_pages int, -- Number of overflow pages used
pri_unused int, -- Number of unused bytes on primary pages
ovfl_unused int -- Number of unused bytes on overflow pages
);}
mem eval $tabledef
# This query will be used to find the root page number for every index and
# table in the database.
#
set sql {
SELECT name, tbl_name, type, rootpage
FROM sqlite_master WHERE type IN ('table','index')
UNION ALL
SELECT 'sqlite_master', 'sqlite_master', 'table', 2
ORDER BY 1
}
# Analyze every table in the database, one at a time.
#
foreach {name tblname type rootpage} [db eval $sql] {
puts stderr "Analyzing $name..."
set cursor [btree_cursor $DB $rootpage 0]
set go [btree_first $cursor]
set size 0
catch {unset pg_used}
set unused_ovfl 0
set n_overflow 0
set cnt_ovfl 0
set n_entry 0
set mx_size 0
set pg_used($rootpage) 1016
while {$go==0} {
incr n_entry
set payload [btree_payload_size $cursor]
incr size $payload
set stat [btree_cursor_dump $cursor]
set pgno [lindex $stat 0]
set freebytes [lindex $stat 4]
set pg_used($pgno) $freebytes
if {$payload>236} {
# if {[lindex $stat 8]==0} {error "overflow is empty with $payload"}
set n [expr {($payload-236+1019)/1020}]
incr n_overflow $n
incr cnt_ovfl
incr unused_ovfl [expr {$n*1020+236-$payload}]
} else {
# if {[lindex $stat 8]!=0} {error "overflow not empty with $payload"}
}
if {$payload>$mx_size} {set mx_size $payload}
set go [btree_next $cursor]
}
btree_close_cursor $cursor
set n_primary [llength [array names pg_used]]
set unused_primary 0
foreach x [array names pg_used] {incr unused_primary $pg_used($x)}
regsub -all ' $name '' name
set sql "INSERT INTO space_used VALUES('$name'"
regsub -all ' $tblname '' tblname
append sql ",'$tblname',[expr {$type=="index"}],$n_entry"
append sql ",$size,$mx_size,$cnt_ovfl,"
append sql "$n_primary,$n_overflow,$unused_primary,$unused_ovfl);"
mem eval $sql
}
# Generate a single line of output in the statistics section of the
# report.
#
proc statline {title value {extra {}}} {
set len [string length $title]
set dots [string range {......................................} $len end]
set len [string length $value]
set sp2 [string range { } $len end]
if {$extra ne ""} {
set extra " $extra"
}
puts "$title$dots $value$sp2$extra"
}
# Generate a formatted percentage value for $num/$denom
#
proc percent {num denom} {
if {$denom==0.0} {return ""}
set v [expr {$num*100.0/$denom}]
if {$v>1.0 && $v<99.0} {
return [format %4.1f%% $v]
} elseif {$v<0.1 || $v>99.9} {
return [format %6.3f%% $v]
} else {
return [format %5.2f%% $v]
}
}
# Generate a subreport that covers some subset of the database.
# the $where clause determines which subset to analyze.
#
proc subreport {title where} {
set hit 0
mem eval "SELECT sum(nentry) AS nentry, \
sum(payload) AS payload, \
sum(CASE is_index WHEN 1 THEN 0 ELSE payload-4*nentry END) \
AS data, \
max(mx_payload) AS mx_payload, \
sum(n_ovfl) as n_ovfl, \
sum(pri_pages) AS pri_pages, \
sum(ovfl_pages) AS ovfl_pages, \
sum(pri_unused) AS pri_unused, \
sum(ovfl_unused) AS ovfl_unused \
FROM space_used WHERE $where" {} {set hit 1}
if {!$hit} {return 0}
puts ""
set len [string length $title]
incr len 5
set stars "***********************************"
append stars $stars
set stars [string range $stars $len end]
puts "*** $title $stars"
puts ""
statline "Percentage of total database" \
[percent [expr {$pri_pages+$ovfl_pages}] $::file_pgcnt]
statline "Number of entries" $nentry
set storage [expr {($pri_pages+$ovfl_pages)*1024}]
statline "Bytes of storage consumed" $storage
statline "Bytes of payload" $payload [percent $payload $storage]
statline "Bytes of data" $data [percent $data $storage]
set key [expr {$payload-$data}]
statline "Bytes of key" $key [percent $key $storage]
set avgpay [expr {$nentry>0?$payload/$nentry:0}]
statline "Average payload per entry" $avgpay
set avgunused [expr {$nentry>0?($pri_unused+$ovfl_unused)/$nentry:0}]
statline "Average unused bytes per entry" $avgunused
statline "Average fanout" \
[format %.2f [expr {$pri_pages==0?0:($nentry+0.0)/$pri_pages}]]
statline "Maximum payload per entry" $mx_payload
statline "Entries that use overflow" $n_ovfl [percent $n_ovfl $nentry]
statline "Total pages used" [set allpgs [expr {$pri_pages+$ovfl_pages}]]
statline "Primary pages used" $pri_pages ;# [percent $pri_pages $allpgs]
statline "Overflow pages used" $ovfl_pages ;# [percent $ovfl_pages $allpgs]
statline "Unused bytes on primary pages" $pri_unused \
[percent $pri_unused [expr {$pri_pages*1024}]]
statline "Unused bytes on overflow pages" $ovfl_unused \
[percent $ovfl_unused [expr {$ovfl_pages*1024}]]
set allunused [expr {$ovfl_unused+$pri_unused}]
statline "Unused bytes on all pages" $allunused \
[percent $allunused [expr {$allpgs*1024}]]
return 1
}
# Output summary statistics:
#
puts "/** Disk-Space Utilization Report For $file_to_analyze"
puts "*** As of [clock format [clock seconds] -format {%Y-%b-%d %H:%M:%S}]"
puts ""
set fsize [file size [lindex $argv 0]]
set file_pgcnt [expr {$fsize/1024}]
set usedcnt [mem eval {SELECT sum(pri_pages+ovfl_pages) FROM space_used}]
set freecnt [expr {$file_pgcnt-$usedcnt-1}]
set freecnt2 [lindex [btree_get_meta $DB] 0]
statline {Pages in the whole file (measured)} $file_pgcnt
set file_pgcnt2 [expr {$usedcnt+$freecnt2+1}]
statline {Pages in the whole file (calculated)} $file_pgcnt2
statline {Pages that store data} $usedcnt [percent $usedcnt $file_pgcnt]
statline {Pages on the freelist (per header)}\
$freecnt2 [percent $freecnt2 $file_pgcnt]
statline {Pages on the freelist (calculated)}\
$freecnt [percent $freecnt $file_pgcnt]
statline {Header pages} 1 [percent 1 $file_pgcnt]
set ntable [db eval {SELECT count(*)+1 FROM sqlite_master WHERE type='table'}]
statline {Number of tables in the database} $ntable
set nindex [db eval {SELECT count(*) FROM sqlite_master WHERE type='index'}]
set autoindex [db eval {SELECT count(*) FROM sqlite_master
WHERE type='index' AND name LIKE '(% autoindex %)'}]
set manindex [expr {$nindex-$autoindex}]
statline {Number of indices} $nindex
statline {Number of named indices} $manindex [percent $manindex $nindex]
statline {Automatically generated indices} $autoindex \
[percent $autoindex $nindex]
set bytes_data [mem eval "SELECT sum(payload-4*nentry) FROM space_used
WHERE NOT is_index AND name!='sqlite_master'"]
set total_payload [mem eval "SELECT sum(payload) FROM space_used"]
statline "Size of the file in bytes" $fsize
statline "Bytes of payload stored" $total_payload \
[percent $total_payload $fsize]
statline "Bytes of user data stored" $bytes_data \
[percent $bytes_data $fsize]
# Output table rankings
#
puts ""
puts "*** Page counts for all tables with their indices ********************"
puts ""
mem eval {SELECT tblname, count(*) AS cnt, sum(pri_pages+ovfl_pages) AS size
FROM space_used GROUP BY tblname ORDER BY size DESC, tblname} {} {
statline [string toupper $tblname] $size [percent $size $file_pgcnt]
}
# Output subreports
#
if {$nindex>0} {
subreport {All tables and indices} 1
}
subreport {All tables} {NOT is_index}
if {$nindex>0} {
subreport {All indices} {is_index}
}
foreach tbl [mem eval {SELECT name FROM space_used WHERE NOT is_index
ORDER BY name}] {
regsub ' $tbl '' qn
set name [string toupper $tbl]
set n [mem eval "SELECT count(*) FROM space_used WHERE tblname='$qn'"]
if {$n>1} {
subreport "Table $name and all its indices" "tblname='$qn'"
subreport "Table $name w/o any indices" "name='$qn'"
subreport "Indices of table $name" "tblname='$qn' AND is_index"
} else {
subreport "Table $name" "name='$qn'"
}
}
# Output instructions on what the numbers above mean.
#
puts {
*** Definitions ******************************************************
Number of pages in the whole file
The number of 1024-byte pages that go into forming the complete database
Pages that store data
The number of pages that store data, either as primary B*Tree pages or
as overflow pages. The number at the right is the data pages divided by
the total number of pages in the file.
Pages on the freelist
The number of pages that are not currently in use but are reserved for
future use. The percentage at the right is the number of freelist pages
divided by the total number of pages in the file.
Header pages
The number of pages of header overhead in the database. This value is
always 1. The percentage at the right is the number of header pages
divided by the total number of pages in the file.
Number of tables in the database
The number of tables in the database, including the SQLITE_MASTER table
used to store schema information.
Number of indices
The total number of indices in the database.
Number of named indices
The number of indices created using an explicit CREATE INDEX statement.
Automatically generated indices
The number of indices used to implement PRIMARY KEY or UNIQUE constraints
on tables.
Size of the file in bytes
The total amount of disk space used by the entire database files.
Bytes of payload stored
The total number of bytes of payload stored in the database. Payload
includes both key and data. The content of the SQLITE_MASTER table is
counted when computing this number. The percentage at the right shows
the payload divided by the total file size.
Bytes of user data stored
The total number of bytes of data stored in the database, not counting
the database schema information stored in the SQLITE_MASTER table. The
percentage at the right is the user data size divided by the total file
size.
Percentage of total database
The amount of the complete database file that is devoted to storing
information described by this category.
Number of entries
The total number of B*Tree key/value pairs stored under this category.
Bytes of storage consumed
The total amount of disk space required to store all B*Tree entries
under this category. The is the total number of pages used times
the pages size (1024).
Bytes of payload
The amount of payload stored under this category. Payload is the sum
of keys and data. Each table entry has 4 bytes of key and an arbitrary
amount of data. Each index entry has 4 or more bytes of key and no
data. The percentage at the right is the bytes of payload divided by
the bytes of storage consumed.
Bytes of data
The amount of data stored under this category. The data space reported
includes formatting information such as nul-terminators and field-lengths
that are stored with the data. The percentage at the right is the bytes
of data divided by bytes of storage consumed.
Bytes of key
The sum of the sizes of all keys under this category. The percentage at
the right is the bytes of key divided by the bytes of storage consumed.
Average payload per entry
The average amount of payload on each entry. This is just the bytes of
payload divided by the number of entries.
Average unused bytes per entry
The average amount of free space remaining on all pages under this
category on a per-entry basis. This is the number of unused bytes on
all pages divided by the number of entries.
Maximum payload per entry
The largest payload size of any entry.
Entries that use overflow
Up to 236 bytes of payload for each entry are stored directly in the
primary B*Tree page. Any additional payload is stored on a linked list
of overflow pages. This is the number of entries that exceed 236 bytes
in size. The value to the right is the number of entries that overflow
divided by the total number of entries.
Total pages used
This is the number of 1024 byte pages used to hold all information in
the current category. This is the sum of primary and overflow pages.
Primary pages used
This is the number of primary B*Tree pages used.
Overflow pages used
The total number of overflow pages used for this category.
Unused bytes on primary pages
The total number of bytes of unused space on all primary pages. The
percentage at the right is the number of unused bytes divided by the
total number of bytes on primary pages.
Unused bytes on overflow pages
The total number of bytes of unused space on all overflow pages. The
percentage at the right is the number of unused bytes divided by the
total number of bytes on overflow pages.
Unused bytes on all pages
The total number of bytes of unused space on all primary and overflow
pages. The percentage at the right is the number of unused bytes
divided by the total number of bytes.
}
# Output the database
#
puts "**********************************************************************"
puts "The entire text of this report can be sourced into any SQL database"
puts "engine for further analysis. All of the text above is an SQL comment."
puts "The data used to generate this report follows:"
puts "*/"
puts "BEGIN;"
puts $tabledef
unset -nocomplain x
mem eval {SELECT * FROM space_used} x {
puts -nonewline "INSERT INTO space_used VALUES("
regsub ' $x(name) '' qn
regsub ' $x(tblname) '' qtn
puts -nonewline "'$qn','$qtn',"
puts -nonewline "$x(is_index),$x(nentry),$x(payload),$x(mx_payload),"
puts -nonewline "$x(n_ovfl),$x(pri_pages),$x(ovfl_pages),$x(pri_unused),"
puts "$x(ovfl_unused));"
}
puts "COMMIT;"