Project

General

Profile

Feature #16335 » benchmark-stats.rb

colin13rg (Colin Bartlett), 11/12/2019 02:16 PM

 

#:TODO:379: maybe get a new value for nanos from the line
#:TODO: in *.c benchmarking programs maybe store average and plus from loop values as "%.9e" (or even more decimals) and don't scale to nanos - or maybe put the "%.9e" after the nanos

data_file_v = case -12
when -1 then "benchmark--cruby--date_core--gregorian--stats-data--lenovo-1.txt"
when -2 then "benchmark--cruby--date_core--gregorian--stats-data--lenovo-2.txt"
when -12 then "benchmark--cruby--date_core--gregorian--stats-data--lenovo-1-2.txt"
when 1 then "benchmark--cruby--date_core--gregorian--stats-data--samsung-1.txt"
when 2 then "benchmark--cruby--date_core--gregorian--stats-data--samsung-2.txt"
when 12 then "benchmark--cruby--date_core--gregorian--stats-data--samsung-1-2.txt"
else nil
end

Benchmark_stats.new(data_file_v, 0 == 1 ? 0.017e-9 : nil)


BEGIN {

class Benchmark_stats

ENCODING = "ASCII-8bit".encode("ASCII-8bit").freeze

SECOND_ABBREVIATIONS = "afpnums".encode(ENCODING).freeze
SECOND_ABBREVIATIONS_REGEXP = /(?:[afpnums])/i
SECOND_SCALE_TEN = [-18, -15, -12, -9, -6, -3, 0].freeze
SECOND_SCALE = [1e-18, 1e-15, 1e-12, 1e-9, 1e-6, 1e-3, 1e0].freeze
LOOP_ID = "loop".freeze

FORMAT_LENGTH_TYPE = "TYPE".freeze
FORMAT_LENGTH_FUNCTION = "FUNCTION".freeze
FORMAT_LENGTH_OPTIMIZE = "OPTIMIZE".freeze

INDEX_TYPE = 0
INDEX_FUNCTION = 1
INDEX_OPTIMIZE = 2

INDEX_COUNT = 0
INDEX_AVERAGE = 1
INDEX_VARIANCE = 2
INDEX_MIN = 3
INDEX_MAX = 4
INDEX_SHOW = 5

INDEX_NANOS_PLUS = INDEX_OPTIMIZE + 1
INDEX_NANOS = INDEX_NANOS_PLUS + INDEX_SHOW + 1

INDEXES_NANOS = [INDEX_NANOS_PLUS, INDEX_NANOS]

INDEX_FORMAT_LENGTH_NANOS_PLUS = 0
INDEX_FORMAT_LENGTH_NANOS = 2

def initialize(data_filev = nil, less_than_loop_duration_tolerancev = nil)
unless data_filev.kind_of?(String) || (! data_filev)
# When accidentally called with datafilev = 0.125
# this hanged the program at :readline.
puts "#:ERROR: data_filev must be nil or a String, but is: #{data_filev.inspect}"
exit
end

@data_file = data_filev || "./benchmark-stats-data.txt"
# If negative then ignore data items with slightly negative difference;
# if positive, include them but change value to 0.0.
@less_than_loop_duration_tolerance =
case less_than_loop_duration_tolerancev
when Integer then less_than_loop_duration_tolerancev / 1e9
else less_than_loop_duration_tolerancev
end
@line
@stats_hash = Hash.new
@stats_length_hash = Hash.new
@exe_hash = Hash.new
@name_to_id_hash = Hash.new
@id_to_name_hash = Hash.new
@max_id_type = @max_id_function = @max_id_optimize = nil
@max_id = 10 - 1
@benchtype = @exe = nil
@type = @function = @optimize = nil
@type_id = @function_id = @optimize_id = nil
@loop_nanos = @loop_nanos_string = nil
@line_number = nil
@error_count = @warning_count = nil
@show_only_var = nil

puts
puts
puts "# benchmark stats data file: #{@data_file}"
puts

parse_lines!()
if @error_count && @error_count != 0
exit
end

puts "# all average durations shown are in nanoseconds"
puts

######## Change these to get averages and/or variations
show_stats(false, false, false, false)
# show_stats(true, true, true, true)
# show_stats(true, false, false, true)
puts
puts
end

def calculate_stats(q_show_stdev = false, q_show_range = false, q_show_min_max = false, q_show_only_var = false)
@nanos_plus_and_actual = 0
@show_only_var = q_show_only_var
@stats_hash.each do |keyv, vvv|
INDEXES_NANOS.each do |ii|
if (countv = vvv[ii + INDEX_COUNT]) && countv > 0
avg = vvv[ii + INDEX_AVERAGE] / countv
@nanos_plus_and_actual |= ii == INDEX_NANOS_PLUS ? 1 : 2
if q_show_only_var
vs = "".dup
else
vs = (ii == INDEX_NANOS_PLUS ? "%+.1f" : "%.1f") % avg
end
avglen = vs.length
if (q_show_stdev || q_show_range || q_show_min_max) && countv >= 1
if q_show_stdev && vv = vvv[ii + INDEX_VARIANCE]
vv = vv / countv - avg * avg
vs << "~"
if countv < 2
vs << "Z"
elsif vv == 0
vs << "z"
elsif q_show_stdev != :v && (v = 100.0 * vv / avg) < 99
vs << ("%.0f" % v.ceil) << "%"
elsif vv < 99
vs << ("%.1f" % vv)
else
vs << "B"
end
end
vmin = vvv[ii + INDEX_MIN]
vmax = vvv[ii + INDEX_MAX]
if q_show_range && vmin && vmax
vv = vmax - vmin
vs << ":"
if countv < 2
vs << "Z"
elsif vv == 0
vs << "z"
elsif q_show_range != :v && (v = 100.0 * vv / avg) < 99
vs << ("%.0f" % v.ceil) << "%"
elsif vv < 99
vs << ("%.1f" % vv)
else
vs << "B"
end
end
if q_show_min_max && vmin && vmax
vmin -= avg
vmax -= avg
if countv < 2
vs << "-+Z"
elsif vmin == 0 && vmax == 0
vs << "-+z"
else
if q_show_min_max != :v && (v = 100.0 * vmin / avg) > -99
vs << ("%.0f" % v.floor) << "%"
elsif vv < 99
vs << ("%.1f" % vmin)
else
vs << "-B"
end
if q_show_min_max != :v && (v = 100.0 * vmax / avg) < 99
vs << ("%+.0f" % v.ceil) << "%"
elsif vmax < 99
vs << ("%+.1f" % vmax)
else
vs << "+B"
end
end
end
end
varvlen = vs.length - avglen
vvv[ii + INDEX_SHOW] = vs
@optimize = keyv[INDEX_OPTIMIZE]
unless vv = @stats_length_hash[@optimize]
@stats_length_hash[@optimize] = vv = [0, 0, 0, 0]
end
#:TODO: iii = ii == INDEX_NANOS_PLUS ? INDEX_FORMAT_LENGTH_NANOS_PLUS : INDEX_FORMAT_LENGTH_NANOS
iii = INDEX_FORMAT_LENGTH_NANOS_PLUS
vv[iii] = avglen if vv[iii] < avglen
iii += 1
vv[iii] = varvlen if vv[iii] < varvlen
end
end
end
end

def show_stats(q_show_stdev = false, q_show_range = false, q_show_min_max = false, q_show_only_var = false)
calculate_stats(q_show_stdev, q_show_range, q_show_min_max, q_show_only_var)
puts
puts "## exe:"
arrayv = Array.new
@exe_hash.each { |keyv, valuev| arrayv << [valuev, keyv] }
arrayv.sort!.each do |vk|
puts "# #{vk[0]}: #{vk[1]}"
end
puts
arrayv = @stats_hash.to_a
arrayv.each do |vvv|
vv = vvv[0]
vv[0] = @name_to_id_hash[vv[0]]
vv[1] = @name_to_id_hash[vv[1]]
vv[2] = @name_to_id_hash[vv[2]]
end
arrayv.sort!

#:TODO: For now use: across page: exe:0 exe:1 exe:2 exe3:
#:TODO: down page: loop; stat_id
#:TODO: function
#:TODO: Later if only one exe maybe allow stat_id or function across page

linev = stat_description = nil
arrayv.each do |vvv|
vv = vvv[1]
if (vo = vv[INDEX_OPTIMIZE]) == "-O0"
if linev
#:TODO:
linev << " " << stat_description
puts if @function == "loop"
puts linev
end
stat_description = nil
linev = "".dup
end
if linev.length == 0
linev << " "
else
linev << " "
end
@type = vv[INDEX_TYPE]
@function = vv[INDEX_FUNCTION]
@optimize = vo
unless stat_description
stat_description = @function.dup
if @function == "loop"
stat_description << ": " << @type
end
end
v = vv[(vi = INDEX_NANOS_PLUS) + INDEX_SHOW] || vv[(vi = INDEX_NANOS) + INDEX_SHOW]
vv = @stats_length_hash[@optimize]
#:TODO: vi = vi == INDEX_NANOS_PLUS ? 0 : 2
vi = 0
ii = @show_only_var ? 0 : v.index(/(?:[^\d.]|\z)/, 1)
#:TODO:# p [ :v, v, :ii, ii, :len, vv[vi], vv[vi + 1], :pad, vv[vi] - ii - (v.length - ii), vv[vi + 1], :vi, vi ]
if (iii = vv[vi + 1] - (v.length - ii)) > 0
v << " " * iii
end
if (iii = vv[vi] - ii) > 0
v[0, 0] = " " * iii
end
linev << v
end
puts linev << " " << stat_description if linev
end

def get_type(index_type_after)
#: : type 1; y (1901..2099) * 3000;
#: : sg 2299161.000000, iv 2; y (1901..2099) * 3000;
#:TODO: for @type maybe delete: "type 9" y ""9999..9999") * 30000"

return unless @line && index_type_after

ii = @line.index(/(?:[^\/\\,;:# ])/)
ii = @line.index(/(?: |: )/, ii)
vv = @line[ii, index_type_after - ii]
iii = vv.rindex(") * ")
vv[iii, vv.length - iii] = ""
ii = vv.index(" type ")
ii = vv.index(/(?:\S)/, ii + 5)
vv[0, ii] = ""
vv.sub!("; y (", ": y ")
@type = vv.freeze
end

def parse_lines!()
puts
puts
v = @less_than_loop_duration_tolerance ? "%.3e" % @less_than_loop_duration_tolerance : "none"
puts "# Ruby benchmark-stats.rb: less_than_loop_duration_tolerance= #{v}"
puts "# benchmark stats data file: #{@data_file}"
puts "# all average durations shown are in nanoseconds"
puts

@line_number = 0
@error_count = nil

File.open(@data_file, "r").each_line do |linev|
@line_number += 1
@line = linev
qv = parse_line!()
if qv
puts qv
puts "#: line #{@line_number}: #{@line}"
if qv.index(/(?:warning)/i)
@warning_count = (@warning_count || 0) + 1
else
@error_count = (@error_count || 0) + 1
exit
end
end
end

puts
puts "# benchmark stats data file: #{@data_file}"
puts "# at: #{Time.now}"
v = @error_count || "no"
vv = @warning_count || "no"
puts "#: #{v} errors and #{vv} warnings in benchmark stats data file"
puts
return @error_count
end

def parse_line!()
@line.sub!(/(?:\s+\z)/, "")

#: # benchmark: type 1; y (1901..2099) * 3000;
#: # c_julian_leap_p: type 1; y (1901..2099) * 3000; run 1; sum 120000000; 2.184s +1.3ns;
#: # loop: type 1; y (1901..2099) * 3000; run 1; sum 0; 1.544s 3.2ns;
#: # null_loop: sg 2299161.000000, iv 2; y (1901..2099) * 3000; run 1; sum 0; 0.000s 2.0ns;

if iii = @line.rindex(/(?:\.exe\s*)/i)
if (! @line.index(/(?:\d\d[\-\/]\d\d[\-\/]\d\d)/)) && (ii = @line.rindex(/(?:\A|[\/\\>: ])/, iii))
iii += 4
@line[iii, @line.length - iii] = ""
ii += 1 unless ii == 0 && @line.rindex(/(?:[^\/\\>: ])/, 0) == 0
@line[0, ii] = ""
@exe = @line
ii = @line.rindex("-O")
@optimize = @exe[ii, @exe.index(".", ii) - ii]
unless @exe_hash[@exe]
@exe_hash[@exe] = v = case @optimize
when "-Ofast" then 4
when "-Og" then 5
when "-Os" then 6
else
begin
v = @optimize[2, @optimize.length - 2].to_i
v <= 3 ? v : 8
rescue
9
end
end
end
unless @optimize_id = @name_to_id_hash[@optimize]
@name_to_id_hash[@optimize] = @optimize_id = v
@id_to_name_hash[v] = @optimize
end
end
return nil
end

@type = @function = q_error_warning = qnegnanos = nil
if (ii = @line.index(/(?:\A *#* *benchmark *:)/i))
index_type_after = @line.length
elsif ii = @line.rindex(/(?: [-+]?\d[\d_,]*\d?(?:\.\d+)?(?:e[\-+]?\d+)? *[munpfa]?s)/i)
#:TODO: what if, for example, "+ 13.42"? Simplest solution: don't allow it!
iii = @line.index(/(?: *[munpfa]?s)/i, ii)
scalev = @line.index(SECOND_ABBREVIATIONS_REGEXP, iii)
(scalev = @line[scalev, 1]).downcase!
scalev = SECOND_SCALE[SECOND_ABBREVIATIONS.index(scalev)]
ii += 1
nanos = @line[ii, iii -ii]

index_type_after = @line.rindex(/(?:[,;:# ]run *(?:[\-: ] *\d+[,;:# ]))/i, ii)

is_loop = @line.index(/(?:\A *#* *(?:loop|null(?:[- _:]*loop))(?: | *:))/i)

nanos_plus_minus = nanos.rindex("+", 0) ? 1 : nanos.rindex("-", 0) ? -1 : 0

begin
nanos = (vvv = nanos).to_f
nanos *= scalev if scalev != 1.0
nanos *= 1e9 #:TODO: later change this so we work in seconds, and change "nanos" to "sperit" - seconds per iteration
rescue
nanos = nil
end
if ! nanos
q_error_warning = "#:ERROR: invalid nanos"
elsif nanos_plus_minus < 0 || nanos < 0
if is_loop
q_error_warning = "#:ERROR: negative loop nanos"
else
qnegnanos = true
end
elsif is_loop
@loop_nanos = nanos
@loop_nanos_string = vvv
elsif @loop_nanos && nanos_plus_minus == 0
nanos_plus_minus = 1
nanos -= @loop_nanos
qnegnanos = true if nanos < 0
end
if qnegnanos
# @less_than_loop_duration_tolerance
#:TODO: p [ :at, 375, :nanos, nanos, :nanos_plus_minus, nanos_plus_minus ]
#:TODO: p [ :at, 376, :less_than_loop_duration_tolerance, @less_than_loop_duration_tolerance, :nanos, "%.8e" % nanos, :loop_nanos, @loop_nanos ]
#:TODO:379: maybe get a new value for nanos from the line
#:TODO: in *.c benchmarking programs maybe store average and plus from loop values as "%.9e" (or even more decimals) and don't scale to nanos - or maybe put the "%.9e" after the nanos
q_error_warning = " loop_nanos '#{@loop_nanos_string}' #{'%.3e' % @loop_nanos};"
q_error_warning << " nanos '#{vvv}' #{'%.2e' % nanos};"
if @less_than_loop_duration_tolerance && nanos.abs <= @less_than_loop_duration_tolerance.abs
q_error_warning[0, 0] = "#:Warning:"
q_error_warning << " but within tolerance #{'%+.2e' % @less_than_loop_duration_tolerance};"
if @less_than_loop_duration_tolerance < 0.0
q_error_warning << " Discarded"
nanos = nil
else
q_error_warning << " Used as zero"
nanos = 0.0
end
else
q_error_warning[0, 0] = "#:ERROR:"
end
end
if q_error_warning
qe = ! q_error_warning.index(/(?:warning)/i)
if is_loop && qe
@loop_nanos = @loop_nanos_string = nil
end
return q_error_warning if qe || (! nanos)
end

if is_loop
@function = LOOP_ID
else
ii = @line.index(/(?:[^\/\\,;:# ])/)
iii = @line.index(/(?: |: )/, ii)
(@function = @line[ii, iii - ii])
@function.sub!(/(?:[,;:# ]+\z)/, "")
@function.freeze
end
else
return nil
end

get_type(index_type_after)

#:TODO: what if nanos and this is a function data item?
return nil unless @type && @function

unless @name_to_id_hash[@type]
@max_id = (@max_id || -1) + 1
@name_to_id_hash[@type] = @type_id = @max_id
@id_to_name_hash[@max_id] = @type
end

unless @name_to_id_hash[@function]
@max_id = (@max_id || -1) + 1
@name_to_id_hash[@function] = @function_id = @max_id
@id_to_name_hash[@max_id] = @function
end

keyv = Array.new
keyv[INDEX_TYPE] = @type
keyv[INDEX_FUNCTION] = @function
keyv[INDEX_OPTIMIZE] = @optimize
unless vvv = @stats_hash[keyv]
@stats_hash[keyv] = vvv = Array.new
end
vvv[INDEX_TYPE] = @type
vvv[INDEX_FUNCTION] = @function
vvv[INDEX_OPTIMIZE] = @optimize
ii = nanos_plus_minus == 1 ? INDEX_NANOS_PLUS : INDEX_NANOS
iii = ii + INDEX_COUNT
vvv[iii] = (vvv[iii] || 0) + 1
iii = ii + INDEX_AVERAGE
vvv[iii] = (vvv[iii] || 0) + nanos
iii = ii + INDEX_VARIANCE
vvv[iii] = (vvv[iii] || 0) + nanos * nanos
iii = ii + INDEX_MIN
vvv[iii] = nanos if (! (v = vvv[iii])) || v > nanos
iii = ii + INDEX_MAX
vvv[iii] = nanos if (! (v = vvv[iii])) || v < nanos

return q_error_warning
end

end

}

(8-8/8)