>>From 911be062bb722a860ca760010907e5a29f7be38f Mon Sep 17 00:00:00 2001 From: Frank Dittrich Date: Wed, 23 Oct 2013 23:04:48 +0200 Subject: [PATCH 3/3] minor relbench improvements and adjustments Suggest runnig benchmark-unify only if there really is a chance that this might increase the number for benchmarks that can be compared (e.g., not if the new john version just has "Many salts" and "Only one salt" for a particular format instead of just "Raw" in the old version). Suppress some of the "Could not parse" errors which had been caused by minor changes in john's --test output. Sort the list of formats that exist only in one file (and, for -v, the ratio). Depending on the definition of a dynamic format, the --test output might contain nested [ [ ... ] ], e.g., for dynamic_1400 and dynamic_1401: Expression=sha1(unicode($p)) [Microsoft CREDHIST] Expression=md5($u.\nskyper\n.$p) [Skype MD5] So, parsing the "Benchmarking: " output had to be adjusted. --- run/relbench | 101 ++++++++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 89 insertions(+), 12 deletions(-) diff --git a/run/relbench b/run/relbench index ecd5d7d..02b48ac 100755 --- a/run/relbench +++ b/run/relbench @@ -1,8 +1,9 @@ #!/usr/bin/perl -w # -# John the Ripper benchmark output comparison tool, revision 4.1 +# John the Ripper benchmark output comparison tool, revision 4.2 # Copyright (c) 2011 Solar Designer -# Enhancements for revision 4.1, copyright (c) 2012 Frank Dittrich +# Enhancements for revisions 4.1 and 4.2, +# copyright (c) 2012-2013 Frank Dittrich # Redistribution and use in source and binary forms, with or without # modification, are permitted. # There's ABSOLUTELY NO WARRANTY, express or implied. @@ -28,10 +29,6 @@ # values will tend to deviate from 0.0 and 1.0, respectively. # -$warned = 0; -$onlyin1 = 0; -$onlyin2 = 0; - sub parse { chomp; @@ -43,7 +40,16 @@ sub parse undef $kind; undef $real; undef $virtual; return; } - return if (/All \d+ tests have passed self-tests/); + if (/^(All|\d+ out of) \d+ (tests|formats)( have)? (passed self-tests|FAILED)/) { + ($total) = /^All (\d+) (tests|formats)( have)? passed self-tests/; + if (!defined($total)) { + ($failed, $total) = + /^(\d+) out of (\d+) (tests|formats)( have)? FAILED/; + } + return; + } + return if /Benchmarking: .*\.\.\. FAILED/; + my $ok = 0; if (defined($name)) { ($kind, $real, $reals, $virtual, $virtuals) = @@ -65,12 +71,34 @@ sub parse return; } } else { - ($name) = /^\t?Benchmarking: (.+) \[.*\].* (DONE|Warning:.*)$/; + ($name) = /^\t?Benchmarking: ([^\[]+) \[.*\].* (DONE|Warning:.*)$/; $ok = defined($name); } print STDERR "Could not parse: $_\n" if (!$ok); } +sub sort_benchmarks +{ + my $_; +# $a cmp $b; + $_ = "$a"; + ($name_a, $number_a, $benchmark_a) = /^(.*[^\d])(\d+):(.*)$/; + if(!defined($number_a)) { + $number_a = -1; + ($name_a, $benchmark_a) = /^(.*):(.*)$/; + } + $_ = "$b"; + ($name_b, $number_b, $benchmark_b) = /^(.*[^\d])(\d+):(.*)$/; + if(!defined($number_b)) { + $number_b = -1; + ($name_b, $benchmark_b) = /^(.*):(.*)$/; + } + if ($name_a ne $name_b) { return $name_a cmp $name_b } + elsif ($number_a != $number_b) { return $number_a <=> $number_b } + elsif ($benchmark_b eq "Short") { return 1 } + else { return $benchmark_a cmp $benchmark_b } +} + die "Usage: $0 [-v] BENCHMARK-FILE-1 BENCHMARK-FILE-2\n" if ($#ARGV != 1 && ($#ARGV != 2 || $ARGV[0] ne '-v')); if ($#ARGV != 1) { @@ -83,10 +111,29 @@ if ($#ARGV != 1) { $verbose = 0; } +$warned = 0; +$onlyin1 = 0; +$onlyin2 = 0; +$name1 = ""; +$name2 = ""; +$namesonlyin1 = 0; +$namesonlyin2 = 0; + +$failed = 0; +$total = 0; $_ = ''; parse(); while () { parse(); + if (defined($name) && $name ne $name1) { + $name1 = $name; + if(defined($n1{$name1})) { + $n1{$name1} += 1; + } + else { + $n1{$name1} = 1; + } + } next unless (defined($id)); if(defined($b1r{$id})) { print STDERR "More than one benchmark for $id in file 1\n"; @@ -99,11 +146,25 @@ while () { $b1v{$id} = $virtual; } } close(B1); +if($failed != 0) { + print "File 1: $failed out of $total tests have FAILED\n" +} +$failed = 0; +$total = 0; $_ = ''; parse(); while () { parse(); + if (defined($name) && $name ne $name2) { + $name2 = $name; + if(defined($n2{$name2})) { + $n2{$name2} += 1; + } + else { + $n2{$name2} = 1; + } + } next unless (defined($id)); if(defined($b2r{$id})) { print STDERR "More than one benchmark for $id in file 2\n"; @@ -117,8 +178,24 @@ while () { } } close(B2); +if($failed != 0) { + print "File 2: $failed out of $total tests have FAILED\n" +} + +foreach $name (keys %n1) { + if (!defined($n2{$name})) { + $namesonlyin1 += 1; + next; + } +} +foreach $name (keys %n2) { + if (!defined($n1{$name})) { + $namesonlyin1 += 1; + next; + } +} -foreach $id (keys %b1r) { +foreach $id (sort sort_benchmarks keys %b1r) { if (!defined($b2r{$id})) { print "Only in file 1: $id\n"; $onlyin1 += 1; @@ -129,14 +206,14 @@ foreach $id (keys %b1r) { $minr = $maxr = $minv = $maxv = -1.0; $mr = $mv = 1.0; $n = 0; -foreach $id (keys %b2r) { +foreach $id (sort sort_benchmarks keys %b2r) { if (!defined($b1r{$id})) { print "Only in file 2: $id\n"; $onlyin2 += 1; next; } } -foreach $id (keys %b2r) { +foreach $id (sort sort_benchmarks keys %b2r) { if (!defined($b1r{$id})) { next; } @@ -155,7 +232,7 @@ foreach $id (keys %b2r) { printf "Ratio:\t%.5f real, %.5f virtual\t$id\n", $kr, $kv; } } -if ($onlyin1 != 0 && $onlyin2 != 0) { +if ($onlyin1 != 0 && $onlyin2 != 0 && $namesonlyin1 != 0 && $namesonlyin2 != 0) { print STDERR "Converting the two benchmark files using benchmark-unify might\n"; print STDERR "increase the number of benchmarks which can be compared\n"; } -- 1.8.1.4