bmt-0.6/0000755000175000017500000000000012530560723011732 5ustar michaelmichaelbmt-0.6/parse-astree.pl0000644000175000017500000000463212530560050014660 0ustar michaelmichael#!/usr/bin/perl -w # # Copyright (c) 2011 Michael Tautschnig # 2015 Michael Tautschnig # # All rights reserved. Redistribution and use in source and binary forms, with # or without modification, are permitted provided that the following # conditions are met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # 3. All advertising materials mentioning features or use of this software # must display the following acknowledgement: # # This product includes software developed by Michael Tautschnig, # Queen Mary University of London # # 4. Neither the name of the University nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # # # THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS `AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE # DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # parse the Astree specific part of a benchmark log file use strict; use warnings FATAL => qw(uninitialized); sub parse_log { my ($LOG, $hash) = @_; $hash->{Result} = "ERROR"; while (<$LOG>) { chomp; return 1 if (/^###############################################################################$/); if (/WARN \(A\): assert failure\s*$/) { $hash->{Result} = "FAILED"; } elsif (/^The analysis took / && $hash->{Result} ne "FAILED") { $hash->{Result} = "SUCCESS"; } } } return 1; bmt-0.6/parse-threader.pl0000644000175000017500000000461512530560050015174 0ustar michaelmichael#!/usr/bin/perl -w # # Copyright (c) 2011 Michael Tautschnig # 2015 Michael Tautschnig # # All rights reserved. Redistribution and use in source and binary forms, with # or without modification, are permitted provided that the following # conditions are met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # 3. All advertising materials mentioning features or use of this software # must display the following acknowledgement: # # This product includes software developed by Michael Tautschnig, # Queen Mary University of London # # 4. Neither the name of the University nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # # # THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS `AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE # DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # parse the threader/cream tool specific part of a benchmark log file use strict; use warnings FATAL => qw(uninitialized); sub parse_log { my ($LOG, $hash) = @_; $hash->{Result} = "ERROR"; while (<$LOG>) { chomp; return 1 if (/^###############################################################################$/); if (/^Feasible counterexample\.$/) { $hash->{Result} = "FAILED"; } elsif (/^Program is correct\.$/) { $hash->{Result} = "SUCCESSFUL"; } } } return 1; bmt-0.6/parse-bfc.pl0000644000175000017500000002053112530560050014123 0ustar michaelmichael#!/usr/bin/perl -w # # Copyright (c) 2011 Michael Tautschnig # 2015 Michael Tautschnig # # All rights reserved. Redistribution and use in source and binary forms, with # or without modification, are permitted provided that the following # conditions are met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # 3. All advertising materials mentioning features or use of this software # must display the following acknowledgement: # # This product includes software developed by Michael Tautschnig, # Queen Mary University of London # # 4. Neither the name of the University nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # # # THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS `AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE # DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # parse the CPROVER tool specific part of a benchmark log file use strict; use warnings FATAL => qw(uninitialized); sub parse_log { my ($LOG, $hash) = @_; $hash->{Result} = "ERROR"; $hash->{"bw finished first"} = "UNSET"; $hash->{"iterations"} = "UNSET"; $hash->{"work iterations"} = "UNSET"; $hash->{"prune iterations"} = "UNSET"; $hash->{"foward prune cycles"} = "UNSET"; $hash->{"backward prune cycles"} = "UNSET"; $hash->{"oracle dequeues"} = "UNSET"; $hash->{"pruned"} = "UNSET"; $hash->{"known"} = "UNSET"; $hash->{"new predecessors"} = "UNSET"; $hash->{"existing predecessors"} = "UNSET"; $hash->{"locally_minimal"} = "UNSET"; $hash->{"locally_non_minimal"} = "UNSET"; $hash->{"globally_minimal"} = "UNSET"; $hash->{"neq_nge_nle"} = "UNSET"; $hash->{"globally_non_minimal"} = "UNSET"; $hash->{"lowerset intersections"} = "UNSET"; $hash->{"initial_intersect"} = "UNSET"; $hash->{"root_intersect"} = "UNSET"; $hash->{"lowerset_intersect"} = "UNSET"; $hash->{"locally_pruned"} = "UNSET"; $hash->{"max. backward depth checked"} = "UNSET"; $hash->{"max. backward width checked"} = "UNSET"; $hash->{"fw finished first"} = "UNSET"; $hash->{"time to find projections"} = "UNSET"; $hash->{"forward-reachable global states"} = "UNSET"; $hash->{"projections found"} = "UNSET"; $hash->{"accelerations"} = "UNSET"; $hash->{"max acceleration depth checked"} = "UNSET"; $hash->{"max acceleration depth"} = "UNSET"; $hash->{"max. forward depth checked"} = "UNSET"; $hash->{"max. forward width checked"} = "UNSET"; $hash->{"osz_max"} = "UNSET"; $hash->{"nsz_max"} = "UNSET"; $hash->{"msz_max"} = "UNSET"; $hash->{"dsz_max"} = "UNSET"; $hash->{"wsz_max"} = "UNSET"; $hash->{"local states"} = "UNSET"; $hash->{"shared states"} = "UNSET"; $hash->{"transitions"} = "UNSET"; $hash->{"thread transitions"} = "UNSET"; $hash->{"broadcast transitions"} = "UNSET"; $hash->{"spawn transitions"} = "UNSET"; $hash->{"dimension of target state"} = "UNSET"; while (<$LOG>) { chomp; return 1 if (/^###############################################################################$/); if (/^VERIFICATION\s+(\S+)$/) { $hash->{Result} = "$1"; } elsif (/^bw finished first : (\S+)$/) { $hash->{"bw finished first"} = $1; } elsif (/^iterations : (\d+)$/) { $hash->{"iterations"} = $1; } elsif (/^- work iterations : (\d+)$/) { $hash->{"work iterations"} = $1; } elsif (/^- prune iterations : (\d+)$/) { $hash->{"prune iterations"} = $1; } elsif (/^- foward prune cycles : (\d+)$/) { $hash->{"foward prune cycles"} = $1; } elsif (/^- backward prune cycles : (\d+)$/) { $hash->{"backward prune cycles"} = $1; } elsif (/^oracle dequeues : (\d+)$/) { $hash->{"oracle dequeues"} = $1; } elsif (/^- pruned : (\d+)$/) { $hash->{"pruned"} = $1; } elsif (/^- known : (\d+)$/) { $hash->{"known"} = $1; } elsif (/^new predecessors : (\d+)$/) { $hash->{"new predecessors"} = $1; } elsif (/^existing predecessors : (\d+)$/) { $hash->{"existing predecessors"} = $1; } elsif (/^locally_minimal : (\d+)$/) { $hash->{"locally_minimal"} = $1; } elsif (/^locally_non_minimal : (\d+)$/) { $hash->{"locally_non_minimal"} = $1; } elsif (/^globally_minimal : (\d+)$/) { $hash->{"globally_minimal"} = $1; } elsif (/^- neq_nge_nle : (\d+)$/) { $hash->{"neq_nge_nle"} = $1; } elsif (/^globally_non_minimal : (\d+)$/) { $hash->{"globally_non_minimal"} = $1; } elsif (/^lowerset intersections : (\d+)$/) { $hash->{"lowerset intersections"} = $1; } elsif (/^- initial_intersect : (\d+)$/) { $hash->{"initial_intersect"} = $1; } elsif (/^- root_intersect : (\d+)$/) { $hash->{"root_intersect"} = $1; } elsif (/^- lowerset_intersect : (\d+)$/) { $hash->{"lowerset_intersect"} = $1; } elsif (/^locally_pruned : (\d+)$/) { $hash->{"locally_pruned"} = $1; } elsif (/^max. backward depth checked : (\d+)$/) { $hash->{"max. backward depth checked"} = $1; } elsif (/^max. backward width checked : (\d+)$/) { $hash->{"max. backward width checked"} = $1; } elsif (/^fw finished first : (\S+)$/) { $hash->{"fw finished first"} = $1; } elsif (/^time to find projections : (\S+)$/) { $hash->{"time to find projections"} = $1; } elsif (/^forward-reachable global states : (\d+)$/) { $hash->{"forward-reachable global states"} = $1; } elsif (/^projections found : (\d+)$/) { $hash->{"projections found"} = $1; } elsif (/^accelerations : (\d+)$/) { $hash->{"accelerations"} = $1; } elsif (/^max acceleration depth checked : (\d+)$/) { $hash->{"max acceleration depth checked"} = $1; } elsif (/^max acceleration depth : (\d+)$/) { $hash->{"max acceleration depth"} = $1; } elsif (/^max. forward depth checked : (\d+)$/) { $hash->{"max. forward depth checked"} = $1; } elsif (/^max. forward width checked : (\d+)$/) { $hash->{"max. forward width checked"} = $1; } elsif (/^osz_max : (\d+)$/) { $hash->{"osz_max"} = $1; } elsif (/^nsz_max : (\d+)$/) { $hash->{"nsz_max"} = $1; } elsif (/^msz_max : (\d+)$/) { $hash->{"msz_max"} = $1; } elsif (/^dsz_max : (\d+)$/) { $hash->{"dsz_max"} = $1; } elsif (/^wsz_max : (\d+)$/) { $hash->{"wsz_max"} = $1; } elsif (/^local states : (\d+)$/) { $hash->{"local states"} = $1; } elsif (/^shared states : (\d+)$/) { $hash->{"shared states"} = $1; } elsif (/^transitions : (\d+)$/) { $hash->{"transitions"} = $1; } elsif (/^thread transitions : (\d+)$/) { $hash->{"thread transitions"} = $1; } elsif (/^broadcast transitions : (\d+)$/) { $hash->{"broadcast transitions"} = $1; } elsif (/^spawn transitions : (\d+)$/) { $hash->{"spawn transitions"} = $1; } elsif (/^dimension of target state : (\d+)$/) { $hash->{"dimension of target state"} = $1; } } } return 1; bmt-0.6/parse-ai.pl0000644000175000017500000000465412530560050013772 0ustar michaelmichael#!/usr/bin/perl -w # # Copyright (c) 2011 Michael Tautschnig # 2015 Michael Tautschnig # # All rights reserved. Redistribution and use in source and binary forms, with # or without modification, are permitted provided that the following # conditions are met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # 3. All advertising materials mentioning features or use of this software # must display the following acknowledgement: # # This product includes software developed by Michael Tautschnig, # Queen Mary University of London # # 4. Neither the name of the University nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # # # THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS `AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE # DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # parse the CPROVER tool specific part of a benchmark log file use strict; use warnings FATAL => qw(uninitialized); sub parse_log { my ($LOG, $hash) = @_; $hash->{Result} = "ERROR"; $hash->{Iterations} = 0; while (<$LOG>) { chomp; return 1 if (/^###############################################################################$/); if (/^\*\*\*\* Verification\s+(\S+)/) { $hash->{Result} = "$1"; } elsif (/^ITERATION \((backtrack|decision)\) (\d+)$/) { $hash->{Iterations} = $2; } } } return 1; bmt-0.6/parse-scratch.pl0000644000175000017500000000461712530560050015027 0ustar michaelmichael#!/usr/bin/perl -w # # Copyright (c) 2011 Michael Tautschnig # 2015 Michael Tautschnig # # All rights reserved. Redistribution and use in source and binary forms, with # or without modification, are permitted provided that the following # conditions are met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # 3. All advertising materials mentioning features or use of this software # must display the following acknowledgement: # # This product includes software developed by Michael Tautschnig, # Queen Mary University of London # # 4. Neither the name of the University nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # # # THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS `AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE # DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # parse the CPROVER tool specific part of a benchmark log file use strict; use warnings FATAL => qw(uninitialized); sub parse_log { my ($LOG, $hash) = @_; $hash->{Result} = "ERROR"; while (<$LOG>) { chomp; return 1 if (/^###############################################################################$/); if (/^Too many iterations, giving up/) { $hash->{Result} = "TOO_MANY_ITERATIONS"; } elsif (/^VERIFICATION\s+(\S+)$/) { $hash->{Result} = "$1"; } } } return 1; bmt-0.6/parse-blender.pl0000644000175000017500000000456612530560050015016 0ustar michaelmichael#!/usr/bin/perl -w # # Copyright (c) 2011 Michael Tautschnig # 2015 Michael Tautschnig # # All rights reserved. Redistribution and use in source and binary forms, with # or without modification, are permitted provided that the following # conditions are met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # 3. All advertising materials mentioning features or use of this software # must display the following acknowledgement: # # This product includes software developed by Michael Tautschnig, # Queen Mary University of London # # 4. Neither the name of the University nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # # # THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS `AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE # DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # parse the Astree specific part of a benchmark log file use strict; use warnings FATAL => qw(uninitialized); sub parse_log { my ($LOG, $hash) = @_; $hash->{Result} = "ERROR"; while (<$LOG>) { chomp; return 1 if (/^###############################################################################$/); if (/^Avoid formula:\s+True\s*$/) { $hash->{Result} = "SUCCESSFUL"; } elsif (/^Avoid formula:/) { $hash->{Result} = "FAILED"; } } } return 1; bmt-0.6/parse-llbmc.pl0000644000175000017500000000501612530560050014463 0ustar michaelmichael#!/usr/bin/perl -w # # Copyright (c) 2011 Michael Tautschnig # 2015 Michael Tautschnig # # All rights reserved. Redistribution and use in source and binary forms, with # or without modification, are permitted provided that the following # conditions are met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # 3. All advertising materials mentioning features or use of this software # must display the following acknowledgement: # # This product includes software developed by Michael Tautschnig, # Queen Mary University of London # # 4. Neither the name of the University nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # # # THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS `AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE # DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # parse the CPROVER tool specific part of a benchmark log file use strict; use warnings FATAL => qw(uninitialized); sub parse_log { my ($LOG, $hash) = @_; $hash->{Result} = "ERROR"; $hash->{"SMT solver time"} = 0; while (<$LOG>) { chomp; return 1 if (/^###############################################################################$/); if (/^No error detected\.$/) { $hash->{Result} = "SUCCESSFUL"; } elsif (/^Error detected\.$/) { $hash->{Result} = "FAILED"; } elsif (/^Running SMT solver\.\.\. done\.\s+\[(\d+\.\d+) sec\]$/) { $hash->{"SMT solver time"} = $1; } } } return 1; bmt-0.6/parse-wolverine.pl0000644000175000017500000000461512530560050015410 0ustar michaelmichael#!/usr/bin/perl -w # # Copyright (c) 2011 Michael Tautschnig # 2015 Michael Tautschnig # # All rights reserved. Redistribution and use in source and binary forms, with # or without modification, are permitted provided that the following # conditions are met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # 3. All advertising materials mentioning features or use of this software # must display the following acknowledgement: # # This product includes software developed by Michael Tautschnig, # Queen Mary University of London # # 4. Neither the name of the University nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # # # THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS `AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE # DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # parse the CPROVER tool specific part of a benchmark log file use strict; use warnings FATAL => qw(uninitialized); sub parse_log { my ($LOG, $hash) = @_; $hash->{Result} = "ERROR"; while (<$LOG>) { chomp; return 1 if (/^###############################################################################$/); if (/^Too many iterations, giving up/) { $hash->{Result} = "TOO_MANY_ITERATIONS"; } elsif (/^VERIFICATION\s+(\S+)$/) { $hash->{Result} = "$1"; } } } return 1; bmt-0.6/parse-blast.pl0000644000175000017500000000463312530560050014503 0ustar michaelmichael#!/usr/bin/perl -w # # Copyright (c) 2011 Michael Tautschnig # 2015 Michael Tautschnig # # All rights reserved. Redistribution and use in source and binary forms, with # or without modification, are permitted provided that the following # conditions are met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # 3. All advertising materials mentioning features or use of this software # must display the following acknowledgement: # # This product includes software developed by Michael Tautschnig, # Queen Mary University of London # # 4. Neither the name of the University nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # # # THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS `AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE # DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # parse the BLAST specific part of a benchmark log file use strict; use warnings FATAL => qw(uninitialized); sub parse_log { my ($LOG, $hash) = @_; $hash->{Result} = "ERROR"; while (<$LOG>) { chomp; return 1 if (/^###############################################################################$/); if (/^No error found. The system is safe :-\)$/) { $hash->{Result} = "SUCCESS"; } elsif (/^Error found! The system is unsafe :-\($/) { $hash->{Result} = "FAILED"; } } } return 1; bmt-0.6/LICENSE0000644000175000017500000000345712530560050012741 0ustar michaelmichaelCopyright (c) 2011 Michael Tautschnig 2015 Michael Tautschnig All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. All advertising materials mentioning features or use of this software must display the following acknowledgement: This product includes software developed by Michael Tautschnig, Queen Mary University of London 4. Neither the name of the University nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS `AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. bmt-0.6/make_web.pl0000755000175000017500000001476312530560050014050 0ustar michaelmichael#!/usr/bin/perl -w # # Copyright (c) 2011 Michael Tautschnig # 2015 Michael Tautschnig # # All rights reserved. Redistribution and use in source and binary forms, with # or without modification, are permitted provided that the following # conditions are met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # 3. All advertising materials mentioning features or use of this software # must display the following acknowledgement: # # This product includes software developed by Michael Tautschnig, # Queen Mary University of London # # 4. Neither the name of the University nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # # # THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS `AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE # DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # build an HTML table and a directory collecting log files use strict; use warnings FATAL => qw(uninitialized); use Getopt::Std; sub usage { print <<"EOF"; Usage: $0 [OPTIONS] CSV TARGETDIR where CSV is a comma-separated data file as built by make_csv.pl and TARGETDIR is the directory, which must not yet exist, to store the logs and the index.html file in Options: Purpose: -h show help -z bzip2 log files larger than 1MB -n omit date EOF } our ($opt_z, $opt_h, $opt_n); if (!getopts('hzn') || defined($opt_h) || scalar(@ARGV) != 2) { usage; exit (defined($opt_h) ? 0 : 1); } my $file = $ARGV[0]; open my $CSV, "<$file" or die "File $file not found\n"; my $destdir = $ARGV[1]; mkdir $destdir or die "Failed to create directory $destdir\n"; use File::Copy; my %aliases = (); my %globals = (); use Text::CSV; my $csv = Text::CSV->new(); my $arref = $csv->getline($CSV); defined($arref) or die "Failed to parse headers\n"; $csv->column_names(@$arref); my @skip = qw(command version timeout uname cpuinfo meminfo memlimit); push @skip, "Log File"; push @skip, "Benchmark"; push @skip, "Result"; $opt_n and push @skip, "date"; my %rm; @rm{@skip} = (); my @headers = grep { not exists $rm{$_} } @$arref; my %rows = (); while (my $row = $csv->getline_hr($CSV)) { foreach (qw(command version timeout uname cpuinfo meminfo memlimit)) { defined($row->{$_}) or die "No $_ data in table\n"; defined($globals{$_}) or $globals{$_} = (); $globals{$_}{$row->{$_}} = 1; } my $logfile = $row->{"Log File"}; defined($logfile) or die "No Log File column in table\n"; defined($aliases{$logfile}) and die "Duplicate log file $logfile\n"; $aliases{$logfile} = "$logfile.txt"; $aliases{$logfile} =~ s/^.*\/results\.//; $aliases{$logfile} =~ s/\//__/g; if($opt_n) { `cat $logfile | perl -e 'while(<>) { \ if(/^### date:/) { \$skip=1; next; } if(\$skip) { \$skip=0; next; } \ print; }' > $destdir/$aliases{$logfile}`; } else { copy($logfile, "$destdir/$aliases{$logfile}") or die "Failed to copy log file from $logfile to $destdir/$aliases{$logfile}\n"; } if($opt_z && -s $logfile > 1024*1024) { warn "$logfile has size ".(-s $logfile).", running bzip2\n"; system("bzip2 $destdir/$aliases{$logfile}"); $aliases{$logfile} .= ".bz2"; } my @data = (); defined($row->{Benchmark}) or die "No Benchmark column in table\n"; push @data, "$row->{Benchmark}"; defined($row->{Result}) or die "No Result column in table\n"; my $res = $row->{Result}; $res = "Property VIOLATED" if($row->{Result} eq "FAILED"); $res = "Property HOLDS" if($row->{Result} eq "SUCCESSFUL"); push @data, $res; push @data, defined($row->{$_}) ? $row->{$_} : "n/a" foreach (@headers); $rows{$logfile} = (); push @{ $rows{$logfile} }, @data; } close $CSV; unshift @headers, "Result"; unshift @headers, "Benchmark"; my $ncols = scalar(@headers); my $command = join(';', keys %{ $globals{command} }); my $version = join(';', keys %{ $globals{version} }); my $timeout = join(';', keys %{ $globals{timeout} }); my $memlimit = join(';', keys %{ $globals{memlimit} }); my $currtime = $opt_n ? "" : localtime(); open INDEX_HTML, ">$destdir/index.html" or die "Failed to create $destdir/index.html\n"; print INDEX_HTML <<"EOF"; Experimental Results EOF print INDEX_HTML " \n" foreach (@headers); print INDEX_HTML " \n"; foreach my $r (sort keys %rows) { print INDEX_HTML " \n"; print INDEX_HTML " \n" foreach (@{ $rows{$r} }); print INDEX_HTML " \n"; } my $sysdesc = "The benchmarks were run on a " . join(';', keys %{ $globals{uname} }) . ' ' . join(';', keys %{ $globals{cpuinfo} }) . " system equipped with " . join(';', keys %{ $globals{meminfo} }) . " RAM.\n"; print INDEX_HTML <<"EOF";
* $command version $version results *
Timeout: $timeout Memory limit: $memlimit
$currtime
$_
$_
$sysdesc EOF bmt-0.6/make_graph.pl0000755000175000017500000003401412530560050014363 0ustar michaelmichael#!/usr/bin/perl -w # # Copyright (c) 2011 Michael Tautschnig # 2015 Michael Tautschnig # # All rights reserved. Redistribution and use in source and binary forms, with # or without modification, are permitted provided that the following # conditions are met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # 3. All advertising materials mentioning features or use of this software # must display the following acknowledgement: # # This product includes software developed by Michael Tautschnig, # Queen Mary University of London # # 4. Neither the name of the University nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # # # THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS `AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE # DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # build scripts for making gnuplot figures use strict; use warnings FATAL => qw(uninitialized); use Getopt::Std; sub usage { print <<"EOF"; Usage: $0 [OPTIONS] CSV [COLUMN FILTER] [CSV [COLUMN FILTER]] ... where CSV is a CSV table as produced by make_csv.pl, COLUMN a column name in there, and FILTER a regular expression Options: Purpose: -h show help -s build a scatter plot (requires exactly two CSV files) -c build a cactus plot -l use logarithmic scaling -r use percentage of instances as x axis scaling in cactus plot -m REGEXP:REGEXP map benchmark names from REGEXP to REGEXP -X beautify graph EOF } our ($opt_s, $opt_h, $opt_c, $opt_l, $opt_r, $opt_m, $opt_X); if (!getopts('hsclrm:X') || defined($opt_h) || !scalar(@ARGV)) { usage; exit (defined($opt_h) ? 0 : 1); } use Text::CSV; my %per_file_data = (); my %results = (); my %files_filters = (); while (scalar(@ARGV)) { my $f = shift @ARGV; (-f $f) or die "File $f not found\n"; defined($files_filters{$f}) and die "Duplicate file name $f\n"; $files_filters{$f} = { column => "command", filter => ".*" }; if(scalar(@ARGV) && !(-f $ARGV[0] && $ARGV[0] =~ /\.csv$/)) { (scalar(@ARGV) >= 2) or die "column AND filter required\n"; $files_filters{$f}{column} = shift @ARGV; $files_filters{$f}{filter} = shift @ARGV; } } my $map_from = ""; my $map_to = ""; if($opt_m) { my @map = split(':', $opt_m); (scalar(@map) == 2) or die "Cannot split $opt_m on :\n"; $map_from = shift @map; $map_to = shift @map; } foreach my $f (keys %files_filters) { open my $CSV, "<$f" or die "File $f not found\n"; my %globals = (); my $dbg_lno = 1; my $csv = Text::CSV->new(); my $arref = $csv->getline($CSV); defined($arref) or die "Failed to parse headers\n"; $csv->column_names(@$arref); while (my $row = $csv->getline_hr($CSV)) { ++$dbg_lno; foreach (qw(command timeout uname cpuinfo meminfo memlimit)) { my $val=""; if(!defined($row->{$_})) { if($_ eq "command") { die "No $_ data in $f, line $dbg_lno\n"; } else { warn "No $_ data in $f, line $dbg_lno\n"; } } else { $val=$row->{$_}; } if ($_ eq "command" && defined($globals{$_}) && !defined($globals{$_}{$val})) { my $cmd = ""; for (keys %{ $globals{command} }) { ($cmd eq "") or die "Only single preceding command expected\n"; $cmd = $_; delete $globals{command}{$_}; } my $newcmd = $row->{command}; $cmd =~ s/^(\S+).*$/$1/; $newcmd =~ s/^(\S+).*$/$1/; ($cmd eq $newcmd) or die "Multiple base commands not supported\n"; $globals{command}{"$cmd (multiple options)"} = 1; } else { defined($globals{$_}) or $globals{$_} = (); $globals{$_}{$val} = 1; } } (defined($row->{$_}) && ! ($row->{$_} =~ /^\s*$/)) or die "No $_ data in $f, line $dbg_lno\n" foreach (qw(Benchmark usertime)); (defined($row->{$_}) && ! ($row->{$_} =~ /^\s*$/)) or warn "No $_ data in $f, line $dbg_lno\n" foreach (qw(Result maxmem)); my $col = $files_filters{$f}{column}; my $flt = $files_filters{$f}{filter}; defined($row->{$col}) or die "No column $col in $f\n"; next unless($row->{$col} =~ /$flt/); my $bm = $row->{Benchmark}; $bm =~ s/$map_from/$map_to/ if($opt_m); if (!defined($results{$bm})) { $results{$bm} = (); $results{$bm}{id} = scalar(keys %results); $results{$bm}{status} = $row->{Result}; } elsif ($results{$bm}{status} ne $row->{Result}) { ($results{$bm}{status} =~ /^(ERROR|TIMEOUT)$/ || $row->{Result} =~ /^(ERROR|TIMEOUT)$/) or warn "Tools disagree about verification result for $bm\n"; } defined($results{$bm}{$f}) and die "Results for $bm and $f already known\n"; $results{$bm}{$f} = (); $results{$bm}{$f}{result} = $row->{Result}; my $cpu = $row->{usertime}; $results{$bm}{$f}{cpu} = sprintf "%.2f", $cpu; my $mem = $row->{maxmem}; ($mem =~ /^(\d+)kb$/) or die "Unexpected memory usage format in $mem\n"; $mem = $1 / 1024.0; $results{$bm}{$f}{mem} = sprintf "%.2f", $mem; } close $CSV; $per_file_data{$f} = (); $per_file_data{$f}{name} = $f; $per_file_data{$f}{name} =~ s/\.csv//; $per_file_data{$f}{name} =~ s/.*\///; foreach (qw(command timeout memlimit)) { (scalar(keys %{ $globals{$_} }) == 1) or die "Multiple configurations per file not supported ($_)\n"; $per_file_data{$f}{$_} = (keys %{ $globals{$_} })[0]; } foreach (qw(uname cpuinfo meminfo)) { (scalar(keys %{ $globals{$_} }) == 1) or warn "Multiple configurations for $_ found in $f:\n" . join(" | ", keys %{ $globals{$_} }) . "\n"; } } system("wget --no-check-certificate https://github.com/shreyankg/xkcd-desktop/raw/master/Humor-Sans.ttf") if($opt_X && ! -f "Humor-Sans.ttf"); my $pgfbasedl = 0; if(! -d "base") { $pgfbasedl = 1; system("wget http://mirror.ctan.org/graphics/pgf/base.zip"); system("unzip base.zip"); } my $pgfplotsdl = 0; if(! -d "pgfplots") { $pgfplotsdl = 1; system("wget http://mirror.ctan.org/graphics/pgf/contrib/pgfplots.zip"); system("unzip pgfplots.zip"); } my $fn = ""; my $title = scalar(keys %per_file_data) > 1 ? "Comparison of " : ""; my $legend = ""; foreach (sort keys %per_file_data) { $fn .= $per_file_data{$_}{name} . "_"; $title .= $per_file_data{$_}{name} . ", "; $legend .= $per_file_data{$_}{name} . " (TO " . $per_file_data{$_}{timeout} . "), "; } $fn =~ s/_$//; $opt_s and $fn .= "-scatter"; $opt_c and $fn .= "-cactus"; $opt_l and $fn .= "_log"; $title =~ s/, $//; $title =~ s/, ([^,]+)$/ and $1/; $legend =~ s/, $//; my $decoration = ""; $opt_X and $decoration = "\n decoration={random steps,segment length=1mm,amplitude=0.2pt}"; my $axis_decoration = ""; $opt_X and $axis_decoration = "\n xticklabel style={/pgf/number format/assume math mode},\n". " yticklabel style={/pgf/number format/assume math mode}"; my $font = ""; $opt_X and $font = "\\usepackage{fontspec}\n\\setmainfont[ExternalLocation]{Humor-Sans.ttf}\n"; my $lb = $opt_l ? 0.01 : 0.0; my $width = 4; my $xenlarge = 0; if(!$opt_s && !$opt_c) { my $nfiles = scalar(keys %per_file_data); ($nfiles < 2) and $nfiles = 2; $width = scalar(keys %results) * $nfiles * 0.3; } else { $width = scalar(keys %results) * 0.05; } $width = 4 if($width < 4); $width = 75 if($width > 75); $xenlarge = 1/$width; $width = "\\pgfplotsset{width=${width}cm}\n"; open TEX, ">$fn.tex"; print TEX << "EOF"; \\documentclass{article} $font \\usepackage{pgfplots} \\usetikzlibrary{backgrounds} \\tikzset{ extra padding/.style={ show background rectangle, inner frame sep=#1, background rectangle/.style={ draw=none } }, extra padding/.default=0.5cm,$decoration } $width \\pgfrealjobname{$fn-nn} \\begin{document} \\beginpgfgraphicnamed{$fn} \\begin{tikzpicture}[extra padding] EOF if ($opt_s) { (scalar(keys %per_file_data) == 2) or die "Scatter plot requires exactly two input files\n"; my ($f1, $f2) = sort keys %per_file_data; my $f1_to = $per_file_data{$f1}{timeout}; $f1_to =~ s/s$//; my $f2_to = $per_file_data{$f2}{timeout}; $f2_to =~ s/s$//; my $axis = $opt_l ? "loglogaxis" : "axis"; my $coordstring = ""; my $rlb = $f1_to; foreach my $bm (keys %results) { my %data = (); foreach my $f (($f1, $f2)) { if (defined($results{$bm}{$f}{cpu})) { $data{$f}{cpu} = $results{$bm}{$f}{cpu}; ($opt_l && $data{$f}{cpu} < $lb) and $data{$f}{cpu} = $lb; $data{$f}{cpu} = ($f eq $f1) ? $f1_to : $f2_to if($results{$bm}{$f}{result} ne "SUCCESSFUL" && $results{$bm}{$f}{result} ne "FAILED"); } else { warn "Data for $bm/cpu not found in $f\n"; $data{$f}{cpu} = ($f eq $f1) ? $f1_to : $f2_to; } } $coordstring .= " (".$data{$f1}{cpu}.",".$data{$f2}{cpu}.")\n"; $rlb = $data{$f1}{cpu} if($data{$f1}{cpu} < $rlb); $rlb = $data{$f2}{cpu} if($data{$f2}{cpu} < $rlb); } print TEX << "EOF"; \\begin{$axis}[ title={\\Large $title}, xlabel={$per_file_data{$f1}{name}, TO $per_file_data{$f1}{timeout}}, ylabel={$per_file_data{$f2}{name}, TO $per_file_data{$f2}{timeout}}, xmin=$rlb, ymin=$rlb, xmax=$f1_to, ymax=$f2_to,$axis_decoration ] \\addplot+[only marks] coordinates { $coordstring }; \\addplot[black,domain=$rlb:$f1_to]{x}; \\end{$axis} EOF } elsif ($opt_c) { my $axis = $opt_l ? "semilogyaxis" : "axis"; my %coordstrings = (); foreach my $f (sort keys %per_file_data) { my @tlist = (); my $n = 0; foreach (keys %results) { defined($results{$_}{$f}{cpu}) or next; ++$n; my $val = $results{$_}{$f}{cpu}; ($opt_l && $val < $lb) and $val = $lb; push @tlist, $val if($results{$_}{$f}{result} eq "SUCCESSFUL" || $results{$_}{$f}{result} eq "FAILED"); } ($n > 0) or die "Zero results in $f\n"; (scalar(@tlist)) or die "No usable results for $f\n"; my $i = 1; my $mult = $opt_r ? (100.0/$n) : 1; my $max_str = scalar(@tlist); $max_str = sprintf("%.1f", $max_str * $mult) . "\\%" if($opt_r); $coordstrings{$f} .= " (" . ($i++ * $mult) . ", $_) [".($i>scalar(@tlist)?"($max_str)":" ")."]\n" foreach (sort { $a <=> $b } @tlist); } my $xbounds = ""; my $xlabel = "Solved instances"; if($opt_r) { $xbounds = "xmin=0, xmax=100,"; $xlabel .= " [\\%]"; } print TEX << "EOF"; \\begin{$axis}[ title={\\Large $title}, xlabel={$xlabel}, ylabel={Time}, axis x line*=none, axis y line=box, $xbounds legend style={at={(0.5,-2.2cm)}, anchor=north,legend columns=-1}, nodes near coords, point meta=explicit symbolic,$axis_decoration ] EOF print TEX "\\addplot+ coordinates { $coordstrings{$_} };\n" foreach (sort keys %per_file_data); print TEX << "EOF"; \\legend{ $legend }; \\end{$axis} EOF } else { my $axis = $opt_l ? "semilogyaxis" : "axis"; my $to_max = $lb; foreach my $f (sort keys %per_file_data) { my $cto = $per_file_data{$f}{timeout}; $cto =~ s/s$//; $to_max = $cto if($cto > $to_max); } my $labels = ""; my $labellen = 0; foreach (sort keys %results) { my $cl = "$_ ($results{$_}{status})"; $labellen = length($cl) if (length($cl)>$labellen); $labels .= "$cl,\n"; } $labels =~ s/\//:/g; $labels =~ s/_/-/g; my %coordstrings = (); foreach my $f (sort keys %per_file_data) { foreach (keys %results) { defined($results{$_}{$f}{cpu}) or warn "Data for $_/cpu not found in $f\n"; my $val = defined($results{$_}{$f}{cpu}) ? $results{$_}{$f}{cpu} : $to_max; ($opt_l && $val < $lb) and $val = $lb; $val = $to_max if($results{$_}{$f}{result} ne "SUCCESSFUL" && $results{$_}{$f}{result} ne "FAILED"); $coordstrings{$f} .= " ($_ ($results{$_}{status}), $val)\n"; } $coordstrings{$f} =~ s/\//:/g; $coordstrings{$f} =~ s/_/-/g; } my $labelloc = ($labellen * -0.15) -1; my $legendloc = -2 + $labelloc; print TEX << "EOF"; \\begin{$axis}[ title={\\Large $title}, xlabel={Benchmark}, ylabel={Time}, axis x line*=none, axis y line=box, ymin=$lb, ymax=$to_max, enlarge x limits=$xenlarge, bar width=3pt, ybar, legend style={at={(0.5,${legendloc}cm)}, anchor=north,legend columns=-1}, symbolic x coords={$labels}, xtick=data, x tick label style={rotate=60,anchor=east}, x label style={at={(0.5,${labelloc}cm)}},$axis_decoration ] EOF print TEX "\\addplot coordinates { $coordstrings{$_} };\n" foreach (sort keys %per_file_data); print TEX << "EOF"; \\legend{ $legend }; \\end{$axis} EOF } print TEX << "EOF"; \\end{tikzpicture} \\endpgfgraphicnamed \\end{document} EOF system("TEXMFHOME=base:pgfplots ".($opt_X ? "xelatex" : "pdflatex")." --jobname=$fn $fn"); # This is dangerous in Windows as convert changes the file system... system("convert -density 96 -units PixelsPerInch $fn.pdf $fn.png"); system("rm -r pgfplots") if($pgfplotsdl); system("rm -r base") if($pgfbasedl); bmt-0.6/make_table.pl0000755000175000017500000001040512530560050014347 0ustar michaelmichael#!/usr/bin/perl -w # # Copyright (c) 2011 Michael Tautschnig # 2015 Michael Tautschnig # # All rights reserved. Redistribution and use in source and binary forms, with # or without modification, are permitted provided that the following # conditions are met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # 3. All advertising materials mentioning features or use of this software # must display the following acknowledgement: # # This product includes software developed by Michael Tautschnig, # Queen Mary University of London # # 4. Neither the name of the University nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # # # THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS `AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE # DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # build a LaTeX table of results use strict; use warnings FATAL => qw(uninitialized); sub usage { print <<"EOF"; Usage: $0 CSV column-headers ... where CSV is a comma-separated data file as built by make_csv.pl and column-headers is a list of one or more column names the data of which should be used for the resulting LaTeX table EOF } if (scalar(@ARGV) < 2) { usage; exit 1; } my $file = $ARGV[0]; shift @ARGV; open my $CSV, "<$file" or die "File $file not found\n"; print <<'EOF'; % requires booktabs package \begin{table} \centering EOF print ' \begin{tabular}{' . "l" x scalar(@ARGV) . "}\n"; print " \\toprule\n"; print " \\multicolumn{1}{c}{" . join("} & \n \\multicolumn{1}{c}{", @ARGV) . "}\\\\ \\midrule\n"; my %globals = (); use Text::CSV; my $csv = Text::CSV->new(); my $arref = $csv->getline($CSV); defined($arref) or die "Failed to parse headers\n"; $csv->column_names(@$arref); my %columns = (); $columns{$_} = { width => 0, data => [] } foreach (@ARGV); while (my $row = $csv->getline_hr($CSV)) { foreach (qw(command timeout uname cpuinfo meminfo memlimit)) { defined($row->{$_}) or die "No $_ data in table\n"; defined($globals{$_}) or $globals{$_} = (); $globals{$_}{$row->{$_}} = 1; } foreach my $c (@ARGV) { my $val = defined($row->{$c}) ? $row->{$c} : "n/a"; $val =~ s/_/\\_/g; $val = sprintf("%.3f", $val) if ($val =~ /^\d+\.\d+$/); $columns{$c}{width} = length($val) + 1 if ($columns{$c}{width} < (length($val) + 1)); push @{ $columns{$c}{data} }, $val; } } while (scalar(@{ $columns{$ARGV[0]}{data} })) { my @data = (); foreach my $c (@ARGV) { my $val = shift @{ $columns{$c}{data} }; my $minus = ($val =~ /^\d/) ? "" : "-"; push @data, sprintf("%$minus$columns{$c}{width}s ", $val); } print join(" & ", @data) . "\\\\\n"; } close $CSV; print <<"EOF"; \\bottomrule \\end{tabular} EOF print '\\caption{Benchmarks obtained using ' . join(';', keys %{ $globals{command} }) . ' with a timeout of ' . join(';', keys %{ $globals{timeout} }) . ' and a memory limit of ' . join(';', keys %{ $globals{memlimit} }) . "}\n"; print <<"EOF"; \\label{tab:some-results} \\end{table} EOF print 'The benchmarks were run on a ' . join(';', keys %{ $globals{uname} }) . ' ' . join(';', keys %{ $globals{cpuinfo} }) . ' system equipped with ' . join(';', keys %{ $globals{meminfo} }) . " RAM.\n"; bmt-0.6/parse-poirot.pl0000644000175000017500000000457112530560050014713 0ustar michaelmichael#!/usr/bin/perl -w # # Copyright (c) 2011 Michael Tautschnig # 2015 Michael Tautschnig # # All rights reserved. Redistribution and use in source and binary forms, with # or without modification, are permitted provided that the following # conditions are met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # 3. All advertising materials mentioning features or use of this software # must display the following acknowledgement: # # This product includes software developed by Michael Tautschnig, # Queen Mary University of London # # 4. Neither the name of the University nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # # # THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS `AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE # DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # parse the Poirot specific part of a benchmark log file use strict; use warnings FATAL => qw(uninitialized); sub parse_log { my ($LOG, $hash) = @_; $hash->{Result} = "ERROR"; while (<$LOG>) { chomp; return 1 if (/^###############################################################################$/); if (/^Program has no bugs$/) { $hash->{Result} = "SUCCESSFUL"; } elsif (/This assertion can fail/) { $hash->{Result} = "FAILED"; } } } return 1; bmt-0.6/Changelog0000644000175000017500000001115212530560723013544 0ustar michaelmichael2015-05-25 Michael Tautschnig * make_graph.pl: Reduce restrictions on data input * VERSION: Release 0.6 2013-08-09 Michael Tautschnig * unpack.sh: Fix for newer versions of patch * parse-cseq.pl: Added script to parse CSeq's output 2013-01-13 Michael Tautschnig * parse-cbmc.pl: Improved concurrency statistics parsing * parse-cseq.pl: Added script to parse CSeq's output 2013-01-12 Michael Tautschnig * make_graph.pl, make_piechart.pl: Fixed bugs in non-xkcd output, improved portability of xkcd output * parse-cbmc.pl: Improved and updated concurrency statistics parsing 2012-10-19 Michael Tautschnig * make_graph.pl, make_piechart.pl: Added option -X to format graphs in xkcd style 2012-07-30 Michael Tautschnig * parse-{llbmc,esbmc,mmchecker,poirot,threader,blender,astree}.pl: New result parsers for LLBMC/ESBMC/MMchecker/Poirot/Threader/Blender/Astree * parse-{checkfence,bfc,satabs2}.pl: New result parsers for CheckFence/BFC/CImpact * parse-cbmc.pl: Support statistics for concurrent programs * parse-satabs.pl: Additional statistics for symmetry-aware verification * make_piechart.pl: New subcommand snd-graph to build soundness graphs * cpbm: Document new subcommands, new graph styles * verify.sh: Fixed claim and timeout handling, more version checks supported * make_web.pl: New options -z, -h, -n * make_csv.pl: More flexibility in log file name parsing, minor fixes, sort columns * make_graph.pl: Rewrite to use LaTeX/TikZ instead of GNUplot for graph output, added cactus plot support * make_jtl.pl: Generate JMeter XML output from CSV, new subcommand jtl * VERSION: Release 0.5 2011-04-07 Michael Tautschnig * README: Clarification of description and formatting 2011-03-31 Michael Tautschnig * verify.sh: Proper check for executable * verify.sh, make_csv.pl: Support for Mac OS X, mingw32 and cygwin * README: Updated (future) home URL * parse-ai.pl: New results parser for CPROVER ai tool * unpack.sh, update.sh: Exit with proper error message if source archive contains extra files * parse-*.pl: Set default result to ERROR * VERSION: Release 0.4 2011-03-28 Michael Tautschnig * parse-rr-aa.pl: Collect instruction counts * cil_wrapper.sh: Added option --no-simplify * VERSION: Release 0.3 2011-03-23 Michael Tautschnig * cil_wrapper.sh: Properly handle single source file case * rules.cpbm.generic: Force SHELL to /bin/bash to support trap ERR 2011-03-21 Michael Tautschnig * README: Document new web target * parse-rr-aa.pl: Include time of "repair iterations" in output 2011-03-19 Michael Tautschnig * make_web.pl, cpbm, rules.cpbm.generic: Added command to generate an HTML table and collect the log files for web publication * VERSION: Release 0.2 2011-03-18 Michael Tautschnig * parse-{cbmc,satabs,loopfrog,wolverine,scratch}.pl: Move previous parse-cprover.pl to one parser per tool, currently all sharing the same code; removed executable flag from all parse-*.pl scriptlets * rules.template, rules.cpbm.generic: Moved all generally useful rules to rules.cpbm.generic that is called from default rules file; all targets may be overridden; added more comments * make_csv.pl: Print benchmark result lines sorted by file name; benchmark name is without file suffix and claim name is omitted if ALL_CLAIMS; default Result value to ERROR * make_table.pl: Escape _ in LaTeX output of values, print floating point numbers with rounding to three digits, make columns suitable and fixed width, beautify headers using multicolumn 2011-03-17 Michael Tautschnig * verify.sh: Ignore errors from $TOOL --version; check for executability * make_csv.pl: Use cprover/ as primary search path for log parser snippet; support all [a-z]+ filename suffixes; print the file name being processed to stderr * unpack.sh, update.sh: Support plain tar archives * README: Documented parser lookup in make_csv.pl * cil_wrapper.sh: Use hardly documented CILLY_DONT_LINK_AFTER_MERGE to stop cilly from running the linker * parse-rr-aa.pl: Added parser for experiments using LLVM and an implementation of Rugina and Rinard's pointer analysis * cpbm: Added --help, --version, home options 2011-03-14 Michael Tautschnig * Initial release. bmt-0.6/parse-rr-aa.pl0000644000175000017500000000776312530560050014407 0ustar michaelmichael#!/usr/bin/perl -w # # Copyright (c) 2011 Michael Tautschnig # 2015 Michael Tautschnig # # All rights reserved. Redistribution and use in source and binary forms, with # or without modification, are permitted provided that the following # conditions are met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # 3. All advertising materials mentioning features or use of this software # must display the following acknowledgement: # # This product includes software developed by Michael Tautschnig, # Queen Mary University of London # # 4. Neither the name of the University nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # # # THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS `AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE # DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # parse Rugina/Rinard timing statistics use strict; use warnings FATAL => qw(uninitialized); sub parse_log { my ($LOG, $hash) = @_; my %threads = (); $hash->{Result} = "ERROR"; $hash->{instcount} = 0; while (<$LOG>) { chomp; last if (/^###############################################################################$/); if (/Miscellaneous Ungrouped Timers/) { $hash->{Result} = "SUCCESS"; # 0.0020 ( 33.3%) 0.0020 ( 33.3%) 0.0026 ( 38.1%) Thread 0x7fff429d0a20 Iteration 1 Timer } elsif (/^\s+(\d+\.\d+)\s+.*Thread 0x([a-f0-9]+) Iteration (\d+) Timer\s*$/) { if (!defined($threads{$2})) { $threads{$2} = { total_time => 0, max_iter => 0, per_iter => {} }; } $threads{$2}{total_time} += $1; $threads{$2}{max_iter} = $threads{$2}{max_iter} < $3 ? $3 : $threads{$2}{max_iter}; defined($threads{$2}{per_iter}{$3}) and die "Iteration timer already seen\n"; $threads{$2}{per_iter}{$3} = $1; } elsif (/^\s*(\d+) instcount\s+- Number of/) { $hash->{instcount} += $1; } } return 1 if ($hash->{Result} eq "FAILED"); my $max_iter = 0; my $avg_iter = 0; my $max_time = 0; my $avg_time = 0; my $avg_it1_part = 0; my $total_time_not_it1 = 0; foreach my $t (keys %threads) { $max_iter = $max_iter < $threads{$t}{max_iter} ? $threads{$t}{max_iter} : $max_iter; $avg_iter += $threads{$t}{max_iter}; $max_time = $max_time < $threads{$t}{total_time} ? $threads{$t}{total_time} : $max_time; $avg_time += $threads{$t}{total_time}; $avg_it1_part += $threads{$t}{per_iter}{1} / $threads{$t}{total_time} if ($threads{$t}{total_time} > 0.0); $total_time_not_it1 += $threads{$t}{total_time} - $threads{$t}{per_iter}{1}; } $hash->{maxiter} = $max_iter; $hash->{avgiter} = $avg_iter / scalar(keys %threads); $hash->{maxtime} = $max_time; $hash->{avgtime} = $avg_time / scalar(keys %threads); $hash->{avgit1part} = $avg_it1_part / scalar(keys %threads); $hash->{nthreads} = scalar(keys %threads); $hash->{timenotit1} = $total_time_not_it1; } return 1; bmt-0.6/README0000644000175000017500000002200712530560050012604 0ustar michaelmichael******************************** * CPROVER Benchmarking Toolkit * ******************************** The benchmarking toolkit consists of three main components: - Patch set management tools - Benchmark execution helpers - Result evaluation All steps are performed by subcommands of the main `cpbm' command. Type `cpbm help' to get the list of all subcommands. Please report problems, bugs, questions, suggestions to Michael Tautschnig . Benchmark Execution Example =========================== Download the benchmark package $ wget http://www.cprover.org/software/benchmarks/linux-2.6.19-ddverify.cprover-bm.tar.gz Unpack the kernel sources and the patch set $ cpbm unpack \ http://www.kernel.org/pub/linux/kernel/v2.6/linux-2.6.19.tar.bz2 \ linux-2.6.19-ddverify.cprover-bm.tar.gz Enter the new directory $ cd linux-2.6.19-ddverify Run the default benchmark configuration and produce a LaTeX table $ cprover/rules -j4 table Produce a TikZ graph as TeX, PDF and PNG $ cprover/rules graph Build a web page providing an overview table and all log files in cprover/results.satabs.dfl.web/: $ cprover/rules web Patch Set Management ==================== Benchmarks consist of two files: (1) an original source archive as obtained from its original authors, without modification and (2) a corresponding patch set .cprover-bm.tar.gz that contains all modifications, execution scripts, and other helpers. This design is inspired by Debian v3 source packages and may thus sound familiar to people who are used to working with Debian packages. (1) The original source archive must be a .zip, .tar.gz, .tgz, .tar.bz2, or .tar archive. If the authors of the software to be benchmarked do not offer such an archive, it should be built manually from whatever the authors provide as source. (2) The patch set .cprover-bm.tar.gz is created and managed by cpbm update as described below. The archive ships a cprover/ directory that may contain arbitrary files, but at least a Makefile cprover/rules must be provided. The patch set management scripts will populate a cprover/patches directory that precisely describes all modification to original sources. The basic design of this archive solution is completely independent of the software to be benchmarked. Its main purpose is the precise description of changes to the original source that were made in order to benchmark some tool. Patch set management commands ----------------------------- - cpbm unpack: Takes an archive of original source (or a URL to download the archive from) and the corresponding patch set .cprover-bm.tar.gz. cpbm unpack then builds the directory from the contents of the original source archive, unpacks the cprover directory in , and applies patches from cprover/patches, if any. - cpbm init and cpbm update: Create and maintain a .cprover-bm.tar.gz file. This file is initially created in the parent directory by running cpbm init from within directory resulting from manually unpacking . This step creates the cprover/ directory and populates it with a template cprover/rules file. Once the .cprover-bm.tar.gz file has been created for the first time, it will only be updated by cpbm update. To this end, cpbm update unpacks the source archive into a temporary directory, applies patches previously recorded in cprover/patches, and computes the diff between the current working directory and the contents of the patched source. If new changes are found, these are recorded as new patches in cprover/patches. The sequence of patches to be applied is stored in cprover/patches series. It is recommended that automatically created patches are renamed to more descriptive names. Consequently such renamings must be reflected in cprover/patches/series. Usage Example ------------- Creating a new benchmark suite, e.g., for the Linux kernel: Download the original sources from kernel.org: $ wget http://www.kernel.org/pub/linux/kernel/v2.6/linux-2.6.19.tar.bz2 Manually unpack them: $ tar xjf linux-2.6.19.tar.bz2 We rename the directory to get proper benchmark name "linux-2.6.19-foo" $ mv linux-2.6.19 linux-2.6.19-foo $ cd linux-2.6.19-foo Create the basic patch set linux-2.6.19-foo.cprover-bm.tar.gz $ cpbm init ../linux-2.6.19.tar.bz2 Edit some source files ... and record patches $ cpbm update ../linux-2.6.19.tar.bz2 To make the benchmark source and all patches available for others to use publish linux-2.6.19-foo.cprover-bm.tar.gz and the original source archive or its URL. Using the benchmark suite: $ cpbm unpack \ http://www.kernel.org/pub/linux/kernel/v2.6/linux-2.6.19.tar.bz2 \ linux-2.6.19-foo.cprover-bm.tar.gz $ cd linux-2.6.19-foo Benchmark Execution =================== cpbm init, as described above, creates as template the cprover/rules file (a Makefile). For most basic use cases filling in the names of the benchmarks to be run (e.g., all C files without the .c suffix) in the BENCHMARKS = XXX line and choosing a suitable default configuration in CONFIG = XXX (see examples in there) will suffice to produce a working benchmark package. The file may, however, be fully customized as deemed necessary. The only assumption made by cpbm is that `cprover/rules clean` performs a cleanup. Remember to run cpbm update before distributing the .cprover-bm.tar.gz file. The actual benchmark execution is then triggered by $ cprover/rules -j4 verify to perform verification of all benchmark instances in 4 parallel threads of execution with the default configuration. This step first induces a build of each benchmark from source, if necessary. To choose a different configuration for the verification run, override the CONFIG variable. For instance, to perform verification using CPAchecker's explicit analysis use $ cprover/rules -j4 verify CONFIG=cpachecker.explicit Benchmark Execution Helpers --------------------------- Benchmark execution makes use of a number of helpers: - cpbm cillify: transform C sources using the C Intermediate Language tool to a format suitable, e.g., for CPAchecker or BLAST (requires Cil installation). - cpbm list-claims: the CPROVER tools are able to selectively verify a chosen claim in a program under scrutiny. This tool lists the possible claims as pairs :, where is the identifier used by the verification tool and is either UNKNOWN, or TRUE in case a claim is trivially satisfied. If the verification CONFIG does not support specific claims (as is the case for all non-CPROVER tools), ALL_CLAIMS:UNKNOWN is return as pseudo claim identifier and verification status. - cpbm run: actually executes the verification tool with the configured options and produces a log file. This log file contains the output of the verification tool plus environment information and statistics. All further benchmark evaluation is based on such log files. Result Evaluation ================= The names of all log files produced by cpbm run will be listed in cprover/verified.$CONFIG. Performing $ cprover/rules csv yields a CSV (comma-separated value) formatted summary of all results in cprover/results.$CONFIG.csv. This file may be read using most spreadsheet programs for further manual inspection and evaluation. With cpbm, however, also LaTeX tables, TikZ graphs, or web pages may be produced: $ cprover/rules table yields a LaTeX summary of execution times and memory usage of all benchmark instances. With $ cprover/rules graph furthermore a graph is produced using TikZ and LaTeX. $ cprover/rules web collects all log files in a new directory cprover/results.$CONFIG.web plus an index.html file that contains an HTML formatted version of the CSV table. Each benchmark links to the respective log file. These steps permits a number of customizations: - The CSV data is produced by cpbm csv, which uses a specific parser of the output for each verification tool. These parsers are perl scriptlets found in the parse-*.pl file of the CPROVER benchmarking toolkit distribution. To produce additional columns in the CSV file, add further patterns to these parsers or copy and adapt one of the existing parsers. New parsers are preferably added to the distribution, but can also be put in the cprover/ directory. If a parser exists both in cprover/ and in the distribution, the former will be used. - The LaTeX table is generated using cpbm table, which takes as arguments a CSV file and one or more column names that shall be included (in the specified order) in the LaTeX output. Consequently, adding further columns to CSV output as described above also permits printing this output to LaTeX. - The presently supported graphs include bar charts, scatter plots and cactus plots, as well some others. These are built using TikZ and LaTeX and display CPU time and memory usage (in box plots) or scatter plots (using cpbm graph -s) for comparison of two tools. For box plots an arbitrary number of CSV files may be specified, whereas scatter plots require exactly two CSV files. bmt-0.6/unpack.sh0000755000175000017500000001144412530560050013547 0ustar michaelmichael#!/bin/bash # # Copyright (c) 2011 Michael Tautschnig # 2015 Michael Tautschnig # # All rights reserved. Redistribution and use in source and binary forms, with # or without modification, are permitted provided that the following # conditions are met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # 3. All advertising materials mentioning features or use of this software # must display the following acknowledgement: # # This product includes software developed by Michael Tautschnig, # Queen Mary University of London # # 4. Neither the name of the University nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # # # THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS `AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE # DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # Takes an original source archive and the benchmark package to build an # unpacked and patched source tree with a cprover directory set -e die() { echo $1 exit 1 } checked_patch() { patch=$1 [ -f $patch ] || die "Patch file $patch not found" lc=`patch -f -p1 --dry-run < $patch 2>&1 | egrep -v "^(patching|checking) file" | wc -l` if [ $lc -ne 0 ] ; then patch -f -p1 --dry-run < $patch || true die "Failed to apply patch `basename $patch`" fi echo "Applying patch `basename $patch`" patch -p1 < $patch } usage() { cat < # 2015 Michael Tautschnig # # All rights reserved. Redistribution and use in source and binary forms, with # or without modification, are permitted provided that the following # conditions are met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # 3. All advertising materials mentioning features or use of this software # must display the following acknowledgement: # # This product includes software developed by Michael Tautschnig, # Queen Mary University of London # # 4. Neither the name of the University nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # # # THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS `AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE # DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # main CPROVER benchmarking framework script set -e die() { echo $1 exit 1 } usage() { cat < # 2015 Michael Tautschnig # # All rights reserved. Redistribution and use in source and binary forms, with # or without modification, are permitted provided that the following # conditions are met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # 3. All advertising materials mentioning features or use of this software # must display the following acknowledgement: # # This product includes software developed by Michael Tautschnig, # Queen Mary University of London # # 4. Neither the name of the University nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # # # THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS `AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE # DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # parse the CPROVER tool specific part of a benchmark log file use strict; use warnings FATAL => qw(uninitialized); sub parse_log { my ($LOG, $hash) = @_; $hash->{Result} = "ERROR"; while (<$LOG>) { chomp; return 1 if (/^###############################################################################$/); if (/^Timeout: aborting command/) { $hash->{Result} = "TIMEOUT"; } elsif (/^VERIFICATION\s+(\S+)$/) { $hash->{Result} = "$1"; } elsif (/^size of program expression:\s+(\d+)/) { $hash->{"ssa-size"} = $1; } elsif (/^max-per-address:\s+(\d+)$/) { $hash->{mt_max_per_address} = $1; } elsif (/^num-unique-addresses:\s+(\d+)$/) { $hash->{mt_num_unique_addresses} = $1; } elsif (/^tot-subevent-count:\s+(\d+)$/) { $hash->{mt_tot_subevent_count} = $1; } elsif (/^(po|rf|rfi|ws|fr|ab):\s+(\d+)$/) { $hash->{"mt_$1"} = $2; } elsif (/^(atomic-block|rf-at-least-one|thread-spawn|uniproc|thin-air|ws-preceding):\s+(\d+)$/) { $hash->{"mt_$1"} = $2; } } my $max = 0; $hash->{mt_max_constr} = "po (0)"; for my $r (qw/po rf ws fr ab/) { my $val = defined($hash->{"mt_$r"}) ? $hash->{"mt_$r"} : 0; $val += defined($hash->{mt_rfi}) ? $hash->{mt_rfi} : 0 if($r eq "rf"); $hash->{mt_max_constr} = "$r ($val)" if($val > $max); } } return 1; bmt-0.6/parse-mmchecker.pl0000644000175000017500000000457512530560050015341 0ustar michaelmichael#!/usr/bin/perl -w # # Copyright (c) 2011 Michael Tautschnig # 2015 Michael Tautschnig # # All rights reserved. Redistribution and use in source and binary forms, with # or without modification, are permitted provided that the following # conditions are met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # 3. All advertising materials mentioning features or use of this software # must display the following acknowledgement: # # This product includes software developed by Michael Tautschnig, # Queen Mary University of London # # 4. Neither the name of the University nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # # # THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS `AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE # DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # parse the Poirot specific part of a benchmark log file use strict; use warnings FATAL => qw(uninitialized); sub parse_log { my ($LOG, $hash) = @_; $hash->{Result} = "ERROR"; while (<$LOG>) { chomp; return 1 if (/^###############################################################################$/); if ($hash->{Result} eq "ERROR" && /^REPORT:$/) { $hash->{Result} = "SUCCESSFUL"; } elsif (/^method P/) { $hash->{Result} = "FAILED"; } } } return 1; bmt-0.6/rules.template0000644000175000017500000000302012530560050014605 0ustar michaelmichael#!/usr/bin/make -f # add benchmark files without suffix here BENCHMARKS = XXX # select a default configuration, timeout and memory limit CONFIG = XXX # CONFIG = loopfrog.no-inv # CONFIG = cbmc.u5-nua # CONFIG = cpachecker.explicit # CONFIG = blast.dfl # leave timeout and/or memory limit undefined to not use any limit TIMEOUT = 60 # MAXMEM = 3500 # example configuration - CPROVER tools cprover/verified.cbmc.u5-nua: TOOL_OPTS := --32 --unwind 5 --no-unwinding-assertions cprover/verified.loopfrog.no-inv: TOOL_OPTS := --32 --no-invariants cprover/verified.satabs.dfl: TOOL_OPTS := --32 --iterations 20 cprover/verified.scratch.bf: TOOL_OPTS := --32 --bug-finding cprover/verified.wolverine.u5: TOOL_OPTS := --32 --unwind 5 # CPAchecker example configuration CPACHECKER_HOME = /some/path cprover/verified.cpachecker.%: TOOL := $(CPACHECKER_HOME)/scripts/cpa.sh cprover/verified.cpachecker.%: TOOL_OPTS += -setprop output.disable=true -spec $(CPACHECKER_HOME)/test/config/automata/ErrorLocationAutomaton.txt cprover/verified.cpachecker.explicit: TOOL_OPTS += -config $(CPACHECKER_HOME)/test/config/explicitAnalysisInf.properties cprover/verified.cpachecker.symbpredabsCPA: TOOL_OPTS += -config $(CPACHECKER_HOME)/test/config/symbpredabsCPA.properties # BLAST configuration cprover/verified.blast.%: TOOL := pblast.opt cprover/verified.blast.dfl: TOOL_OPTS += # execute default rules using the variables defined above export # any of these rules may be overridden by defining rules below %: $(MAKE) -f $(shell cpbm home)/rules.cpbm.generic $@ bmt-0.6/verify.sh0000755000175000017500000002075412530560050013576 0ustar michaelmichael#!/bin/bash # # Copyright (c) 2011 Michael Tautschnig # 2015 Michael Tautschnig # # All rights reserved. Redistribution and use in source and binary forms, with # or without modification, are permitted provided that the following # conditions are met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # 3. All advertising materials mentioning features or use of this software # must display the following acknowledgement: # # This product includes software developed by Michael Tautschnig, # Queen Mary University of London # # 4. Neither the name of the University nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # # # THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS `AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE # DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # wrapper around CPROVER verification tools to verify a claim at a time and # generate useful benchmarking data set -e die() { echo $1 exit 1 } ifs_bak=$IFS cleanup() { IFS=$ifs_bak if [ -z "$NO_CLEANUP" ] ; then for f in $TMP_FILES ; do if [ -f $f ] ; then rm -f $f fi done rm -f tmp.stderr*.txt fi } trap 'cleanup' ERR EXIT mktemp_local_prefix_suffix() { local varname=$1 local prefix=$2 local suffix=$3 local tmpf="`TMPDIR=. mktemp -t $prefix.XXXXXX`" TMP_FILES="$TMP_FILES $tmpf" local rand="`echo $tmpf | sed "s#^./$prefix\\\.##"`" eval $varname=\"${prefix}_$rand$suffix\" TMP_FILES="$TMP_FILES ${!varname}" [ ! -f "${!varname}" ] || die "File already exists" mv $tmpf ${!varname} } usage() { cat < /dev/null || die "$TOOL is not an executable" TMP_FILES="$TMP_FILES .smv_lock cegar_tmp_abstract.warn \ cegar_tmp_abstract.smv cegar_tmp_abstract.stats \ cegar_tmp_abstract.out cegar_tmp_abstract.update \ cegar_tmp_smv_out1 cegar_tmp_smv_out2" exit_code=0 tmp_files=$TMP_FILES cl="`echo $CLAIM | sed 's/^ //' | sed 's/ /+/g'`" mktemp_local_prefix_suffix claim_out "`basename $MAIN_SOURCE`_${cl}_`date +%Y-%m-%d_%H%M%S`" .log # we want to keep this file TMP_FILES=$tmp_files echo "SOURCES: $SOURCES" echo "CLAIM: $CLAIM" case "$TOOL" in *pblast.opt) version_info=`$TOOL 2>&1 | grep ^BLAST` ;; *cpa.sh) version_info="no info" ;; checkfence) version_info="no info" ;; *cream) version_info=`$TOOL --help | grep "Cream ver " | awk '{ print $4 }'` ;; *) version_info=`$TOOL --version || true` ;; esac case `uname -s` in Linux|CYGWIN_*) mktemp_local_prefix_suffix time_out stats .stat BENCHMARKING="/usr/bin/time -v -o$time_out" cat > $claim_out < $claim_out < $claim_out <> $claim_out <> $claim_out 2>&1 || exit_code=$? sleep 1 # interactive bash needed for proper support of Ctrl-c ? # bash -i -c "$TIMEOUT $BENCHMARKING $TOOL $OPTS --claim ${!claim_id_var} $SOURCES >> $claim_out 2>&1" || exit_code=$? cat >> $claim_out <> $claim_out <> $claim_out < /dev/null 2>&1 ; then TO_OPTS="-2" if ! timeout $TO_OPTS 3 true > /dev/null 2>&1 ; then die "No working timeout command found" fi fi TIMEOUT="timeout $TO_OPTS $2" ; shift 2;; --maxmem) if ! echo $2 | egrep -q "^[[:digit:]]+$" ; then die "Invalid parameter to --maxmem" fi ulimit -S -v $(( $2 * 1024 )) ; shift 2;; --unknown|--valid|--invalid) EXPECT_RESULT="`echo $1 | cut -b3-`" ; shift 1;; --cmd) TOOL="$2" ; shift 2;; --) shift ; break ;; *) die "Unknown option $1" ;; esac done [ -n "$TOOL" ] || die "Please select the verification tool to be used" if [ -n "$CLAIM" ] && [ "$CLAIM" != " ALL_CLAIMS" ] ; then for c in $CLAIM ; do if [ "$TOOL" = "loopfrog" ] ; then CLAIM_CMD="$CLAIM_CMD --testclaim $c" else CLAIM_CMD="$CLAIM_CMD --claim $c" fi done else CLAIM="ALL_CLAIMS" fi unset MAIN_SOURCE OTHER_SOURCES OPTS for o in $@ ; do case "$o" in -*) OPTS=$@ if [ "$TOOL" = "loopfrog" ] ; then OPTS="--no-progress $OPTS" fi break ;; *) [ -f "$o" ] || die "Source file $o not found" if [ "$TOOL" = "scratch" -a "`file -b $o`" = "data" ] ; then SOURCES="$SOURCES --binary $o" else SOURCES="$SOURCES $o" fi if [ -z "$MAIN_SOURCE" ] ; then MAIN_SOURCE=$o fi shift 1 ;; esac done [ -n "$MAIN_SOURCE" ] || die "No source file given" run_tool bmt-0.6/make_piechart.pl0000755000175000017500000002621312530560050015063 0ustar michaelmichael#!/usr/bin/perl -w # # Copyright (c) 2011 Michael Tautschnig # 2015 Michael Tautschnig # # All rights reserved. Redistribution and use in source and binary forms, with # or without modification, are permitted provided that the following # conditions are met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # 3. All advertising materials mentioning features or use of this software # must display the following acknowledgement: # # This product includes software developed by Michael Tautschnig, # Queen Mary University of London # # 4. Neither the name of the University nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # # # THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS `AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE # DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # build a pie chart of correct results, grouped by pattern-defined groups use strict; use warnings FATAL => qw(uninitialized); use Getopt::Std; sub usage { print <<"EOF"; Usage: $0 [OPTIONS] CSV ... GROUP-NAME GROUP-PATTERN ... where CSV is one ore more CSV tables as produced by make_csv.pl; the sequence of GROUP-NAMEs and GROUP-PATTERNs are applied to the benchmark names to build groups Options: Purpose: -h show help -p make a pie chart (otherwise do a bar chart) -g produce a chart per group instead of per file -A display average runtimes instead of soundness results -E display average runtimes with error bars -X beautify graph EOF } our ($opt_p, $opt_g, $opt_h, $opt_A, $opt_E, $opt_X); if (!getopts('hpgAEX') || defined($opt_h) || !scalar(@ARGV)) { usage; exit (defined($opt_h) ? 0 : 1); } use Text::CSV; my %by_category = (); my @files = (); while(scalar(@ARGV) && $ARGV[0] =~ /\.csv$/ && -f $ARGV[0]) { push @files, shift @ARGV; } (scalar(@ARGV) % 2 == 0) or die "Names/patterns must be pairs\n"; foreach my $f (@files) { open my $CSV, "<$f" or die "File $f not found\n"; my $csv = Text::CSV->new(); my $arref = $csv->getline($CSV); defined($arref) or die "Failed to parse headers\n"; $csv->column_names(@$arref); my $dbg_lno = 1; while (my $row = $csv->getline_hr($CSV)) { $dbg_lno++; (defined($row->{$_}) && ! ($row->{$_} =~ /^\s*$/)) or die "No $_ data in $f, line $dbg_lno\n" foreach (qw(Benchmark Result expected usertime)); my $bm = $row->{Benchmark}; my $group = 0; for (my $i=1; $i<=(scalar(@ARGV)/2); $i++) { my $pat = $ARGV[(($i-1)*2)+1]; $group = $i if($bm =~ m{$pat}); } my $groupname = $group==0 ? "Other" : $ARGV[($group-1)*2]; my $tn = $opt_g ? $groupname : $f; $tn =~ s/\.csv$// if(!$opt_g); if($opt_g) { $groupname = $group = $f; $groupname =~ s/\.csv$//; } my $cpu = $row->{usertime}; defined($by_category{$tn}{$group}) or $by_category{$tn}{$group} = { name => $groupname, total => 0, ok => 0, error => 0, unknown => 0, mintime => $cpu, maxtime => 0, tottime => 0 }; my $val = "unknown"; #$val = "error" if($row->{Result} eq "ERROR"); if($row->{expected} eq "valid") { $val = "ok" if($row->{Result} eq "SUCCESSFUL"); $val = "error" if($row->{Result} eq "FAILED"); } elsif($row->{expected} eq "invalid") { $val = "error" if($row->{Result} eq "SUCCESSFUL"); $val = "ok" if($row->{Result} eq "FAILED"); } $by_category{$tn}{$group}{total}++; $by_category{$tn}{$group}{$val}++; $by_category{$tn}{$group}{mintime} = $cpu if($by_category{$tn}{$group}{mintime} > $cpu); $by_category{$tn}{$group}{maxtime} = $cpu if($by_category{$tn}{$group}{maxtime} < $cpu); $by_category{$tn}{$group}{tottime} += $cpu; } close $CSV; } my $pgfbasedl = 0; my $pgfplotsdl = 0; system("wget --no-check-certificate https://github.com/shreyankg/xkcd-desktop/raw/master/Humor-Sans.ttf") if($opt_X && ! -f "Humor-Sans.ttf"); if(! -f "pgf-pie.sty") { system("wget http://pgf-pie.googlecode.com/files/pgf-pie-0.2.zip"); system("unzip -j pgf-pie-0.2.zip pgf-pie-0.2/pgf-pie.sty"); if(! -d "base") { $pgfbasedl = 1; system("wget http://mirror.ctan.org/graphics/pgf/base.zip"); system("unzip base.zip"); } if(! -d "pgfplots") { $pgfplotsdl = 1; system("wget http://mirror.ctan.org/graphics/pgf/contrib/pgfplots.zip"); system("unzip pgfplots.zip"); } } my $decoration = ""; $opt_X and $decoration = "decoration={random steps,segment length=1mm,amplitude=0.2pt}"; my $axis_decoration = ""; $opt_X and $axis_decoration = "xticklabel style={/pgf/number format/assume math mode},\n". " yticklabel style={/pgf/number format/assume math mode},\n". " decorate"; for my $tn (keys %by_category) { my $fn = $tn; $fn =~ s/[:\/]/_/g if($opt_g); if($opt_p) { my $pie_str = ""; my $N = 0; $N += $by_category{$tn}{$_}{total} foreach (keys %{ $by_category{$tn} }); foreach my $cat (sort keys %{ $by_category{$tn} }) { $pie_str .= sprintf("%.0f", $by_category{$tn}{$cat}{ok}/$N*100) . "/" . $by_category{$tn}{$cat}{name} . " (ok), " if($by_category{$tn}{$cat}{ok} > 0); $pie_str .= sprintf("%.0f", $by_category{$tn}{$cat}{error}/$N*100) . "/" . $by_category{$tn}{$cat}{name} . " (wrong result), " if($by_category{$tn}{$cat}{error} > 0); $pie_str .= sprintf("%.0f", $by_category{$tn}{$cat}{unknown}/$N*100) . "/" . $by_category{$tn}{$cat}{name} . " (error/timeout), " if($by_category{$tn}{$cat}{unknown} > 0); } $pie_str =~ s/, $//; open TEX, ">$fn.tex"; print TEX << "EOF"; \\documentclass{article} \\usepackage{fontspec} \\setmainfont[ExternalLocation]{Humor-Sans.ttf} \\usepackage{pgf-pie} \\pgfrealjobname{$fn-nn} \\begin{document} \\beginpgfgraphicnamed{$fn} \\begin{tikzpicture} \\pie[explode=0.1, text=legend]{ $pie_str } \\end{tikzpicture} \\endpgfgraphicnamed \\end{document} EOF } elsif($opt_A || $opt_E) { my %vals = (); foreach my $cat (sort keys %{ $by_category{$tn} }) { my $N = $by_category{$tn}{$cat}{total}; next if($N == 0); my $cn = $by_category{$tn}{$cat}{name}; defined($vals{$cn}) or $vals{$cn} = ""; my $min = $by_category{$tn}{$cat}{mintime}; my $max = $by_category{$tn}{$cat}{maxtime}; my $avg = sprintf("%.0f", $by_category{$tn}{$cat}{tottime}/$N); $vals{$cn} .= " ($cn, $avg)"; $vals{$cn} .= " +- ($min,$max)" if($opt_E); } my @ll = (); push @ll, $by_category{$tn}{$_}{name} foreach (sort keys %{ $by_category{$tn} }); my $labels = join(", ", @ll); my $width = scalar(@ll); $width = 4 if($width < 4); my $xenlarge = 1/$width; open TEX, ">$fn.tex"; print TEX << "EOF"; \\documentclass{article} \\usepackage{fontspec} \\setmainfont{Humor-Sans.ttf} \\usepackage{pgfplots} \\usetikzlibrary{backgrounds} \\tikzset{ extra padding/.style={ show background rectangle, inner frame sep=#1, background rectangle/.style={ draw=none } }, extra padding/.default=0.5cm, $decoration } \\pgfplotsset{width=${width}cm} \\pgfrealjobname{$fn-nn} \\begin{document} \\beginpgfgraphicnamed{$fn} \\begin{tikzpicture}[extra padding] \\begin{axis}[ title={\\Large Average runtimes for $tn}, axis x line*=none, axis y line=box, ymin=0, enlarge x limits=$xenlarge, legend style={at={(0.5,-2.2cm)}, anchor=north,legend columns=-1}, symbolic x coords={$labels}, xtick=data, x tick label style={rotate=60,anchor=east}, xlabel={Category}, ylabel={Time}, $axis_decoration ] EOF my $style = $opt_E ? "error bars/.cd, y dir=both,y explicit" : "ybar,bar width=3pt"; print TEX "\\addplot+[$style] plot coordinates { $vals{$_} };\n" foreach (sort keys %vals); my $legend = join(",", sort keys %vals); print TEX << "EOF"; \\legend{ $legend }; \\end{axis} \\end{tikzpicture} \\endpgfgraphicnamed \\end{document} EOF } else { my $ok = ""; my $unknown = ""; my $error = ""; foreach my $cat (sort keys %{ $by_category{$tn} }) { my $N = $by_category{$tn}{$cat}{total}; next if($N == 0); my $cn = $by_category{$tn}{$cat}{name}; $ok .= " ($cn, " . sprintf("%.0f", $by_category{$tn}{$cat}{ok}/$N*100) . ")"; $unknown .= " ($cn, " . sprintf("%.0f", $by_category{$tn}{$cat}{unknown}/$N*100) . ")"; $error .= " ($cn, " . sprintf("%.0f", $by_category{$tn}{$cat}{error}/$N*100) . ")"; } my @ll = (); push @ll, $by_category{$tn}{$_}{name} foreach (sort keys %{ $by_category{$tn} }); my $labels = join(", ", @ll); my $width = scalar(@ll); $width = 4 if($width < 4); my $xenlarge = 1/$width; open TEX, ">$fn.tex"; print TEX << "EOF"; \\documentclass{article} \\usepackage{fontspec} \\setmainfont{Humor-Sans.ttf} \\usepackage{pgfplots} \\usetikzlibrary{backgrounds} \\tikzset{ extra padding/.style={ show background rectangle, inner frame sep=#1, background rectangle/.style={ draw=none } }, extra padding/.default=0.5cm, $decoration } \\pgfplotsset{width=${width}cm} \\pgfrealjobname{$fn-nn} \\begin{document} \\beginpgfgraphicnamed{$fn} \\begin{tikzpicture}[extra padding] \\begin{axis}[ title={\\Large Soundness results for $tn}, axis x line*=none, axis y line=box, ybar stacked, ymin=-3, ymax=103, enlarge x limits=$xenlarge, legend style={at={(0.5,-2.2cm)}, anchor=north,legend columns=-1}, %ylabel={Some text}, symbolic x coords={$labels}, xtick=data, x tick label style={rotate=60,anchor=east}, $axis_decoration ] \\addplot+[green!50!black, ybar] plot coordinates { $ok }; \\addplot+[orange!90!black, ybar] plot coordinates { $unknown }; \\addplot+[red!70!black, ybar] plot coordinates { $error }; \\legend{ ok, error/timeout, wrong result }; \\end{axis} \\end{tikzpicture} \\endpgfgraphicnamed \\end{document} EOF } system("TEXMFHOME=base:pgfplots ".($opt_X ? "xelatex" : "pdflatex")." --jobname=$fn $fn"); system("convert -density 96 -units PixelsPerInch $fn.pdf $fn.png"); } system("rm -r pgfplots") if($pgfplotsdl); system("rm -r base") if($pgfbasedl); bmt-0.6/parse-cpachecker.pl0000644000175000017500000000554212530560050015466 0ustar michaelmichael#!/usr/bin/perl -w # # Copyright (c) 2011 Michael Tautschnig # 2015 Michael Tautschnig # # All rights reserved. Redistribution and use in source and binary forms, with # or without modification, are permitted provided that the following # conditions are met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # 3. All advertising materials mentioning features or use of this software # must display the following acknowledgement: # # This product includes software developed by Michael Tautschnig, # Queen Mary University of London # # 4. Neither the name of the University nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # # # THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS `AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE # DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # parse the CPAchecker specific part of a benchmark log file use strict; use warnings FATAL => qw(uninitialized); sub parse_log { my ($LOG, $hash) = @_; $hash->{Result} = "ERROR"; while (<$LOG>) { chomp; return 1 if (/^###############################################################################$/); if (/^No error locations reachable from main, analysis not necessary\./) { $hash->{Result} = "ERROR_TRIV_UNREACHABLE"; } elsif (/^Given specification violated\? YES/) { $hash->{Result} = "FAILED"; } elsif (/^Given specification violated\? NO/) { $hash->{Result} = "SUCCESS"; } elsif (/^Number of refinements:\s+(\d+)$/) { $hash->{refinements} = $1; } elsif (/^Size of reached set:\s+(\d+)$/) { $hash->{reached} = $1; } elsif (/^Number of abstractions:\s+(\d+)\s+/) { $hash->{abstractions} = $1; } elsif (/^Max ABE block size:\s+(\d+)$/) { $hash->{blocksize} = $1; } } } return 1; bmt-0.6/parse-satabs2.pl0000644000175000017500000000473512530560050014740 0ustar michaelmichael#!/usr/bin/perl -w # # Copyright (c) 2011 Michael Tautschnig # 2015 Michael Tautschnig # # All rights reserved. Redistribution and use in source and binary forms, with # or without modification, are permitted provided that the following # conditions are met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # 3. All advertising materials mentioning features or use of this software # must display the following acknowledgement: # # This product includes software developed by Michael Tautschnig, # Queen Mary University of London # # 4. Neither the name of the University nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # # # THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS `AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE # DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # parse the CPROVER tool specific part of a benchmark log file use strict; use warnings FATAL => qw(uninitialized); sub parse_log { my ($LOG, $hash) = @_; $hash->{Result} = "ERROR"; while (<$LOG>) { chomp; return 1 if (/^###############################################################################$/); if (/^Too many iterations, giving up/) { $hash->{Result} = "TOO_MANY_ITERATIONS"; } elsif (/^Timeout: aborting command/) { $hash->{Result} = "TIMEOUT"; } elsif (/^VERIFICATION\s+(\S+)$/) { $hash->{Result} = "$1"; } } } return 1; bmt-0.6/rules.cpbm.generic0000644000175000017500000000521412530560050015335 0ustar michaelmichael#!/usr/bin/make -f # trap ... ERR isn't supported by dash, which is used on some systems SHELL = /bin/bash # building the source code ifeq ($(filter-out cpachecker.%,$(CONFIG)),) COMPILER ?= cpbm cillify -D__CPROVER_assert=__assert_fail -Dassert=__assert_fail SUFFIX ?= i endif ifeq ($(filter-out blast.%,$(CONFIG)),) COMPILER ?= cpbm cillify --blast SUFFIX ?= i endif ifeq ($(filter-out llvm.%,$(CONFIG)),) COMPILER ?= clang -emit-llvm -c SUFFIX ?= s endif COMPILER ?= goto-cc --32 SUFFIX ?= bin build: test -d cprover $(MAKE) -f cprover/rules cprover/binaries.$(CONFIG) cprover/binaries.$(CONFIG): $(addsuffix .$(SUFFIX), $(addprefix build/, $(BENCHMARKS))) $(RM) $@ set -e ; trap '$(RM) $@' ERR ; \ for f in $^ ; do \ echo $$f >> $@ ; \ done build/%.$(SUFFIX): %.c mkdir -p $(dir $@) cd $(dir $<) ; $(COMPILER) -o $(abspath $@) $(realpath $<) # verification rules verify: build test -d cprover $(MAKE) -f cprover/rules cprover/verified.$(CONFIG) cprover/verified.$(CONFIG): $(addsuffix .vr, $(addprefix results.$(CONFIG)/, $(BENCHMARKS))) set -e ; trap '$(RM) $@' ERR ; \ cat $^ > $@ results.$(CONFIG)/%vr: build/%$(SUFFIX) mkdir -p $(dir $@) set -e ; trap '$(RM) $(abspath $@)' ERR ; \ tool="$(TOOL)" ; if [ "x$$tool" = "x" ] ; then tool="$(basename $(CONFIG))" ; fi ; \ timeout="$(TIMEOUT)" ; if [ "x$$timeout" != "x" ] ; then timeout="--timeout $(TIMEOUT)" ; fi ; \ maxmem="$(MAXMEM)" ; if [ "x$$maxmem" != "x" ] ; then maxmem="--maxmem $(MAXMEM)" ; fi ; \ cd $(dir $@) ; \ claims=`cpbm list-claims --cmd $$tool $(realpath $<) -- $(TOOL_OPTS)` ; \ for c in $$claims ; do \ cl=`echo $$c | cut -f1 -d:` ; \ st=`echo $$c | cut -f2 -d:` ; \ if [ "$$st" = "TRUE" ] ; then st="--valid" ; else st="--unknown" ; fi ; \ cpbm run --claim $$cl $$st $$timeout $$maxmem --cmd $$tool $(realpath $<) -- $(TOOL_OPTS) ; \ done | tee $(abspath $@) ; \ exit $${PIPESTATUS[0]} # statistics collection csv: verify test -d cprover $(MAKE) -f cprover/rules cprover/results.$(CONFIG).csv cprover/results.$(CONFIG).csv: cprover/verified.$(CONFIG) set -e ; trap '$(RM) $@' ERR ; \ parser="$(PARSER)" ; if [ "x$$parser" = "x" ] ; then parser="$(basename $(CONFIG))" ; fi ; \ cpbm csv $$parser $^ > $@ table: csv test -d cprover cpbm table cprover/results.$(CONFIG).csv Benchmark Result usertime maxmem graph: csv test -d cprover cpbm graph cprover/results.$(CONFIG).csv web: csv test -d cprover cpbm web cprover/results.$(CONFIG).csv cprover/results.$(CONFIG).web # cleanup clean: test -d cprover rm -rf results.* build rm -f cprover/binaries.* cprover/verified.* rm -rf cprover/results.* .PHONY: clean build verify csv table graph web bmt-0.6/VERSION0000644000175000017500000000000412530560050012765 0ustar michaelmichael0.6 bmt-0.6/list_claims.sh0000755000175000017500000001376412530560050014600 0ustar michaelmichael#!/bin/bash # # Copyright (c) 2011 Michael Tautschnig # 2015 Michael Tautschnig # # All rights reserved. Redistribution and use in source and binary forms, with # or without modification, are permitted provided that the following # conditions are met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # 3. All advertising materials mentioning features or use of this software # must display the following acknowledgement: # # This product includes software developed by Michael Tautschnig, # Queen Mary University of London # # 4. Neither the name of the University nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # # # THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS `AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE # DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # wrapper around CPROVER verification tools to obtain a list of claims; for # other (unsupported) verification tools, ALL_CLAIMS:UNKNOWN is returned set -e die() { echo $1 exit 1 } ifs_bak=$IFS cleanup() { IFS=$ifs_bak if [ -z "$NO_CLEANUP" ] ; then for f in $TMP_FILES ; do if [ -f $f ] ; then rm $f fi done rm -f tmp.stderr*.txt fi } trap 'cleanup' ERR EXIT mktemp_local_prefix_suffix() { local varname=$1 local prefix=$2 local suffix=$3 local tmpf="`TMPDIR=. mktemp -t $prefix.XXXXXX`" TMP_FILES="$TMP_FILES $tmpf" local rand="`echo $tmpf | sed "s#^./$prefix\\\.##"`" eval $varname=\"${prefix}_$rand$suffix\" TMP_FILES="$TMP_FILES ${!varname}" [ ! -f "${!varname}" ] || die "File already exists" mv $tmpf ${!varname} } usage() { cat <&2 mktemp_local_prefix_suffix list_of_claims claims .txt exit_code=0 $TOOL $OPTS --show-claims $SOURCES > $list_of_claims || exit_code=$? if [ $exit_code -ne 0 ] && [ "$TOOL" != "satabs" -o $exit_code -ne 1 ] ; then die "Could not start $TOOL" fi echo "done." 1>&2 echo -n "Parsing $TOOL output ... " 1>&2 mktemp_local_prefix_suffix list_of_claims_converted claimsconv .txt awk ' BEGIN {foundclaim = 0;} /^Claim*/ {foundclaim = 1; claim = NR; n = length($2) - 1; str = substr($2, 1, n); print str} (NR == claim + 1) && (foundclaim == 1) {print $2 "\n" $4} (NR == claim + 2) && (foundclaim == 1) {print substr($0, 3)} (NR == claim + 3) && (foundclaim == 1) {print substr($0, 3) "\n"} ' $list_of_claims > $list_of_claims_converted echo "done." 1>&2 read_claims $list_of_claims_converted echo "$NUMBER_CLAIMS claims" 1>&2 print_claims } SELF=$0 opts=`getopt -n "$0" -o "h" --long "\ help,\ no-cleanup,\ cmd:,\ " -- "$@"` eval set -- "$opts" unset NO_CLEANUP TOOL while true ; do case "$1" in -h|--help) usage ; exit 0;; --no-cleanup) NO_CLEANUP=1 ; shift 1;; --cmd) TOOL="$2" ; shift 2;; --) shift ; break ;; *) die "Unknown option $1" ;; esac done [ -n "$TOOL" ] || die "Please select the verification tool to be used" case "$TOOL" in satabs|cbmc|wolverine|scratch|loopfrog) unset SOURCES OPTS for o in $@ ; do case "$o" in -*) OPTS=$@ break ;; *) [ -f "$o" ] || die "Source file $o not found" if [ "$TOOL" = "scratch" -a "`file -b $o`" = "data" ] ; then SOURCES="$SOURCES --binary $o" else SOURCES="$SOURCES $o" fi shift 1 ;; esac done [ -n "$SOURCES" ] || die "No source file given" run_tool ;; *) echo "No specific support for $TOOL" 1>&2 echo "ALL_CLAIMS:UNKNOWN" ;; esac bmt-0.6/cil_wrapper.sh0000755000175000017500000001305012530560050014570 0ustar michaelmichael#!/bin/bash # # Copyright (c) 2011 Michael Tautschnig # 2015 Michael Tautschnig # # All rights reserved. Redistribution and use in source and binary forms, with # or without modification, are permitted provided that the following # conditions are met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # 3. All advertising materials mentioning features or use of this software # must display the following acknowledgement: # # This product includes software developed by Michael Tautschnig, # Queen Mary University of London # # 4. Neither the name of the University nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # # # THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS `AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE # DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # wrapper around Cil to compile multiple source files into a single # Cil-simplified and preprocessed file. set -e die() { echo $1 exit 1 } unset TMP_FILES TMP_DIRS NO_CLEANUP ifs_bak=$IFS cleanup() { IFS=$ifs_bak if [ -z "$NO_CLEANUP" ] ; then for f in $TMP_FILES ; do if [ -f $f ] ; then rm $f fi done for d in $TMP_DIRS ; do if [ -d $d ] ; then rm -r $d fi done fi } trap 'cleanup' ERR EXIT mktemp_local_prefix_suffix() { local varname=$1 local prefix=$2 local suffix=$3 local tmpf="`TMPDIR=. mktemp -t $prefix.XXXXXX`" TMP_FILES="$TMP_FILES $tmpf" local rand="`echo $tmpf | sed "s#^./$prefix\\\.##"`" eval $varname=\"${prefix}_$rand$suffix\" TMP_FILES="$TMP_FILES ${!varname}" [ ! -f "${!varname}" ] || die "File already exists" mv $tmpf ${!varname} } usage() { cat < $d/$preproc_src <> $d/$preproc_src preproc_files="$preproc_files $preproc_src" done cd $d TMP_DIRS="../$d" if [ "x$OTHER_SOURCES" = "x" ] ; then if [ -n "$NO_SIMPLIFY" ] ; then cd .. TMP_DIRS="$d" mv $d/`basename $preproc_files` $OUTPUT else CILLY_DONT_LINK_AFTER_MERGE=1 cilly --save-temps -x c --dosimplify --printCilAsIs --domakeCFG -o `basename $OUTPUT` $preproc_files cd .. TMP_DIRS="$d" mv $d/`basename $preproc_files .i`.cil.c $OUTPUT fi else CILLY_DONT_LINK_AFTER_MERGE=1 cilly --save-temps -x c --merge --keepmerged -o `basename $OUTPUT` $preproc_files if [ -n "$NO_SIMPLIFY" ] ; then cd .. TMP_DIRS="$d" mv $d/`basename ${OUTPUT}_comb.c` $OUTPUT else CILLY_DONT_LINK_AFTER_MERGE=1 cilly --save-temps -x c --dosimplify --printCilAsIs --domakeCFG -o `basename $OUTPUT` `basename ${OUTPUT}_comb.c` cd .. TMP_DIRS="$d" mv $d/`basename ${OUTPUT}_comb.cil.c` $OUTPUT fi fi bmt-0.6/make_csv.pl0000755000175000017500000002330012530560050014051 0ustar michaelmichael#!/usr/bin/perl -w # # Copyright (c) 2011 Michael Tautschnig # 2015 Michael Tautschnig # # All rights reserved. Redistribution and use in source and binary forms, with # or without modification, are permitted provided that the following # conditions are met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # 3. All advertising materials mentioning features or use of this software # must display the following acknowledgement: # # This product includes software developed by Michael Tautschnig, # Queen Mary University of London # # 4. Neither the name of the University nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # # # THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS `AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE # DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # build a CSV table of results use strict; use warnings FATAL => qw(uninitialized); sub usage { print <<"EOF"; Usage: $0 TOOL RESULTS where TOOL specifies the log parser to be used an RESULTS lists log files produced by verify.sh EOF } if (scalar(@ARGV) != 2) { usage; exit 1; } my $config = $ARGV[0]; my $file = $ARGV[1]; use File::Basename qw(dirname); use lib dirname($0); use lib "cprover/"; require "parse-$config.pl"; sub parse_description { my ($LOG, $hash) = @_; my $mode = ""; while (<$LOG>) { chomp; return 1 if (/^###############################################################################$/); $mode = "uname", next if (/^### uname -a:$/); $mode = "cpuinfo", next if (/^### cat \/proc\/cpuinfo:$/); $mode = "cpuinfo_osx", next if (/^### \/usr\/sbin\/system_profiler -detailLevel full SPHardwareDataType:$/); $mode = "meminfo", next if (/^### cat \/proc\/meminfo:$/); $mode = "meminfo_osx", next if (/^### \/usr\/sbin\/system_profiler -detailLevel full SPMemoryDataType:$/); $mode = "date", next if (/^### date:$/); $mode = "user", next if (/^### user:$/); $mode = "ulimit", next if (/^### ulimit -a:$/); $mode = "timeout", next if (/^### timeout:$/); $mode = "version", next if (/^### tool version info:$/); $mode = "command", next if (/^### tool command:$/); $mode = "commandline", next if (/^### full command line:$/); $mode = "expected", next if (/^### expected verification result:$/); if ($mode eq "uname") { /^(\S+)\s+(\S+)\s+(\S+)\s+/ or die "Unexpected uname output\n"; $hash->{$mode} = "$1 $3"; } elsif ($mode eq "cpuinfo") { defined($hash->{$mode}) or $hash->{$mode} = "0x XXX \@\@YYYMHz"; /^(processor\s*:\s*(\d+))|(model name\s*:\s*(.*))|(cpu MHz\s*:\s*(.*))$/ or next; if (defined($2)) { ($hash->{$mode} =~ /^(\d+)(x .*)$/) or die "Invalid cpu def\n"; $hash->{$mode} = ($1 + 1) . $2; } elsif (defined($4)) { my $model = $4; ($hash->{$mode} =~ /^(\d+)x (.*) @@(.*)MHz$/) or die "Invalid cpu def\n"; ($2 eq "XXX") or ($2 eq $model) or die "NUMA system with both $model and $2 not supported\n"; $hash->{$mode} = "$1x $model \@\@$3MHz"; } elsif (defined($6)) { my $freq = $6; ($hash->{$mode} =~ /^(\d+)x (.*) @@(.*)MHz$/) or die "Invalid cpu def\n"; $hash->{$mode} = "$1x $2 \@\@${freq}MHz"; } } elsif ($mode eq "cpuinfo_osx") { defined($hash->{cpuinfo}) or $hash->{cpuinfo} = "1x XXX \@\@YYYMHz"; if (/^\s+Processor Name: (.*)$/) { my $model = $1; $hash->{cpuinfo} =~ s/XXX/$model/; } elsif (/^\s+Processor Speed: (.*) GHz$/) { my $freq = $1 * 1000; $hash->{cpuinfo} =~ s/YYY/$freq/; } elsif (/^\s+(Number Of Processors|Total Number Of Cores): (\d+)$/) { my $count = $2; ($hash->{cpuinfo} =~ /^(\d+)(x .*)$/) or die "Invalid cpu def\n"; $hash->{cpuinfo} = ($1 * $count) . $2; } } elsif ($mode eq "meminfo") { /^MemTotal:\s+(\d+) kB/ or next; $hash->{$mode} = "$1kb"; } elsif ($mode eq "meminfo_osx") { defined($hash->{meminfo}) or $hash->{meminfo} = "0kb"; /^\s+Size:\s+(\d+)\s+GB/ or next; my $banksize = $1 * 1024 * 1024; ($hash->{meminfo} =~ /^(\d+)kb$/) or die "Invalid mem info\n"; $hash->{meminfo} = ($1 + $banksize) . "kb"; } elsif ($mode eq "date") { $hash->{$mode} = "$_"; } elsif ($mode eq "user") { $hash->{$mode} = "$_"; } elsif ($mode eq "ulimit") { /^virtual memory\s+\(kbytes, -v\) (unlimited|\d+)$/ or next; $hash->{memlimit} = "$1kb"; } elsif ($mode eq "timeout") { /^(\s*|timeout -(s SIGINT|2) (\d+))$/ or die "Unexpected timeout format\n"; $hash->{$mode} = defined($3) ? "$3s" : "-"; } elsif ($mode eq "version") { $hash->{$mode} = "$_"; } elsif ($mode eq "command") { $hash->{$mode} = "$_"; } elsif ($mode eq "commandline") { $hash->{$mode} = "$_"; } elsif ($mode eq "expected") { $hash->{$mode} = "$_"; } else { die "Don't know what to do in mode $mode\n"; } } } sub parse_exit { my ($LOG, $hash) = @_; my $mode = ""; while (<$LOG>) { chomp; $mode = "exitcode", next if (/^### exit code:$/); $mode = "time", next if (/^### \/usr\/bin\/time -v:$/); $mode = "time_osx", next if (/^### \/usr\/bin\/time -l:$/); if ($mode eq "exitcode") { #/^(\d+)$/ or die "Unexpected exit code format\n"; /^(\d+)$/ or next; $hash->{$mode} = $1; if ($1 == 124) { $hash->{Result} = "TIMEOUT"; } } elsif ($mode eq "time") { if (/^\s+User time \(seconds\): (\d+\.\d+)$/) { $hash->{usertime} = $1; } elsif (/^\s+Elapsed \(wall clock\) time \(h:mm:ss or m:ss\): ((\d+:)?\d+:\d+\.\d+)$/) { $hash->{wallclock} = $1; } elsif (/^\s+Maximum resident set size \(kbytes\): (\d+)$/) { $hash->{maxmem} = "$1kb"; } elsif (/^SC_time:\s+(\d+\.\d+)$/) { $hash->{sc_time} = "$1"; defined($hash->{usertime}) and $hash->{overhead} = $hash->{usertime}/$1; } } elsif ($mode eq "time_osx") { if (/^\s+(\d+\.\d+)\s+real\s+(\d+\.\d+)\s+user\s+(\d+\.\d+)\s+sys$/) { $hash->{usertime} = $2; $hash->{wallclock} = $1; } elsif (/^\s+(\d+)\s+maximum resident set size$/) { $hash->{maxmem} = ($1/1024) . "kb"; } } else { die "Don't know what to do in mode $mode\n"; } } } sub parse_file { my ($file, $hash) = @_; open my $LOG, "<$file" or die "Logfile $file not found\n"; &parse_description($LOG, $hash); &parse_log($LOG, $hash); &parse_exit($LOG, $hash); close $LOG; } my %results = (); open RESULTS, "<$file" or die "File $file not found\n"; while () { chomp; next unless (/^LOGFILE:\s+(\S+)$/); print STDERR "Parsing $1\n"; defined($results{$1}) and die "Duplicate log file $1\n"; my $fname = $1; $results{$fname} = (); my $bm_name = $fname; if ($bm_name =~ /^.*\/results\.[^\/]+\/(\S+)\.[a-z]+_(\S+)_\d{4}-\d\d-\d\d_\d{6}_.{6}\.log$/) { $bm_name = ($2 eq "ALL_CLAIMS") ? $1 : "$1:$2"; } elsif ($bm_name =~ /^.*\/(tests|results\.[^\/]+)\/([^\/]+).*(\/[a-z]+\/([^\/_]+\d+|iriw[^\/_]+))_([a-z]+(.oepc)?).*_(\S+)_\d{4}-\d\d-\d\d_\d{6}_.{6}\.log$/) { $bm_name = ($7 eq "CLAIMS" || $7 eq "ALL_CLAIMS" || $7 eq "main.1") ? "$2$3($5)" : "$2$3($5):$7"; defined($6) and $bm_name =~ s/\)/) 1d/; } elsif ($bm_name =~ /^.*\/(tests|results\.[^\/]+)\/([^\/]+).*(\/[a-z]+\/([^\/_]+\d+|iriw[^\/_]+))\.goto_(\S+)_\d{4}-\d\d-\d\d_\d{6}_.{6}\.log$/) { ($config eq "cbmc") or die "Benchmark name only acceptable for cbmc\n"; my $mm = `grep memory-model $fname | sed 's/.*memory-model //' | awk '{ print \$1 }' | head -n 1`; chomp $mm; $bm_name = ($5 eq "CLAIMS" || $5 eq "ALL_CLAIMS" || $5 eq "main.1") ? "$2$3($mm)" : "$2$3($mm):$5"; defined($4) and $bm_name =~ s/\)/) 1d/; ($mm =~ /^(sc|tso|pso|rmo|power)(\.oepc)?$/) or die "Invalid memory model $mm\n"; } else { die "Don't know how to extract benchmark name from log file name $fname\n"; } $results{$fname}{Benchmark} = $bm_name; # default to ERROR $results{$fname}{Result} = "ERROR"; &parse_file($fname, \%{ $results{$fname} }); } close RESULTS; my %all_fields = (); foreach my $f (keys %results) { $all_fields{$_} = 1 foreach (keys %{ $results{$f} }); } use Text::CSV; my $csv = Text::CSV->new(); my @headings = sort keys %all_fields; unshift @headings, "Log File"; $csv->combine(@headings) or die "Error: ".$csv->error_input()."\n"; print $csv->string() . "\n"; foreach my $f (sort keys %results) { my @row = (); push @row, $f; foreach my $field (sort keys %all_fields) { push @row, defined($results{$f}{$field}) ? $results{$f}{$field} : ""; } $csv->combine(@row) or die "Error: ".$csv->error_input()."\n"; print $csv->string() . "\n"; } bmt-0.6/update.sh0000755000175000017500000001445712530560050013557 0ustar michaelmichael#!/bin/bash # # Copyright (c) 2011 Michael Tautschnig # 2015 Michael Tautschnig # # All rights reserved. Redistribution and use in source and binary forms, with # or without modification, are permitted provided that the following # conditions are met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # 3. All advertising materials mentioning features or use of this software # must display the following acknowledgement: # # This product includes software developed by Michael Tautschnig, # Queen Mary University of London # # 4. Neither the name of the University nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # # # THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS `AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE # DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # To be run in an unpacked CPROVER benchmark; updates the patch series with # respect to the original source, the known patches, and the current code base; # then rebuilds PKG.cprover-bm.tar.gz from the resulting cprover/ directory set -e die() { echo $1 exit 1 } checked_patch() { patch=$1 [ -f $patch ] || die "Patch file $patch not found" echo -n "Checking patch `basename $patch` ... " lc=`patch -f -p1 --dry-run < $patch 2>&1 | egrep -v "^patching file" | wc -l` if [ $lc -ne 0 ] ; then echo "clean application failed:" patch -f -p1 --dry-run < $patch || true die "Failed to apply patch `basename $patch`" fi echo "patch ok, applying" patch -p1 < $patch } usage() { cat < $patch_tmp ; then diff_exit_code=${PIPESTATUS[0]} if [ $diff_exit_code -ne 1 ] ; then cat $patch_tmp | egrep -v '^(-|\+|@| |diff -urN)' 1>&2 die "diff had error exit code $diff_exit_code -- check for binary files as listed above" fi if [ ! -d $PKG_NAME/cprover/patches ] ; then mkdir $PKG_NAME/cprover/patches fi if [ ! -f $PKG_NAME/cprover/patches/series ] ; then touch $PKG_NAME/cprover/patches/series fi patch_idx="`printf "%03d" \`ls $PKG_NAME/cprover/patches | wc -l\``" new_patch="`TMPDIR=$PKG_NAME/cprover/patches mktemp -t "$patch_idx-new_patch.XXXXXX"`" mv $patch_tmp $new_patch echo "`basename $new_patch`" >> $PKG_NAME/cprover/patches/series echo "Added patch $new_patch to record the following changes:" diffstat $new_patch fi tar czf `basename $BM_PKG` --exclude-vcs $PKG_NAME/cprover bmt-0.6/parse-cseq.pl0000644000175000017500000000467112530560050014333 0ustar michaelmichael#!/usr/bin/perl -w # # Copyright (c) 2011 Michael Tautschnig # 2015 Michael Tautschnig # # All rights reserved. Redistribution and use in source and binary forms, with # or without modification, are permitted provided that the following # conditions are met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # 3. All advertising materials mentioning features or use of this software # must display the following acknowledgement: # # This product includes software developed by Michael Tautschnig, # Queen Mary University of London # # 4. Neither the name of the University nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # # # THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS `AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE # DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # parse the CSeq tool specific part of a benchmark log file use strict; use warnings FATAL => qw(uninitialized); sub parse_log { my ($LOG, $hash) = @_; $hash->{Result} = "ERROR"; while (<$LOG>) { chomp; return 1 if (/^###############################################################################$/); if (/^Timeout: aborting command/) { $hash->{Result} = "TIMEOUT"; } elsif (/^.....(FALSE|TRUE)....$/) { # ignore UNKNOWN -> ERROR $hash->{Result} = ($1 eq "FALSE") ? "FAILED" : "SUCCESSFUL"; } } } return 1; bmt-0.6/parse-esbmc.pl0000644000175000017500000000457412530560050014473 0ustar michaelmichael#!/usr/bin/perl -w # # Copyright (c) 2011 Michael Tautschnig # 2015 Michael Tautschnig # # All rights reserved. Redistribution and use in source and binary forms, with # or without modification, are permitted provided that the following # conditions are met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # 3. All advertising materials mentioning features or use of this software # must display the following acknowledgement: # # This product includes software developed by Michael Tautschnig, # Queen Mary University of London # # 4. Neither the name of the University nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # # # THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS `AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE # DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # parse the CPROVER tool specific part of a benchmark log file use strict; use warnings FATAL => qw(uninitialized); sub parse_log { my ($LOG, $hash) = @_; $hash->{Result} = "ERROR"; while (<$LOG>) { chomp; return 1 if (/^###############################################################################$/); if (/^Timeout: aborting command/) { $hash->{Result} = "TIMEOUT"; } elsif (/^VERIFICATION\s+(\S+)$/) { $hash->{Result} = "$1"; } } } return 1; bmt-0.6/parse-loopfrog.pl0000644000175000017500000000461512530560050015225 0ustar michaelmichael#!/usr/bin/perl -w # # Copyright (c) 2011 Michael Tautschnig # 2015 Michael Tautschnig # # All rights reserved. Redistribution and use in source and binary forms, with # or without modification, are permitted provided that the following # conditions are met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # 3. All advertising materials mentioning features or use of this software # must display the following acknowledgement: # # This product includes software developed by Michael Tautschnig, # Queen Mary University of London # # 4. Neither the name of the University nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # # # THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS `AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE # DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # parse the CPROVER tool specific part of a benchmark log file use strict; use warnings FATAL => qw(uninitialized); sub parse_log { my ($LOG, $hash) = @_; $hash->{Result} = "ERROR"; while (<$LOG>) { chomp; return 1 if (/^###############################################################################$/); if (/^Too many iterations, giving up/) { $hash->{Result} = "TOO_MANY_ITERATIONS"; } elsif (/^VERIFICATION\s+(\S+)$/) { $hash->{Result} = "$1"; } } } return 1; bmt-0.6/make_jtl.pl0000755000175000017500000000651212530560050014055 0ustar michaelmichael#!/usr/bin/perl -w # # Copyright (c) 2011 Michael Tautschnig # 2015 Michael Tautschnig # # All rights reserved. Redistribution and use in source and binary forms, with # or without modification, are permitted provided that the following # conditions are met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # 3. All advertising materials mentioning features or use of this software # must display the following acknowledgement: # # This product includes software developed by Michael Tautschnig, # Queen Mary University of London # # 4. Neither the name of the University nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # # # THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS `AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE # DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # build a JMeter XML report use strict; use warnings FATAL => qw(uninitialized); use Date::Parse; sub usage { print <<"EOF"; Usage: $0 CSV where CSV is a comma-separated data file as built by make_csv.pl to build a JMeter XML file EOF } if (scalar(@ARGV) != 1) { usage; exit 1; } my $file = $ARGV[0]; shift @ARGV; open my $CSV, "<$file" or die "File $file not found\n"; print <<'EOF'; EOF my %globals = (); use Text::CSV; my $csv = Text::CSV->new(); my $arref = $csv->getline($CSV); defined($arref) or die "Failed to parse headers\n"; $csv->column_names(@$arref); my %col_map = ( "Benchmark" => "label", "Result" => "rm", "exitcode" => "rc", "date" => "ts", "usertime" => "t" ); while (my $row = $csv->getline_hr($CSV)) { foreach (qw(command timeout uname cpuinfo meminfo memlimit)) { defined($row->{$_}) or die "No $_ data in table\n"; defined($globals{$_}) or $globals{$_} = (); $globals{$_}{$row->{$_}} = 1; } print "{$c}) or die "No $c data in table\n"; my $val = $row->{$c}; if($c eq "date") { my $time = str2time($val); print $time*1000; } elsif($c eq "usertime") { print $val*1000; } else { print $val; } print "\""; } print "/>\n"; } print "\n\n"; close $CSV; bmt-0.6/parse-checkfence.pl0000644000175000017500000000445612530560050015457 0ustar michaelmichael#!/usr/bin/perl -w # # Copyright (c) 2011 Michael Tautschnig # 2015 Michael Tautschnig # # All rights reserved. Redistribution and use in source and binary forms, with # or without modification, are permitted provided that the following # conditions are met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # 3. All advertising materials mentioning features or use of this software # must display the following acknowledgement: # # This product includes software developed by Michael Tautschnig, # Queen Mary University of London # # 4. Neither the name of the University nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # # # THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS `AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE # DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # parse the Poirot specific part of a benchmark log file use strict; use warnings FATAL => qw(uninitialized); sub parse_log { my ($LOG, $hash) = @_; $hash->{Result} = "ERROR"; while (<$LOG>) { chomp; return 1 if (/^###############################################################################$/); if (/RESULT:\s+"[^"]+"\s+([^\.]+)\./) { $hash->{Result} = "$1"; } } } return 1; bmt-0.6/parse-satabs.pl0000644000175000017500000001305312530560050014647 0ustar michaelmichael#!/usr/bin/perl -w # # Copyright (c) 2011 Michael Tautschnig # 2015 Michael Tautschnig # # All rights reserved. Redistribution and use in source and binary forms, with # or without modification, are permitted provided that the following # conditions are met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # 3. All advertising materials mentioning features or use of this software # must display the following acknowledgement: # # This product includes software developed by Michael Tautschnig, # Queen Mary University of London # # 4. Neither the name of the University nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # # # THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS `AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE # DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # parse the CPROVER tool specific part of a benchmark log file use strict; use warnings FATAL => qw(uninitialized); sub parse_log { my ($LOG, $hash) = @_; $hash->{Result} = "ERROR"; $hash->{"Transition refinements"} = -1; $hash->{"Invalid states requiring more than 1 passive thread"} = 0; $hash->{"Spurious guard transitions requiring more than 1 passive thread"} = 0; $hash->{"Spurious assignment transitions requiring more than 1 passive thread"} = 0; while (<$LOG>) { chomp; return 1 if (/^###############################################################################$/); if (/^Too many iterations, giving up/) { $hash->{Result} = "TOO_MANY_ITERATIONS"; } elsif (/^Timeout: aborting command/) { $hash->{Result} = "TIMEOUT"; } elsif (/^VERIFICATION\s+(\S+)$/) { $hash->{Result} = "$1"; } elsif (/^Time: (\d+\.\d+) total, (\d+\.\d+) abstractor, (\d+\.\d+) model checker, (\d+\.\d+) simulator, (\d+\.\d+) refiner$/) { $hash->{"Time abstractor"} = $2; $hash->{"Time model checker"} = $3; $hash->{"Time simulator"} = $4; $hash->{"Time refiner"} = $5; } elsif (/^Time: (\d+\.\d+) total, (\d+\.\d+) abstractor, (\d+\.\d+) model checker, (\d+) simulator, (\d+\.\d+) refiner$/) { $hash->{"Time abstractor"} = $2; $hash->{"Time model checker"} = $3; $hash->{"Time simulator"} = $4; $hash->{"Time refiner"} = $5; } elsif (/^Time: (\d+\.\d+) total, (\d+\.\d+) abstractor, (\d+\.\d+) model checker, (\d+) simulator, (\d+) refiner$/) { $hash->{"Time abstractor"} = $2; $hash->{"Time model checker"} = $3; $hash->{"Time simulator"} = $4; $hash->{"Time refiner"} = $5; } elsif (/^Time: (\d+\.\d+) total, (\d+\.\d+) abstractor, (\d+\.\d+) model checker, (\d+\.\d+) simulator, (\d+) refiner$/) { $hash->{"Time abstractor"} = $2; $hash->{"Time model checker"} = $3; $hash->{"Time simulator"} = $4; $hash->{"Time refiner"} = $5; } elsif (/^Iterations: (\d+)$/) { $hash->{Iterations} = $1; $hash->{"Transition refinements"} += $1; } elsif (/^\s+shared: (\d+)$/) { $hash->{"Shared predicates"} = $1; } elsif (/^\s+local: (\d+)$/) { $hash->{"Local predicates"} = $1; } elsif (/^\s+mixed: (\d+)$/) { $hash->{"Mixed predicates"} = $1; } elsif (/^Transitions are not spurious$/) { --$hash->{"Transition refinements"}; } elsif (/^Broadcast assignment operations executed: (\d+)$/) { $hash->{"Broadcast assignment operations executed"} = $1; } elsif (/^Max number of slots used: (\d+)$/) { $hash->{"Max number of slots used"} = $1; } elsif (/^Non-broadcast assignment operations executed: (\d+)$/) { $hash->{"Non-broadcast assignment operations executed"} = $1; } elsif (/^Time spent in broadcast assignment operations: (\S+)$/) { $hash->{"Time spent in broadcast assignment operations"} = $1; } elsif (/^Time spent in non-broadcast assignment operations: (\S+)$/) { $hash->{"Time spent in non-broadcast assignment operations"} = $1; } elsif (/^Invalid states requiring more than 1 passive thread: (\d+)$/) { $hash->{"Invalid states requiring more than 1 passive thread"} = $1; } elsif (/^Spurious guard transitions requiring more than 1 passive thread: (\d+)$/) { $hash->{"Spurious guard transitions requiring more than 1 passive thread"} = $1; } elsif (/^Spurious assignment transitions requiring more than 1 passive thread: (\d+)$/) { $hash->{"Spurious assignment transitions requiring more than 1 passive thread"} = $1; } elsif (/^Total transition refinements: (\d+)$/) { $hash->{"Total transition refinements"} = $1; } } } return 1;