Implement benchmark framework for gpython along with a couple of benchmarks (#51)

This reworks the existing test framework so it can be used for
benchmarks too.  Note that it now uses the subtest framework which
makes the test output much neater.

To run the benchmarks use

    cd vm
    go test -v -run XXX -bench .

Fixes #50

Author: ncw

pytest/pytest.go

@@ -18,8 +18,8 @@ import (
 	"github.com/go-python/gpython/vm"
 )
 
-// Run the code in str
-func Run(t *testing.T, prog string) {
+// Compile the program in the file prog to code in the module that is returned
+func compileProgram(t testing.TB, prog string) (*py.Module, *py.Code) {
 	f, err := os.Open(prog)
 	if err != nil {
 		t.Fatalf("%s: Open failed: %v", prog, err)
@@ -43,55 +43,80 @@ func Run(t *testing.T, prog string) {
 	code := obj.(*py.Code)
 	module := py.NewModule("__main__", "", nil, nil)
 	module.Globals["__file__"] = py.String(prog)
+	return module, code
+}
 
-	_, err = vm.Run(module.Globals, module.Globals, code, nil)
+// Run the code in the module
+func run(t testing.TB, module *py.Module, code *py.Code) {
+	_, err := vm.Run(module.Globals, module.Globals, code, nil)
 	if err != nil {
 		if wantErr, ok := module.Globals["err"]; ok {
 			wantErrObj, ok := wantErr.(py.Object)
 			if !ok {
-				t.Fatalf("%s: want err is not py.Object: %#v", prog, wantErr)
+				t.Fatalf("want err is not py.Object: %#v", wantErr)
 			}
 			gotExc, ok := err.(py.ExceptionInfo)
 			if !ok {
-				t.Fatalf("%s: got err is not ExceptionInfo: %#v", prog, err)
+				t.Fatalf("got err is not ExceptionInfo: %#v", err)
 			}
 			if gotExc.Value.Type() != wantErrObj.Type() {
-				t.Fatalf("%s: Want exception %v got %v", prog, wantErrObj, gotExc.Value)
+				t.Fatalf("Want exception %v got %v", wantErrObj, gotExc.Value)
 			}
-			t.Logf("%s: matched exception", prog)
+			// t.Logf("matched exception")
 			return
 		} else {
 			py.TracebackDump(err)
-			t.Fatalf("%s: Run failed: %v at %q", prog, err, module.Globals["doc"])
+			t.Fatalf("Run failed: %v at %q", err, module.Globals["doc"])
 		}
 	}
 
 	// t.Logf("%s: Return = %v", prog, res)
 	if doc, ok := module.Globals["doc"]; ok {
 		if docStr, ok := doc.(py.String); ok {
 			if string(docStr) != "finished" {
-				t.Fatalf("%s: Didn't finish at %q", prog, docStr)
+				t.Fatalf("Didn't finish at %q", docStr)
 			}
 		} else {
-			t.Fatalf("%s: Set doc variable to non string: %#v", prog, doc)
+			t.Fatalf("Set doc variable to non string: %#v", doc)
 		}
 	} else {
-		t.Fatalf("%s: Didn't set doc variable at all", prog)
+		t.Fatalf("Didn't set doc variable at all")
 	}
 }
 
-// Runs the tests in the directory passed in
-func RunTests(t *testing.T, testDir string) {
+// find the python files in the directory passed in
+func findFiles(t testing.TB, testDir string) (names []string) {
 	files, err := ioutil.ReadDir(testDir)
 	if err != nil {
 		t.Fatalf("ReadDir failed: %v", err)
 	}
 	for _, f := range files {
 		name := f.Name()
 		if !strings.HasPrefix(name, "lib") && strings.HasSuffix(name, ".py") {
-			name := path.Join(testDir, name)
-			t.Logf("%s: Running", name)
-			Run(t, name)
+			names = append(names, name)
 		}
 	}
+	return names
+}
+
+// RunTests runs the tests in the directory passed in
+func RunTests(t *testing.T, testDir string) {
+	for _, name := range findFiles(t, testDir) {
+		t.Run(name, func(t *testing.T) {
+			module, code := compileProgram(t, path.Join(testDir, name))
+			run(t, module, code)
+		})
+	}
+}
+
+// RunBenchmarks runs the benchmarks in the directory passed in
+func RunBenchmarks(b *testing.B, testDir string) {
+	for _, name := range findFiles(b, testDir) {
+		module, code := compileProgram(b, path.Join(testDir, name))
+		b.Run(name, func(b *testing.B) {
+			for i := 0; i < b.N; i++ {
+				run(b, module, code)
+			}
+		})
+	}
 }

vm/benchmarks/fib.py

@@ -0,0 +1,15 @@
+# Copyright 2019 The go-python Authors.  All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+# Benchmark adapted from https://github.com/d5/tengobench/
+doc="fib recursion test"
+def fib(n):
+    if n == 0:
+        return 0
+    elif n == 1:
+        return 1
+    return fib(n - 2) + fib(n - 1)
+
+fib(25)
+doc="finished"

vm/benchmarks/fibtc.py

@@ -0,0 +1,15 @@
+# Copyright 2019 The go-python Authors.  All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+# Benchmark adapted from https://github.com/d5/tengobench/
+doc="fib tail call recursion test"
+def fib(n, a, b):
+    if n == 0:
+        return a
+    elif n == 1:
+        return b
+    return fib(n-1, b, a+b)
+
+fib(35, 0, 1)
+doc="finished"

vm/vm_test.go

@@ -13,3 +13,7 @@ import (
 func TestVm(t *testing.T) {
 	pytest.RunTests(t, "tests")
 }
+
+func BenchmarkVM(b *testing.B) {
+	pytest.RunBenchmarks(b, "benchmarks")
+}