Implement failpoint counter

README.md

@@ -86,6 +86,12 @@ List the failpoints,
 $ curl http://127.0.0.1:1234/SomeFuncString=return("hello")
 ```
 
+Retrieve the execution count of a failpoint,
+
+```sh
+$curl http://127.0.0.1:1234/SomeFuncString/count -XGET
+```
+
 Deactivate a failpoint,
 
 ```sh

doc/design.md

@@ -69,6 +69,16 @@ Similarly, you can set multiple failpoints using endpoint `/failpoints`,
 curl http://127.0.0.1:22381/failpoints -X PUT -d'failpoint1=return("hello");failpoint2=sleep(10)'
 ```
 
+You can get the execution count of a failpoint in the dynamic way,
+```
+$curl http://127.0.0.1:1234/SomeFuncString/count -XGET
+```
+
+To deactivate a failpoint,
+```
+$ curl http://127.0.0.1:1234/SomeFuncString -XDELETE
+```
+
 #### 3.2 Unit test
 Assuming there is a function with a failpoint something like below,
 ```

runtime/http.go

@@ -15,11 +15,13 @@
 package runtime
 
 import (
+	"errors"
 	"fmt"
 	"io"
 	"net"
 	"net/http"
 	"sort"
+	"strconv"
 	"strings"
 )
 
@@ -87,12 +89,24 @@ func (*httpHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			sort.Strings(fps)
 			lines := make([]string, len(fps))
 			for i := range lines {
-				s, _ := status(fps[i])
+				s, _, _ := status(fps[i])
 				lines[i] = fps[i] + "=" + s
 			}
 			w.Write([]byte(strings.Join(lines, "\n") + "\n"))
+		} else if strings.HasSuffix(key, "/count") {
+			fp := key[:len(key)-len("/count")]
+			_, count, err := status(fp)
+			if err != nil {
+				if errors.Is(err, ErrNoExist) {
+					http.Error(w, "failed to GET: "+err.Error(), http.StatusNotFound)
+				} else {
+					http.Error(w, "failed to GET: "+err.Error(), http.StatusInternalServerError)
+				}
+				return
+			}
+			w.Write([]byte(strconv.Itoa(count)))
 		} else {
-			status, err := status(key)
+			status, _, err := status(key)
 			if err != nil {
 				http.Error(w, "failed to GET: "+err.Error(), http.StatusNotFound)
 			}

runtime/runtime.go

@@ -112,25 +112,25 @@ func disable(name string) error {
 	return nil
 }
 
-// Status gives the current setting for the failpoint
-func Status(failpath string) (string, error) {
+// Status gives the current setting and execution count for the failpoint
+func Status(failpath string) (string, int, error) {
 	failpointsMu.Lock()
 	defer failpointsMu.Unlock()
 	return status(failpath)
 }
 
-func status(failpath string) (string, error) {
+func status(failpath string) (string, int, error) {
 	fp := failpoints[failpath]
 	if fp == nil {
-		return "", ErrNoExist
+		return "", 0, ErrNoExist
 	}
 
 	t := fp.t
 	if t == nil {
-		return "", ErrDisabled
+		return "", 0, ErrDisabled
 	}
 
-	return t.desc, nil
+	return t.desc, t.counter, nil
 }
 
 func List() []string {

runtime/terms.go

@@ -41,6 +41,8 @@ type terms struct {
 
 	// mu protects the state of the terms chain
 	mu sync.Mutex
+	// tracks executions count of terms that are actually evaluated
+	counter int
 }
 
 // term is an executable unit of the failpoint terms chain
@@ -102,6 +104,7 @@ func (t *terms) eval() interface{} {
 	defer t.mu.Unlock()
 	for _, term := range t.chain {
 		if term.mods.allow() {
+			t.counter++
 			return term.do()
 		}
 	}

runtime/terms_test.go

@@ -71,3 +71,44 @@ func TestTermsTypes(t *testing.T) {
 		}
 	}
 }
+
+func TestTermsCounter(t *testing.T) {
+	tests := []struct {
+		failpointTerm    string
+		runAfterEnabling int
+		wantCount        int
+	}{
+		{
+			failpointTerm:    `10*sleep(10)->1*return("abc")`,
+			runAfterEnabling: 12,
+			// Note the chain of terms is allowed to be executed 11 times at most,
+			// including 10 times for the first term `10*sleep(10)` and 1 time for
+			// the second term `1*return("abc")`. So it's only evaluated 11 times
+			// even it's triggered 12 times.
+			wantCount: 11,
+		},
+		{
+			failpointTerm:    `10*sleep(10)->1*return("abc")`,
+			runAfterEnabling: 3,
+			wantCount:        3,
+		},
+		{
+			failpointTerm:    `10*sleep(10)->1*return("abc")`,
+			runAfterEnabling: 0,
+			wantCount:        0,
+		},
+	}
+	for _, tt := range tests {
+		ter, err := newTerms("test", tt.failpointTerm)
+		if err != nil {
+			t.Fatal(err)
+		}
+		for i := 0; i < tt.runAfterEnabling; i++ {
+			_ = ter.eval()
+		}
+
+		if ter.counter != tt.wantCount {
+			t.Errorf("counter is not properly incremented, got: %d, want: %d", ter.counter, tt.wantCount)
+		}
+	}
+}