help for count token about tiktoken-go HOT 1 CLOSED

const tokenizerEncoding = "cl100k_base"

func countTokens(prompt string) (int, error) {
	enc, err := tiktoken.GetEncoding(tokenizerEncoding)
	if err != nil {
		return 0, err
	}

	tokens := enc.Encode(prompt, nil, nil)

	return len(tokens), nil
}

// main_test.go
func TestCountTokens(t *testing.T) {
	const prompt = "hello world!你好，世界！"
	const want = 10

	got, err := countTokens(prompt)
	if err != nil {
		t.Error(err)
	}

	if got != want {
		t.Errorf("got %v want %v", got, want)
	}
}