go get github.com/samber/go-gpt-3-encoder

import tokenizer "github.com/samber/go-gpt-3-encoder"

encoder, err := tokenizer.NewEncoder()
if err != nil {
    log.Fatal(err)
}

str := "This is an example sentence to try encoding out on!"

encoded, err := encoder.Encode(str)
if err != nil {
    log.Fatal(err)
}

fmt.Println("We can look at each token and what it represents:")
for _, token := range encoded {
    fmt.Printf("%d -- %s\n", token, encoder.Decode([]int{token}))
}

decoded := encoder.Decode(encoded)
fmt.Printf("We can decode it back into: %s\n", decoded)

Contributors

Stargazers

Forkers

casistack danielswe88 jessykalycia sandwich-go coggsfl folospace impolqris pullpo-io code-hex munisystem ganymedenil nicexai icyxp seriozhakorneev gargantuax bradfair diefisheye navicstein yexingzhe54 synaxis

go: github.com/samber/[email protected]: parsing go.mod:
        module declares its path as: gpt-3-encoder
                but was required as: github.com/samber/go-gpt-3-encoder