Hashicorp Plugin System Design and Implementation

Hashicorp Plugin Abstract Behavior

In Hashicorp plugin system, main-service exec plugin-service and communicates it with RPC
Fig. 1. main-service exec plugin-service and communicates it with RPC in the plugin system

Hashicorp Plugin Design

type Plugin interface {
Server(*MuxBroker) (interface{}, error)
Client(*MuxBroker, *rpc.Client) (interface{}, error)
}
type Greeter interface {
Greet() string
}
import (
"net/rpc"
)
// GreeterRPC is the client of the Greeter service
type GreeterRPC struct {
client *rpc.Client
}
func (g *GreeterRPC) Greet() string {
var resp string
err := g.client.Call("Plugin.Greet", new(interface{}), &resp)
if err != nil {
panic(err)
}
return resp
}
// GreeterRPCServer is the server of the Greeter service
type GreeterRPCServer struct {
Impl Greeter
}
func (s *GreeterRPCServer) Greet(args interface{}, resp *string) error {
*resp = s.Impl.Greet()
return nil
}
// GreeterPlugin is the plugin which enables the user to use the Greeter service
type GreeterPlugin struct {
Impl Greeter
}
func (p *GreeterPlugin) Server(*powerstrip.MuxBroker) (interface{}, error) {
return &GreeterRPCServer{Impl: p.Impl}, nil
}
func (GreeterPlugin) Client(b *powerstrip.MuxBroker, c *rpc.Client) (interface{}, error) {
return &GreeterRPC{client: c}, nil
}
Fig. 2. Service is dependency-injected to Application
Fig. 3. Application exec service binary and communicate with it

Hashicorp Plugin Implementation

type GreeterHello struct{}func (g *GreeterHello) Greet() string {
return "Hello!"
}
func main() {
greeter := &GreeterHello{}
var pluginMap = map[string]powerstrip.Plugin{
"greeter": &common.GreeterPlugin{Impl: greeter},
}
powerstrip.Serve(&powerstrip.ServeConfig{
Plugins: pluginMap,
})
}
var pluginMap = map[string]powerstrip.Plugin{
"greeter": &common.GreeterPlugin{},
}
func main() {
client := powerstrip.NewClient(&powerstrip.ClientConfig{
Plugins: pluginMap,
Cmd: exec.Command("./plugin/greeter"),
})
defer client.Kill()
rpcClient, err := client.Protocol()
if err != nil {
log.Fatal(err)
}
raw, err := rpcClient.Dispense("greeter")
if err != nil {
log.Fatal(err)
}
greeter := raw.(common.Greeter)
fmt.Println(greeter.Greet())
}

How client process create server process and connects to it

func (c *Client) Protocol() (ClientProtocol, error) {
_, err := c.Start()
...
c.proto, err = newRPCClient(c)
...
return c.proto, nil
}
func newRPCClient(c *Client) (*RPCClient, error) {
conn, err := net.Dial(c.addr.Network(), c.addr.String())
...
}
func (c *Client) Start() (net.Addr, error) {
cmdStdout, err := cmd.StdoutPipe()
...
cmdStderr, err := cmd.StderrPipe()
...
err = cmd.Start()
...
c.proc = cmd.Process
c.doneCtx, c.ctxCancel = context.WithCancel(context.Background()) // when server process exit, client updates its flag
c.clientWg.Add(1)
go func() {
defer c.ctxCancel()
defer c.clientWg.Done()
...
err := cmd.Wait()
...
c.exited = true
}()
// client reads the character printed from stdout of server process
linesCh := make(chan string)
c.clientWg.Add(1)
go func() {
defer c.clientWg.Done()
defer close(linesCh)

sc := bufio.NewScanner(cmdStdout)
for sc.Scan() {
linesCh <- sc.Text()
}
}()
select {
case <-timeout:
// return with error
case <-c.doneCtx.Done():
// return with error
case line := <-linesCh:
// get the server address `addr`
}
c.addr = addr
return addr, nil
}
func Serve(opts *ServeConfig) {
...
lis, err := serverListener()
...
server := &RPCServer{
Plugins: opts.Plugins,
Stdout: stdoutReader,
Stderr: stderrReader,
DoneCh: doneCh,
}
...
// prints "Protocol|Address" format characters to stdout
fmt.Printf("%s|%s\n",
lis.Addr().Network(),
lis.Addr().String())
os.Stdout.Sync()
...
go server.Serve(lis)
}
func serverListener() (net.Listener, error) {
tf, err := ioutil.TempFile("", "plugin")
...
path := tf.Name()
...
l, err := net.Listen("unix", path)
...
}
func (c *Client) Start() (net.Addr, error) {
...
select {
case <-timeout:
// return with error
case <-c.doneCtx.Done():
// return with error
case line := <-linesCh:
line = strings.TrimSpace(line)
parts := strings.SplitN(line, "|", 2)
if len(parts) < 2 {
return nil, fmt.Errorf("", line)
}
switch parts[0] {
case "tcp":
addr, err = net.ResolveTCPAddr("tcp", parts[1])
case "unix":
addr, err = net.ResolveUnixAddr("unix", parts[1])
default:
err = fmt.Errorf("Unknown address type: %s", parts[0])
}
}
}

How Client Select Plugin and Call the Method

func (c *RPCClient) Dispense(name string) (interface{}, error) {
p, ok := c.plugins[name]
if !ok {
return nil, fmt.Errorf("unknown plugin type: %s", name)
}
var id uint32
if err := c.control.Call(
"Dispenser.Dispense", name, &id); err != nil {
return nil, err
}
conn, err := c.broker.Dial(id)
if err != nil {
return nil, err
}
return p.Client(c.broker, rpc.NewClient(conn))
}
func (s *RPCServer) Serve(lis net.Listener) {
for {
conn, err := lis.Accept()
...
go s.ServeConn(conn)
}
}
func (s *RPCServer) ServeConn(conn io.ReadWriteCloser) {
...
server := rpc.NewServer()
...
server.RegisterName("Dispenser", &dispenseServer{
broker: broker,
plugins: s.Plugins,
})
server.ServeConn(control)
}
type dispenseServer struct {
broker *MuxBroker
plugins map[string]Plugin
}
func (d *dispenseServer) Dispense(name string, response *uint32) error {
p, ok := d.plugins[name]
...
impl, err := p.Server(d.broker)
...
id := d.broker.NextId()
*response = id
...
go func() {
conn, err := d.broker.Accept(id)
...
serve(conn, "Plugin", impl)
}()
return nil
}
func serve(conn io.ReadWriteCloser, name string, v interface{}) {
server := rpc.NewServer()
err := server.RegisterName(name, v)
...
server.ServeConn(conn)
}

Hashicorp Plugin Behavior in Terraform

func main() {
tfsdk.Serve(context.Background(), hashicups.New, tfsdk.ServeOpts{
Name: "hashicups",
})
}
func Serve(ctx context.Context, factory func() Provider, opts ServeOpts) error {
return tf6server.Serve(opts.Name, func() tfprotov6.ProviderServer {
return &server{
p: factory(),
}
})
}
func Serve(name string, serverFactory func() tfprotov6.ProviderServer, opts ...ServeOpt) error {
serveConfig := &plugin.ServeConfig{
...
Plugins: plugin.PluginSet{
"provider": &GRPCProviderPlugin{
GRPCProvider: serverFactory,
},
},
GRPCServer: plugin.DefaultGRPCServer,
}
...
plugin.Serve(serveConfig)
return nil
}

ETC

Testing Client

func TestClient(t *testing.T) {
proc := helperProcess("mock")
c := NewClient(&ClientConfig{
Cmd: proc,
Plugins: testPluginMap,
})
defer c.Kill()
addr, err := c.Start()
...
}
func helperProcess(s ...string) *exec.Cmd {
cs := []string{"-test.run=TestHelperProcess", "--"}
cs = append(cs, s...)
env := []string{
"GO_WANT_HELPER_PROCESS=1",
}
cmd := exec.Command(os.Args[0], cs...)
cmd.Env = append(env, os.Environ()...)
return cmd
}
func TestHelperProcess(t *testing.T) {
if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" {
return
}
...
cmd, args := args[0], args[1:]
switch cmd {
case "mock":
fmt.Printf("tcp|:1234\n")
<-make(chan int)
...
}

Software/Blockchain Engineer, https://github.com/zeroFruit

Love podcasts or audiobooks? Learn on the go with our new app.

Recommended from Medium

[Announcement] Claim NEER Reward, Upcoming Events & Roadmap

Validation — HackTheBox

Day 11 of 31-Day May LeetCode Challenge

stuff you should probably know before you walk into an interview for a big name tech company

A Beginner’s Guide to Web Development

Spring Web MVC framework

Google Coding Interview Questions Part: #G1 — Extended Analysis

How to turn your video meetings into something much more with Symbl, without writing any code

A zoom meeting displayed on a laptop from a home setting

Get the Medium app

A button that says 'Download on the App Store', and if clicked it will lead you to the iOS App store
A button that says 'Get it on, Google Play', and if clicked it will lead you to the Google Play store
zeroFruit

zeroFruit

Software/Blockchain Engineer, https://github.com/zeroFruit

More from Medium

Building robust platform for containers part 1 — Terraform, EKS & Cillium

High level design of the EKS cluster

Devtron: Open-Source Software Delivery Workflow for K8s

Reduce AWS cost with a Graviton EKS cluster

Cloud Native Journey Part 2: Technical Adventure