tendermint/libs/pubsub/pubsub.go

// Package pubsub implements a pub-sub model with a single publisher (Server)
// and multiple subscribers (clients).
//
// Though you can have multiple publishers by sharing a pointer to a server or
// by giving the same channel to each publisher and publishing messages from
// that channel (fan-in).
//
// Clients subscribe for messages, which could be of any type, using a query.
// When some message is published, we match it with all queries. If there is a
// match, this message will be pushed to all clients, subscribed to that query.
// See query subpackage for our implementation.
//
// Example:
//
//     q, err := query.New("account.name='John'")
//     if err != nil {
//         return err
//     }
//     ctx, cancel := context.WithTimeout(context.Background(), 1 * time.Second)
//     defer cancel()
//     subscription, err := pubsub.Subscribe(ctx, "johns-transactions", q)
//     if err != nil {
//         return err
//     }
//
//     for {
//         select {
//         case msg <- subscription.Out():
//             // handle msg.Data() and msg.Events()
//         case <-subscription.Canceled():
//             return subscription.Err()
//         }
//     }
//
package pubsub

import (
	"context"
	"errors"
	"fmt"

	"github.com/tendermint/tendermint/abci/types"
	tmsync "github.com/tendermint/tendermint/internal/libs/sync"
	"github.com/tendermint/tendermint/libs/pubsub/query"
	"github.com/tendermint/tendermint/libs/service"
)

type operation int

const (
	sub operation = iota
	pub
	unsub
	shutdown
)

var (
	// ErrSubscriptionNotFound is returned when a client tries to unsubscribe
	// from not existing subscription.
	ErrSubscriptionNotFound = errors.New("subscription not found")

	// ErrAlreadySubscribed is returned when a client tries to subscribe twice or
	// more using the same query.
	ErrAlreadySubscribed = errors.New("already subscribed")
)

// Query defines an interface for a query to be used for subscribing. A query
// matches against a map of events. Each key in this map is a composite of the
// even type and an attribute key (e.g. "{eventType}.{eventAttrKey}") and the
// values are the event values that are contained under that relationship. This
// allows event types to repeat themselves with the same set of keys and
// different values.
type Query interface {
	Matches(events []types.Event) (bool, error)
	String() string
}

type UnsubscribeArgs struct {
	ID         string
	Subscriber string
	Query      Query
}

func (args UnsubscribeArgs) Validate() error {
	if args.Subscriber == "" {
		return errors.New("must specify a subscriber")
	}

	if args.ID == "" && args.Query == nil {
		return fmt.Errorf("subscription is not fully defined [subscriber=%q]", args.Subscriber)
	}

	return nil
}

type cmd struct {
	op operation

	// subscribe, unsubscribe
	query        Query
	subscription *Subscription
	clientID     string

	// publish
	msg    interface{}
	events []types.Event
}

// Server allows clients to subscribe/unsubscribe for messages, publishing
// messages with or without events, and manages internal state.
type Server struct {
	service.BaseService

	cmds    chan cmd
	cmdsCap int

	// check if we have subscription before
	// subscribing or unsubscribing
	mtx tmsync.RWMutex

	// subscriber -> [query->id (string) OR id->query (string))],
	//   track connections both by ID (new) and query (legacy) to
	//   avoid breaking the interface.
	subscriptions map[string]map[string]string
}

// Option sets a parameter for the server.
type Option func(*Server)

// NewServer returns a new server. See the commentary on the Option functions
// for a detailed description of how to configure buffering. If no options are
// provided, the resulting server's queue is unbuffered.
func NewServer(options ...Option) *Server {
	s := &Server{
		subscriptions: make(map[string]map[string]string),
	}
	s.BaseService = *service.NewBaseService(nil, "PubSub", s)

	for _, option := range options {
		option(s)
	}

	// if BufferCapacity option was not set, the channel is unbuffered
	s.cmds = make(chan cmd, s.cmdsCap)

	return s
}

// BufferCapacity allows you to specify capacity for the internal server's
// queue. Since the server, given Y subscribers, could only process X messages,
// this option could be used to survive spikes (e.g. high amount of
// transactions during peak hours).
func BufferCapacity(cap int) Option {
	return func(s *Server) {
		if cap > 0 {
			s.cmdsCap = cap
		}
	}
}

// BufferCapacity returns capacity of the internal server's queue.
func (s *Server) BufferCapacity() int {
	return s.cmdsCap
}

// Subscribe creates a subscription for the given client.
//
// An error will be returned to the caller if the context is canceled or if
// subscription already exist for pair clientID and query.
//
// outCapacity can be used to set a capacity for Subscription#Out channel (1 by
// default). Panics if outCapacity is less than or equal to zero. If you want
// an unbuffered channel, use SubscribeUnbuffered.
func (s *Server) Subscribe(
	ctx context.Context,
	clientID string,
	query Query,
	outCapacity ...int) (*Subscription, error) {
	outCap := 1
	if len(outCapacity) > 0 {
		if outCapacity[0] <= 0 {
			panic("Negative or zero capacity. Use SubscribeUnbuffered if you want an unbuffered channel")
		}
		outCap = outCapacity[0]
	}

	return s.subscribe(ctx, clientID, query, outCap)
}

// SubscribeUnbuffered does the same as Subscribe, except it returns a
// subscription with unbuffered channel. Use with caution as it can freeze the
// server.
func (s *Server) SubscribeUnbuffered(ctx context.Context, clientID string, query Query) (*Subscription, error) {
	return s.subscribe(ctx, clientID, query, 0)
}

func (s *Server) subscribe(ctx context.Context, clientID string, query Query, outCapacity int) (*Subscription, error) {
	s.mtx.RLock()
	clientSubscriptions, ok := s.subscriptions[clientID]
	if ok {
		_, ok = clientSubscriptions[query.String()]
	}
	s.mtx.RUnlock()
	if ok {
		return nil, ErrAlreadySubscribed
	}

	subscription := NewSubscription(outCapacity)
	select {
	case s.cmds <- cmd{op: sub, clientID: clientID, query: query, subscription: subscription}:
		s.mtx.Lock()
		if _, ok = s.subscriptions[clientID]; !ok {
			s.subscriptions[clientID] = make(map[string]string)
		}
		s.subscriptions[clientID][query.String()] = subscription.id
		s.subscriptions[clientID][subscription.id] = query.String()
		s.mtx.Unlock()
		return subscription, nil
	case <-ctx.Done():
		return nil, ctx.Err()
	case <-s.Quit():
		return nil, nil
	}
}

// Unsubscribe removes the subscription on the given query. An error will be
// returned to the caller if the context is canceled or if subscription does
// not exist.
func (s *Server) Unsubscribe(ctx context.Context, args UnsubscribeArgs) error {
	if err := args.Validate(); err != nil {
		return err
	}
	var qs string

	if args.Query != nil {
		qs = args.Query.String()
	}

	clientSubscriptions, err := func() (map[string]string, error) {
		s.mtx.RLock()
		defer s.mtx.RUnlock()

		clientSubscriptions, ok := s.subscriptions[args.Subscriber]
		if args.ID != "" {
			qs, ok = clientSubscriptions[args.ID]

			if ok && args.Query == nil {
				var err error
				args.Query, err = query.New(qs)
				if err != nil {
					return nil, err
				}
			}
		} else if qs != "" {
			args.ID, ok = clientSubscriptions[qs]
		}

		if !ok {
			return nil, ErrSubscriptionNotFound
		}

		return clientSubscriptions, nil
	}()

	if err != nil {
		return err
	}

	select {
	case s.cmds <- cmd{op: unsub, clientID: args.Subscriber, query: args.Query, subscription: &Subscription{id: args.ID}}:
		s.mtx.Lock()
		defer s.mtx.Unlock()

		delete(clientSubscriptions, args.ID)
		delete(clientSubscriptions, qs)

		if len(clientSubscriptions) == 0 {
			delete(s.subscriptions, args.Subscriber)
		}
		return nil
	case <-ctx.Done():
		return ctx.Err()
	case <-s.Quit():
		return nil
	}
}

// UnsubscribeAll removes all client subscriptions. An error will be returned
// to the caller if the context is canceled or if subscription does not exist.
func (s *Server) UnsubscribeAll(ctx context.Context, clientID string) error {
	s.mtx.RLock()
	_, ok := s.subscriptions[clientID]
	s.mtx.RUnlock()
	if !ok {
		return ErrSubscriptionNotFound
	}

	select {
	case s.cmds <- cmd{op: unsub, clientID: clientID}:
		s.mtx.Lock()
		defer s.mtx.Unlock()

		delete(s.subscriptions, clientID)

		return nil
	case <-ctx.Done():
		return ctx.Err()
	case <-s.Quit():
		return nil
	}
}

// NumClients returns the number of clients.
func (s *Server) NumClients() int {
	s.mtx.RLock()
	defer s.mtx.RUnlock()
	return len(s.subscriptions)
}

// NumClientSubscriptions returns the number of subscriptions the client has.
func (s *Server) NumClientSubscriptions(clientID string) int {
	s.mtx.RLock()
	defer s.mtx.RUnlock()
	return len(s.subscriptions[clientID]) / 2
}

// Publish publishes the given message. An error will be returned to the caller
// if the context is canceled.
func (s *Server) Publish(ctx context.Context, msg interface{}) error {
	return s.PublishWithEvents(ctx, msg, []types.Event{})
}

// PublishWithEvents publishes the given message with the set of events. The set
// is matched with clients queries. If there is a match, the message is sent to
// the client.
func (s *Server) PublishWithEvents(ctx context.Context, msg interface{}, events []types.Event) error {
	select {
	case s.cmds <- cmd{op: pub, msg: msg, events: events}:
		return nil
	case <-ctx.Done():
		return ctx.Err()
	case <-s.Quit():
		return nil
	}
}

// OnStop implements Service.OnStop by shutting down the server.
func (s *Server) OnStop() {
	s.cmds <- cmd{op: shutdown}
}

// NOTE: not goroutine safe
type state struct {
	// query string -> client -> subscription
	subscriptions map[string]map[string]*Subscription
	// query string -> queryPlusRefCount
	queries map[string]*queryPlusRefCount
}

// queryPlusRefCount holds a pointer to a query and reference counter. When
// refCount is zero, query will be removed.
type queryPlusRefCount struct {
	q        Query
	refCount int
}

// OnStart implements Service.OnStart by starting the server.
func (s *Server) OnStart() error {
	go s.loop(state{
		subscriptions: make(map[string]map[string]*Subscription),
		queries:       make(map[string]*queryPlusRefCount),
	})
	return nil
}

// OnReset implements Service.OnReset
func (s *Server) OnReset() error {
	return nil
}

func (s *Server) loop(state state) {
loop:
	for cmd := range s.cmds {
		switch cmd.op {
		case unsub:
			if cmd.query != nil {
				state.remove(cmd.clientID, cmd.query.String(), cmd.subscription.id, ErrUnsubscribed)
			} else {
				state.removeClient(cmd.clientID, ErrUnsubscribed)
			}
		case shutdown:
			state.removeAll(nil)
			break loop
		case sub:
			state.add(cmd.clientID, cmd.query, cmd.subscription)
		case pub:
			if err := state.send(cmd.msg, cmd.events); err != nil {
				s.Logger.Error("Error querying for events", "err", err)
			}
		}
	}
}

func (state *state) add(clientID string, q Query, subscription *Subscription) {
	qStr := q.String()

	// initialize subscription for this client per query if needed
	if _, ok := state.subscriptions[qStr]; !ok {
		state.subscriptions[qStr] = make(map[string]*Subscription)
	}

	if _, ok := state.subscriptions[subscription.id]; !ok {
		state.subscriptions[subscription.id] = make(map[string]*Subscription)
	}

	// create subscription
	state.subscriptions[qStr][clientID] = subscription
	state.subscriptions[subscription.id][clientID] = subscription

	// initialize query if needed
	if _, ok := state.queries[qStr]; !ok {
		state.queries[qStr] = &queryPlusRefCount{q: q, refCount: 0}
	}
	// increment reference counter
	state.queries[qStr].refCount++
}

func (state *state) remove(clientID string, qStr, id string, reason error) {
	clientSubscriptions, ok := state.subscriptions[qStr]
	if !ok {
		return
	}

	subscription, ok := clientSubscriptions[clientID]
	if !ok {
		return
	}

	subscription.cancel(reason)

	// remove client from query map.
	// if query has no other clients subscribed, remove it.
	delete(state.subscriptions[qStr], clientID)
	delete(state.subscriptions[id], clientID)
	if len(state.subscriptions[qStr]) == 0 {
		delete(state.subscriptions, qStr)
	}

	// decrease ref counter in queries
	if ref, ok := state.queries[qStr]; ok {
		ref.refCount--
		if ref.refCount == 0 {
			// remove the query if nobody else is using it
			delete(state.queries, qStr)
		}
	}
}

func (state *state) removeClient(clientID string, reason error) {
	seen := map[string]struct{}{}
	for qStr, clientSubscriptions := range state.subscriptions {
		if sub, ok := clientSubscriptions[clientID]; ok {
			if _, ok = seen[sub.id]; ok {
				// all subscriptions are double indexed by ID and query, only
				// process them once.
				continue
			}
			state.remove(clientID, qStr, sub.id, reason)
			seen[sub.id] = struct{}{}
		}
	}
}

func (state *state) removeAll(reason error) {
	for qStr, clientSubscriptions := range state.subscriptions {
		sub, ok := clientSubscriptions[qStr]
		if !ok || ok && sub.id == qStr {
			// all subscriptions are double indexed by ID and query, only
			// process them once.
			continue
		}

		for clientID := range clientSubscriptions {
			state.remove(clientID, qStr, sub.id, reason)
		}
	}
}

func (state *state) send(msg interface{}, events []types.Event) error {
	for qStr, clientSubscriptions := range state.subscriptions {
		if sub, ok := clientSubscriptions[qStr]; ok && sub.id == qStr {
			continue
		}
		var q Query
		if qi, ok := state.queries[qStr]; ok {
			q = qi.q
		} else {
			continue
		}

		match, err := q.Matches(events)
		if err != nil {
			return fmt.Errorf("failed to match against query %s: %w", q.String(), err)
		}

		if match {
			for clientID, subscription := range clientSubscriptions {
				if cap(subscription.out) == 0 {
					// block on unbuffered channel
					subscription.out <- NewMessage(subscription.id, msg, events)
				} else {
					// don't block on buffered channels
					select {
					case subscription.out <- NewMessage(subscription.id, msg, events):
					default:
						state.remove(clientID, qStr, subscription.id, ErrOutOfCapacity)
					}
				}
			}
		}
	}

	return nil
}