failover_test.go

package desync

import (
	"context"
	"crypto/rand"
	"errors"
	"sync"
	"testing"
	"time"

	"github.com/stretchr/testify/require"
	"golang.org/x/sync/errgroup"
)

func TestFailoverMissingChunk(t *testing.T) {
	s := &TestStore{}
	g := NewFailoverGroup(s)
	_, err := g.GetChunk(ChunkID{0})
	require.IsType(t, ChunkMissing{}, err)
}

func TestFailoverAllError(t *testing.T) {
	var failed = errors.New("failed")
	storeFail := &TestStore{
		GetChunkFunc: func(ChunkID) (*Chunk, error) { return nil, failed },
	}
	g := NewFailoverGroup(storeFail, storeFail)
	_, err := g.GetChunk(ChunkID{0})
	require.ErrorIs(t, err, failed)
}

func TestFailoverSimple(t *testing.T) {
	// Create two stores, one that always fails and one that works
	storeFail := &TestStore{
		GetChunkFunc: func(ChunkID) (*Chunk, error) { return nil, errors.New("failed") },
	}
	storeSucc := &TestStore{
		GetChunkFunc: func(ChunkID) (*Chunk, error) { return nil, nil },
	}

	// Group the two stores together, the failing ones first
	g := NewFailoverGroup(storeFail, storeFail, storeSucc)

	// Request a chunk, should succeed
	_, err := g.GetChunk(ChunkID{0})
	require.NoError(t, err)

	// Look inside the group to confirm we failed over to the last one
	require.Equal(t, 2, g.active)
}

func TestFailoverMutliple(t *testing.T) {
	// Create two stores, one that fails when x is 1 and the other fails when x is 0.
	//
	// mu holds x still for the duration of a whole GetChunk call. The group
	// tries each store at most once per call, so a switch landing between
	// those two attempts leaves the call having seen both stores fail, and it
	// returns an error - which is what the group documents it will do when
	// every store fails, not a bug. Without this the test asks for something
	// the implementation never promised, and fails whenever a goroutine is
	// descheduled mid-call: rarely on an idle machine, readily on a loaded or
	// emulated one.
	var (
		mu sync.RWMutex
		x  int
	)
	storeA := &TestStore{
		GetChunkFunc: func(id ChunkID) (*Chunk, error) {
			if x == 0 {
				return nil, nil
			}
			return nil, errors.New("failed")
		},
	}
	storeB := &TestStore{
		GetChunkFunc: func(id ChunkID) (*Chunk, error) {
			if x == 1 {
				return nil, nil
			}
			return nil, errors.New("failed")
		},
	}

	// Group the two stores together, the failing ones first
	g := NewFailoverGroup(storeA, storeB)

	var (
		ctx, cancel = context.WithTimeout(t.Context(), time.Second)
		eg, gCtx    = errgroup.WithContext(ctx)
		failOver    = time.Tick(10 * time.Millisecond)
	)
	defer cancel()

	// Run several goroutines querying the group in a tight loop
	for range 16 {
		eg.Go(func() error {
			var id ChunkID
			for {
				time.Sleep(time.Millisecond)
				select {
				case <-gCtx.Done():
					return nil
				default:
					rand.Read(id[:])
					mu.RLock()
					_, err := g.GetChunk(id)
					mu.RUnlock()
					if err != nil {
						return err
					}
				}
			}
		})
	}

	// Make the stores fail over every 10 ms
	eg.Go(func() error {
		for {
			select {
			case <-gCtx.Done(): // done running
				return nil
			case <-failOver: // switch over to the other store
				mu.Lock()
				x = (x + 1) % 2
				mu.Unlock()
			}
		}
	})

	require.NoError(t, eg.Wait())
}