first commit
All checks were successful
continuous-integration/drone/push Build is passing

This commit is contained in:
andrey1s
2021-04-26 17:13:36 +03:00
commit 7da0cd57ce
45 changed files with 3703 additions and 0 deletions

View File

@@ -0,0 +1,194 @@
package provider_test
import (
"context"
"crypto/rand"
"fmt"
"math"
"math/big"
"testing"
"time"
glru "github.com/hashicorp/golang-lru"
"github.com/stretchr/testify/require"
"gitoa.ru/go-4devs/cache"
"gitoa.ru/go-4devs/cache/provider/lru"
"gitoa.ru/go-4devs/cache/provider/memcache"
"gitoa.ru/go-4devs/cache/provider/memory"
"gitoa.ru/go-4devs/cache/provider/pebble"
"gitoa.ru/go-4devs/cache/provider/redis"
"gitoa.ru/go-4devs/cache/provider/ristretto"
"gitoa.ru/go-4devs/cache/test"
)
const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
type provider struct {
name string
prov cache.Provider
}
func providers() []provider {
client, _ := glru.New(10000)
db, cl := test.PebbleDB()
defer cl()
return []provider{
{"encoding", memory.NewEncoding()},
{"map", memory.NewMap()},
{"shard", memory.NewMapShard()},
{"lru", lru.New(client)},
{"ristretto", ristretto.New(test.RistrettoClient())},
{"memcache", memcache.New(test.MemcacheClient())},
{"redis", redis.New(test.RedisClient())},
{"pebble", pebble.New(db)},
}
}
func randStringBytes(n int64) string {
b := make([]byte, n)
for i := range b {
b[i] = letterBytes[randInt64(int64(len(letterBytes)))]
}
return string(b)
}
func randInt64(max int64) int64 {
m := big.NewInt(max)
nBig, _ := rand.Int(rand.Reader, m)
return nBig.Int64()
}
func BenchmarkCacheGetRandomKeyString(b *testing.B) {
ctx := context.Background()
keysLen := 10000
for _, p := range providers() {
prov := p.prov
items := make([]*cache.Item, keysLen)
for i := 0; i < keysLen; i++ {
var val string
key := randStringBytes(55)
items[i] = cache.NewItem(key, &val)
require.Nil(b, prov(ctx, cache.OperationSet, cache.NewItem(key, "value: "+p.name, cache.WithTTL(time.Minute))))
}
b.Run(p.name, func(b *testing.B) {
for i := 0; i < b.N; i++ {
_ = prov(ctx, cache.OperationGet, items[i%keysLen])
}
})
}
}
func BenchmarkCacheGetRandomKeyInt(b *testing.B) {
ctx := context.Background()
keysLen := 10000
for _, p := range providers() {
prov := p.prov
items := make([]*cache.Item, keysLen)
for i := 0; i < keysLen; i++ {
var val int64
key := randInt64(math.MaxInt64)
items[i] = cache.NewItem(key, &val)
require.Nil(b, prov(ctx, cache.OperationSet, cache.NewItem(key, randInt64(math.MaxInt64), cache.WithTTL(time.Minute))))
}
b.Run(p.name, func(b *testing.B) {
for i := 0; i < b.N; i++ {
_ = prov(ctx, cache.OperationGet, items[i%keysLen])
}
})
}
}
func BenchmarkCacheGetStruct(b *testing.B) {
ctx := context.Background()
type testStruct struct {
Key string
val string
}
var val testStruct
item := cache.NewItem("key", &val)
for _, p := range providers() {
prov := p.prov
require.Nil(b, prov(ctx, cache.OperationSet, cache.NewItem("key", testStruct{Key: "key", val: ""}, cache.WithTTL(time.Minute))))
b.Run(p.name, func(b *testing.B) {
for i := 0; i < b.N; i++ {
_ = prov(ctx, cache.OperationGet, item)
}
})
}
}
func BenchmarkCacheSetStruct(b *testing.B) {
ctx := context.Background()
type testStruct struct {
Key string
Val int
}
for _, p := range providers() {
prov := p.prov
b.Run(p.name, func(b *testing.B) {
for i := 0; i < b.N; i++ {
item := cache.NewItem(i, testStruct{"k", i}, cache.WithTTL(time.Hour))
_ = prov(ctx, cache.OperationSet, item)
}
})
}
}
func BenchmarkCacheGetParallel(b *testing.B) {
ctx := context.Background()
for _, p := range providers() {
prov := p.prov
key := fmt.Sprintf("key_%s", p.name)
val := fmt.Sprintf("value_%s", p.name)
item := cache.NewItem(key, &val, cache.WithTTL(time.Minute))
require.Nil(b, prov(ctx, cache.OperationSet, item))
b.Run(p.name, func(b *testing.B) {
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
_ = prov(ctx, cache.OperationGet, item)
}
})
})
}
}
func BenchmarkCacheSetParallel(b *testing.B) {
ctx := context.Background()
for _, p := range providers() {
prov := p.prov
key := fmt.Sprintf("key: %v", prov)
val := fmt.Sprintf("value: %v", prov)
b.Run(p.name, func(b *testing.B) {
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
item := cache.NewItem(key, val, cache.WithTTL(time.Hour))
_ = prov(ctx, cache.OperationSet, item)
}
})
})
}
}

66
provider/lru/provider.go Normal file
View File

@@ -0,0 +1,66 @@
package lru
import (
"context"
"fmt"
"time"
lru "github.com/hashicorp/golang-lru"
"gitoa.ru/go-4devs/cache"
)
// New create new lru cache provider.
func New(client *lru.Cache) cache.Provider {
return func(ctx context.Context, operation string, item *cache.Item) error {
switch operation {
case cache.OperationGet:
val, ok := client.Get(item.Key)
if !ok {
return wrapErr(cache.ErrCacheMiss)
}
it, _ := val.(expired)
if !it.ex.IsZero() {
item.TTL = time.Until(it.ex)
}
if item.IsExpired() {
return wrapErr(cache.ErrCacheExpired)
}
return wrapErr(cache.TypeAssert(it.value, item.Value))
case cache.OperationSet:
it := expired{
value: item.Value,
ex: time.Time{},
}
if item.TTL > 0 {
it.ex = item.Expired()
}
_ = client.Add(item.Key, it)
return nil
case cache.OperationDelete:
_ = client.Remove(item.Key)
return nil
}
return wrapErr(cache.ErrOperationNotAllwed)
}
}
type expired struct {
ex time.Time
value interface{}
}
func wrapErr(err error) error {
if err != nil {
return fmt.Errorf("%w: lru", err)
}
return nil
}

View File

@@ -0,0 +1,18 @@
package lru_test
import (
"testing"
glru "github.com/hashicorp/golang-lru"
"github.com/stretchr/testify/require"
"gitoa.ru/go-4devs/cache/provider/lru"
"gitoa.ru/go-4devs/cache/test"
)
func TestEncoding(t *testing.T) {
t.Parallel()
client, err := glru.New(10)
require.Nil(t, err)
test.RunSute(t, lru.New(client))
}

View File

@@ -0,0 +1,52 @@
package memcache
import (
"context"
"errors"
"fmt"
"github.com/bradfitz/gomemcache/memcache"
"gitoa.ru/go-4devs/cache"
)
// New memcache provider.
func New(client *memcache.Client) cache.Provider {
return func(ctx context.Context, operation string, item *cache.Item) error {
key := item.Key.String()
switch operation {
case cache.OperationGet:
ci, err := client.Get(item.Key.String())
switch {
case errors.Is(err, memcache.ErrCacheMiss):
return wrapErr(cache.ErrCacheMiss)
case errors.Is(err, memcache.ErrMalformedKey):
return wrapErr(cache.ErrKeyNotValid)
case err != nil:
return wrapErr(err)
}
return wrapErr(item.Unmarshal(ci.Value))
case cache.OperationSet:
data, err := item.Marshal()
if err != nil {
return wrapErr(err)
}
return wrapErr(client.Set(&memcache.Item{Key: key, Flags: 0, Value: data, Expiration: int32(item.TTL.Seconds())}))
case cache.OperationDelete:
return wrapErr(client.Delete(key))
}
return wrapErr(cache.ErrOperationNotAllwed)
}
}
func wrapErr(err error) error {
if err != nil {
return fmt.Errorf("%w: memcache", err)
}
return nil
}

View File

@@ -0,0 +1,14 @@
package memcache_test
import (
"testing"
"gitoa.ru/go-4devs/cache"
"gitoa.ru/go-4devs/cache/provider/memcache"
"gitoa.ru/go-4devs/cache/test"
)
func TestProvider(t *testing.T) {
t.Parallel()
test.RunSute(t, memcache.New(test.MemcacheClient()), test.WithExpire(cache.ErrCacheMiss))
}

View File

@@ -0,0 +1,93 @@
package memory
import (
"context"
"fmt"
"sync"
"time"
"gitoa.ru/go-4devs/cache"
)
// NewEncoding create new provider.
func NewEncoding() cache.Provider {
items := make(map[cache.Key]encodedEntry)
mu := sync.RWMutex{}
return func(ctx context.Context, operation string, item *cache.Item) error {
switch operation {
case cache.OperationSet:
i, err := newEncodedEntry(item)
if err != nil {
return err
}
mu.Lock()
items[item.Key] = i
mu.Unlock()
return nil
case cache.OperationDelete:
mu.Lock()
delete(items, item.Key)
mu.Unlock()
return nil
case cache.OperationGet:
mu.RLock()
i, ok := items[item.Key]
mu.RUnlock()
if !ok {
return wrapErr(cache.ErrCacheMiss)
}
return resolveEncodedEntry(i, item)
}
return wrapErr(cache.ErrOperationNotAllwed)
}
}
type encodedEntry struct {
data []byte
expired time.Time
}
func wrapErr(err error) error {
if err != nil {
return fmt.Errorf("%w: encoding", err)
}
return nil
}
func newEncodedEntry(item *cache.Item) (encodedEntry, error) {
var (
e encodedEntry
err error
)
e.data, err = item.Marshal()
if err != nil {
return e, wrapErr(err)
}
if item.TTL > 0 {
e.expired = item.Expired()
}
return e, nil
}
func resolveEncodedEntry(e encodedEntry, item *cache.Item) error {
if !e.expired.IsZero() {
item.TTL = time.Until(e.expired)
}
if item.IsExpired() {
return wrapErr(cache.ErrCacheExpired)
}
return wrapErr(item.Unmarshal(e.data))
}

View File

@@ -0,0 +1,13 @@
package memory_test
import (
"testing"
"gitoa.ru/go-4devs/cache/provider/memory"
"gitoa.ru/go-4devs/cache/test"
)
func TestEncoding(t *testing.T) {
t.Parallel()
test.RunSute(t, memory.NewEncoding())
}

155
provider/memory/map.go Normal file
View File

@@ -0,0 +1,155 @@
package memory
import (
"context"
"fmt"
"hash/crc64"
"sync"
"time"
"gitoa.ru/go-4devs/cache"
)
const defaultShards = 255
// NewMap creates new map cache.
func NewMap() cache.Provider {
m := sync.Map{}
return func(ctx context.Context, op string, item *cache.Item) error {
switch op {
case cache.OperationDelete:
m.Delete(item.Key)
return nil
case cache.OperationSet:
m.Store(item.Key, newEntry(item))
return nil
case cache.OperationGet:
e, ok := m.Load(item.Key)
if !ok {
return fmt.Errorf("%w: map", cache.ErrCacheMiss)
}
return resolveEntry(e.(entry), item)
}
return fmt.Errorf("%w: map", cache.ErrOperationNotAllwed)
}
}
func resolveEntry(e entry, item *cache.Item) error {
if !e.expired.IsZero() {
item.TTL = time.Until(e.expired)
}
if item.IsExpired() {
return fmt.Errorf("%w: map", cache.ErrCacheExpired)
}
if err := cache.TypeAssert(e.data, item.Value); err != nil {
return fmt.Errorf("%w: map", err)
}
return nil
}
func newEntry(item *cache.Item) entry {
e := entry{data: item.Value}
if item.TTL > 0 {
e.expired = item.Expired()
}
return e
}
type entry struct {
data interface{}
expired time.Time
}
type settings struct {
numShards uint64
hashString func(in cache.Key) uint64
}
type Option func(*settings)
func WithNumShards(num uint64) Option {
return func(s *settings) {
s.numShards = num
}
}
func WithHashKey(f func(in cache.Key) uint64) Option {
return func(s *settings) {
s.hashString = f
}
}
//nolint: gochecknoglobals
var table = crc64.MakeTable(crc64.ISO)
func hashString(in cache.Key) uint64 {
switch k := in.Key.(type) {
case int64:
return uint64(k)
case int32:
return uint64(k)
case int:
return uint64(k)
case uint64:
return k
case uint32:
return uint64(k)
case uint:
return uint64(k)
default:
return crc64.Checksum([]byte(in.String()), table)
}
}
func NewMapShard(opts ...Option) cache.Provider {
s := settings{
numShards: defaultShards,
hashString: hashString,
}
for _, opt := range opts {
opt(&s)
}
items := make([]*sync.Map, s.numShards)
for i := range items {
items[i] = &sync.Map{}
}
return func(ctx context.Context, operation string, item *cache.Item) error {
idx := s.hashString(item.Key)
switch operation {
case cache.OperationDelete:
items[idx%s.numShards].Delete(item.Key)
return nil
case cache.OperationSet:
items[idx%s.numShards].Store(item.Key, newEntry(item))
return nil
case cache.OperationGet:
e, ok := items[idx%s.numShards].Load(item.Key)
if !ok {
return wrapShardErr(cache.ErrCacheMiss)
}
return resolveEntry(e.(entry), item)
}
return wrapShardErr(cache.ErrOperationNotAllwed)
}
}
func wrapShardErr(err error) error {
return fmt.Errorf("%w: memory shards", err)
}

View File

@@ -0,0 +1,18 @@
package memory_test
import (
"testing"
"gitoa.ru/go-4devs/cache/provider/memory"
"gitoa.ru/go-4devs/cache/test"
)
func TestMap(t *testing.T) {
t.Parallel()
test.RunSute(t, memory.NewMap())
}
func TestMapShard(t *testing.T) {
t.Parallel()
test.RunSute(t, memory.NewMapShard())
}

22
provider/ns/provider.go Normal file
View File

@@ -0,0 +1,22 @@
package ns
import (
"context"
"errors"
"gitoa.ru/go-4devs/cache"
)
var ErrProviderNotFound = errors.New("provider not found")
func New(providers map[string]cache.Provider) cache.Provider {
return func(ctx context.Context, operation string, item *cache.Item) error {
if prov, ok := providers[item.Key.Prefix]; ok {
item.Key.Prefix = ""
return prov(ctx, operation, item)
}
return ErrProviderNotFound
}
}

View File

@@ -0,0 +1,54 @@
package pebble
import (
"context"
"errors"
"fmt"
"github.com/cockroachdb/pebble"
"gitoa.ru/go-4devs/cache"
"gitoa.ru/go-4devs/cache/item"
)
func New(db *pebble.DB) cache.Provider {
return func(ctx context.Context, operation string, i *cache.Item) error {
key := []byte(i.Key.String())
switch operation {
case cache.OperationGet:
val, cl, err := db.Get([]byte(i.Key.String()))
if err != nil {
if errors.Is(err, pebble.ErrNotFound) {
return wrapErr(cache.ErrCacheMiss)
}
return wrapErr(err)
}
defer func() {
_ = cl.Close()
}()
return wrapErr(item.UnmarshalExpired(i, val))
case cache.OperationSet:
b, err := item.MarshalExpired(i)
if err != nil {
return wrapErr(err)
}
return wrapErr(db.Set(key, b, pebble.Sync))
case cache.OperationDelete:
return wrapErr(db.Delete(key, pebble.Sync))
}
return wrapErr(cache.ErrOperationNotAllwed)
}
}
func wrapErr(err error) error {
if err != nil {
return fmt.Errorf("%w: pebble", err)
}
return nil
}

View File

@@ -0,0 +1,17 @@
package pebble_test
import (
"testing"
"gitoa.ru/go-4devs/cache/provider/pebble"
"gitoa.ru/go-4devs/cache/test"
)
func TestPebble(t *testing.T) {
t.Parallel()
db, cl := test.PebbleDB()
defer cl()
test.RunSute(t, pebble.New(db))
}

110
provider/redis/pool.go Normal file
View File

@@ -0,0 +1,110 @@
package redis
import (
"context"
"fmt"
"gitoa.ru/go-4devs/cache"
)
type Conn interface {
Do(commandName string, args ...interface{}) (reply interface{}, err error)
Send(commandName string, args ...interface{}) error
Flush() error
Close() error
}
// New creates new provider.
func New(pool func(context.Context) (Conn, error)) cache.Provider {
return func(ctx context.Context, operation string, item *cache.Item) error {
conn, err := pool(ctx)
if err != nil {
return wrapErr(err)
}
defer conn.Close()
key := item.Key.String()
switch operation {
case cache.OperationGet:
data, ttl, err := get(conn, key)
if err != nil {
return err
}
item.TTLInSecond(ttl)
return wrapErr(item.Unmarshal(data))
case cache.OperationSet:
data, err := item.Marshal()
if err != nil {
return wrapErr(err)
}
return set(conn, key, data, int(item.TTL.Seconds()))
case cache.OperationDelete:
return del(conn, key)
}
return wrapErr(cache.ErrOperationNotAllwed)
}
}
func get(conn Conn, key string) ([]byte, int64, error) {
data, err := conn.Do("GET", key)
if err != nil {
return nil, 0, wrapErr(err)
}
if data == nil {
return nil, 0, wrapErr(cache.ErrCacheMiss)
}
v, ok := data.([]byte)
if !ok {
return nil, 0, wrapErr(cache.ErrSourceNotValid)
}
expire, err := conn.Do("TTL", key)
if err != nil {
return v, 0, wrapErr(err)
}
ex, _ := expire.(int64)
return v, ex, nil
}
func set(conn Conn, key string, data []byte, ttl int) error {
if err := conn.Send("SET", key, data); err != nil {
return wrapErr(err)
}
if ttl > 0 {
if err := conn.Send("EXPIRE", key, ttl); err != nil {
return wrapErr(err)
}
}
if err := conn.Flush(); err != nil {
return fmt.Errorf("failed flush then set %s by %w", key, conn.Flush())
}
return nil
}
func del(conn Conn, key string) error {
if _, err := conn.Do("DEL", key); err != nil {
return wrapErr(err)
}
return nil
}
func wrapErr(err error) error {
if err != nil {
return fmt.Errorf("%w: redis pool", err)
}
return nil
}

View File

@@ -0,0 +1,14 @@
package redis_test
import (
"testing"
"gitoa.ru/go-4devs/cache"
"gitoa.ru/go-4devs/cache/provider/redis"
"gitoa.ru/go-4devs/cache/test"
)
func TestRedisPool(t *testing.T) {
t.Parallel()
test.RunSute(t, redis.New(test.RedisClient()), test.WithExpire(cache.ErrCacheMiss))
}

20
provider/redis/redigo.go Normal file
View File

@@ -0,0 +1,20 @@
package redis
import (
"context"
"fmt"
"github.com/gomodule/redigo/redis"
)
// NewPool creates redigo pool.
func NewPool(pool *redis.Pool) func(context.Context) (Conn, error) {
return func(ctx context.Context) (Conn, error) {
conn, err := pool.GetContext(ctx)
if err != nil {
return nil, fmt.Errorf("failed get connect: %w", err)
}
return conn, nil
}
}

View File

@@ -0,0 +1,69 @@
package ristretto
import (
"context"
"errors"
"fmt"
"github.com/dgraph-io/ristretto"
"gitoa.ru/go-4devs/cache"
)
var ErrSetValue = errors.New("failed set value")
type Option func(*setting)
func WithCost(cost int64) Option {
return func(s *setting) {
s.cost = cost
}
}
type setting struct {
cost int64
}
func New(retto *ristretto.Cache, opts ...Option) cache.Provider {
s := setting{
cost: 1,
}
for _, opt := range opts {
opt(&s)
}
return func(ctx context.Context, operation string, item *cache.Item) error {
var key interface{}
if item.Key.Prefix != "" {
key = item.Key.String()
} else {
key = item.Key.Key
}
switch operation {
case cache.OperationGet:
res, ok := retto.Get(key)
if !ok {
return fmt.Errorf("%w: ristretto", cache.ErrCacheMiss)
}
if err := cache.TypeAssert(res, item.Value); err != nil {
return fmt.Errorf("failed assert type: %w", err)
}
return nil
case cache.OperationDelete:
retto.Del(key)
return nil
case cache.OperationSet:
if ok := retto.SetWithTTL(key, item.Value, s.cost, item.TTL); !ok {
return ErrSetValue
}
return nil
}
return cache.ErrOperationNotAllwed
}
}

View File

@@ -0,0 +1,26 @@
package ristretto_test
import (
"testing"
"time"
"github.com/dgraph-io/ristretto"
"github.com/stretchr/testify/require"
"gitoa.ru/go-4devs/cache"
provider "gitoa.ru/go-4devs/cache/provider/ristretto"
"gitoa.ru/go-4devs/cache/test"
)
func TestRistretto(t *testing.T) {
t.Parallel()
retto, err := ristretto.NewCache(&ristretto.Config{
NumCounters: 1e7, // number of keys to track frequency of (10M).
MaxCost: 1 << 30, // maximum cost of cache (1GB).
BufferItems: 64, // number of keys per Get buffer.
})
require.Nil(t, err)
test.RunSute(t, provider.New(retto), test.WithWaitGet(func() {
time.Sleep(10 * time.Millisecond)
}), test.WithExpire(cache.ErrCacheMiss))
}