#1: Implemented in-memory storage for textures

This commit is contained in:
ErickSkrauch 2019-04-21 20:28:58 +03:00
parent ad300e8c1c
commit d7f03ce182
7 changed files with 299 additions and 5 deletions

9
Gopkg.lock generated
View File

@ -247,6 +247,14 @@
revision = "ffdc059bfe9ce6a4e144ba849dbedead332c6053" revision = "ffdc059bfe9ce6a4e144ba849dbedead332c6053"
version = "v1.3.0" version = "v1.3.0"
[[projects]]
branch = "master"
digest = "1:86e6712cfd4070a2120c03fcec41cfcbbc51813504a74e28d74479edfaf669ee"
name = "github.com/tevino/abool"
packages = ["."]
pruneopts = ""
revision = "9b9efcf221b50905aab9bbabd3daed56dc10f339"
[[projects]] [[projects]]
branch = "issue-18" branch = "issue-18"
digest = "1:123d45cdeb4dbefa402fb700760b0a5f8d1cb5ed55c78a757dc4bb5c12a7b3db" digest = "1:123d45cdeb4dbefa402fb700760b0a5f8d1cb5ed55c78a757dc4bb5c12a7b3db"
@ -318,6 +326,7 @@
"github.com/stretchr/testify/assert", "github.com/stretchr/testify/assert",
"github.com/stretchr/testify/mock", "github.com/stretchr/testify/mock",
"github.com/stretchr/testify/suite", "github.com/stretchr/testify/suite",
"github.com/tevino/abool",
"github.com/thedevsaddam/govalidator", "github.com/thedevsaddam/govalidator",
"gopkg.in/h2non/gock.v1", "gopkg.in/h2non/gock.v1",
] ]

View File

@ -29,6 +29,10 @@ ignored = ["github.com/elyby/chrly"]
source = "https://github.com/erickskrauch/govalidator.git" source = "https://github.com/erickskrauch/govalidator.git"
branch = "issue-18" branch = "issue-18"
[[constraint]]
branch = "master"
name = "github.com/tevino/abool"
# Testing dependencies # Testing dependencies
[[constraint]] [[constraint]]

View File

@ -0,0 +1,107 @@
package queue
import (
"errors"
"sync"
"time"
"github.com/elyby/chrly/api/mojang"
"github.com/tevino/abool"
)
var inMemoryStorageGCPeriod = time.Second
var inMemoryStoragePersistPeriod = time.Second * 60
var now = time.Now
type inMemoryItem struct {
textures *mojang.SignedTexturesResponse
timestamp int64
}
type inMemoryTexturesStorage struct {
lock sync.Mutex
data map[string]*inMemoryItem
working *abool.AtomicBool
}
func CreateInMemoryTexturesStorage() *inMemoryTexturesStorage {
return &inMemoryTexturesStorage{
data: make(map[string]*inMemoryItem),
}
}
func (s *inMemoryTexturesStorage) Start() {
if s.working == nil {
s.working = abool.New()
}
if !s.working.IsSet() {
go func() {
time.Sleep(inMemoryStorageGCPeriod)
// TODO: this can be reimplemented in future with channels, but right now I have no idea how to make it right
for s.working.IsSet() {
start := time.Now()
s.gc()
time.Sleep(inMemoryStorageGCPeriod - time.Since(start))
}
}()
}
s.working.Set()
}
func (s *inMemoryTexturesStorage) Stop() {
s.working.UnSet()
}
func (s *inMemoryTexturesStorage) GetTextures(uuid string) (*mojang.SignedTexturesResponse, error) {
s.lock.Lock()
defer s.lock.Unlock()
item, exists := s.data[uuid]
if !exists || now().Add(inMemoryStoragePersistPeriod*time.Duration(-1)).UnixNano()/10e5 > item.timestamp {
return nil, &ValueNotFound{}
}
return item.textures, nil
}
func (s *inMemoryTexturesStorage) StoreTextures(textures *mojang.SignedTexturesResponse) {
s.lock.Lock()
defer s.lock.Unlock()
var texturesProp *mojang.Property
for _, prop := range textures.Props {
if prop.Name == "textures" {
texturesProp = prop
break
}
}
if texturesProp == nil {
panic(errors.New("unable to find textures property"))
}
decoded, err := mojang.DecodeTextures(texturesProp.Value)
if err != nil {
panic(err)
}
s.data[textures.Id] = &inMemoryItem{
textures: textures,
timestamp: decoded.Timestamp,
}
}
func (s *inMemoryTexturesStorage) gc() {
s.lock.Lock()
defer s.lock.Unlock()
maxTime := now().Add(inMemoryStoragePersistPeriod*time.Duration(-1)).UnixNano() / 10e5
for uuid, value := range s.data {
if maxTime > value.timestamp {
delete(s.data, uuid)
}
}
}

View File

@ -0,0 +1,174 @@
package queue
import (
"time"
"github.com/elyby/chrly/api/mojang"
testify "github.com/stretchr/testify/assert"
"testing"
)
var texturesWithSkin = &mojang.SignedTexturesResponse{
Id: "dead24f9a4fa4877b7b04c8c6c72bb46",
Name: "mock",
Props: []*mojang.Property{
{
Name: "textures",
Value: mojang.EncodeTextures(&mojang.TexturesProp{
Timestamp: time.Now().UnixNano() / 10e5,
ProfileID: "dead24f9a4fa4877b7b04c8c6c72bb46",
ProfileName: "mock",
Textures: &mojang.TexturesResponse{
Skin: &mojang.SkinTexturesResponse{
Url: "http://textures.minecraft.net/texture/74d1e08b0bb7e9f590af27758125bbed1778ac6cef729aedfcb9613e9911ae75",
},
},
}),
},
},
}
var texturesWithoutSkin = &mojang.SignedTexturesResponse{
Id: "dead24f9a4fa4877b7b04c8c6c72bb46",
Name: "mock",
Props: []*mojang.Property{
{
Name: "textures",
Value: mojang.EncodeTextures(&mojang.TexturesProp{
Timestamp: time.Now().UnixNano() / 10e5,
ProfileID: "dead24f9a4fa4877b7b04c8c6c72bb46",
ProfileName: "mock",
Textures: &mojang.TexturesResponse{},
}),
},
},
}
func TestInMemoryTexturesStorage_GetTextures(t *testing.T) {
t.Run("get error when uuid is not exists", func(t *testing.T) {
assert := testify.New(t)
storage := CreateInMemoryTexturesStorage()
result, err := storage.GetTextures("b5d58475007d4f9e9ddd1403e2497579")
assert.Nil(result)
assert.Error(err, "value not found in the storage")
})
t.Run("get textures object, when uuid is stored in the storage", func(t *testing.T) {
assert := testify.New(t)
storage := CreateInMemoryTexturesStorage()
storage.StoreTextures(texturesWithSkin)
result, err := storage.GetTextures("dead24f9a4fa4877b7b04c8c6c72bb46")
assert.Equal(texturesWithSkin, result)
assert.Nil(err)
})
t.Run("get error when uuid is exists, but textures are expired", func(t *testing.T) {
assert := testify.New(t)
storage := CreateInMemoryTexturesStorage()
storage.StoreTextures(texturesWithSkin)
now = func() time.Time {
return time.Now().Add(time.Minute * 2)
}
result, err := storage.GetTextures("dead24f9a4fa4877b7b04c8c6c72bb46")
assert.Nil(result)
assert.Error(err, "value not found in the storage")
now = time.Now
})
}
func TestInMemoryTexturesStorage_StoreTextures(t *testing.T) {
t.Run("store textures for previously not existed uuid", func(t *testing.T) {
assert := testify.New(t)
storage := CreateInMemoryTexturesStorage()
storage.StoreTextures(texturesWithSkin)
result, err := storage.GetTextures("dead24f9a4fa4877b7b04c8c6c72bb46")
assert.Equal(texturesWithSkin, result)
assert.Nil(err)
})
t.Run("override already existed textures for uuid", func(t *testing.T) {
assert := testify.New(t)
storage := CreateInMemoryTexturesStorage()
storage.StoreTextures(texturesWithoutSkin)
storage.StoreTextures(texturesWithSkin)
result, err := storage.GetTextures("dead24f9a4fa4877b7b04c8c6c72bb46")
assert.NotEqual(texturesWithoutSkin, result)
assert.Equal(texturesWithSkin, result)
assert.Nil(err)
})
}
func TestInMemoryTexturesStorage_GarbageCollection(t *testing.T) {
assert := testify.New(t)
inMemoryStorageGCPeriod = 10 * time.Millisecond
inMemoryStoragePersistPeriod = 10 * time.Millisecond
textures1 := &mojang.SignedTexturesResponse{
Id: "dead24f9a4fa4877b7b04c8c6c72bb46",
Name: "mock1",
Props: []*mojang.Property{
{
Name: "textures",
Value: mojang.EncodeTextures(&mojang.TexturesProp{
Timestamp: time.Now().Add(inMemoryStorageGCPeriod-time.Millisecond*time.Duration(5)).UnixNano() / 10e5,
ProfileID: "dead24f9a4fa4877b7b04c8c6c72bb46",
ProfileName: "mock1",
Textures: &mojang.TexturesResponse{},
}),
},
},
}
textures2 := &mojang.SignedTexturesResponse{
Id: "b5d58475007d4f9e9ddd1403e2497579",
Name: "mock2",
Props: []*mojang.Property{
{
Name: "textures",
Value: mojang.EncodeTextures(&mojang.TexturesProp{
Timestamp: time.Now().Add(inMemoryStorageGCPeriod-time.Millisecond*time.Duration(15)).UnixNano() / 10e5,
ProfileID: "b5d58475007d4f9e9ddd1403e2497579",
ProfileName: "mock2",
Textures: &mojang.TexturesResponse{},
}),
},
},
}
storage := CreateInMemoryTexturesStorage()
storage.StoreTextures(textures1)
storage.StoreTextures(textures2)
storage.Start()
time.Sleep(inMemoryStorageGCPeriod + time.Millisecond) // Let it start first iteration
_, textures1Err := storage.GetTextures("dead24f9a4fa4877b7b04c8c6c72bb46")
_, textures2Err := storage.GetTextures("b5d58475007d4f9e9ddd1403e2497579")
assert.Nil(textures1Err)
assert.Error(textures2Err)
time.Sleep(inMemoryStorageGCPeriod + time.Millisecond) // Let another iteration happen
_, textures1Err = storage.GetTextures("dead24f9a4fa4877b7b04c8c6c72bb46")
_, textures2Err = storage.GetTextures("b5d58475007d4f9e9ddd1403e2497579")
assert.Error(textures1Err)
assert.Error(textures2Err)
storage.Stop()
}

View File

@ -13,7 +13,7 @@ import (
var usernamesToUuids = mojang.UsernamesToUuids var usernamesToUuids = mojang.UsernamesToUuids
var uuidToTextures = mojang.UuidToTextures var uuidToTextures = mojang.UuidToTextures
var delay = time.Second var uuidsQueuePeriod = time.Second
var forever = func() bool { var forever = func() bool {
return true return true
} }
@ -81,11 +81,11 @@ func (ctx *JobsQueue) GetTexturesForUsername(username string) chan *mojang.Signe
func (ctx *JobsQueue) startQueue() { func (ctx *JobsQueue) startQueue() {
go func() { go func() {
time.Sleep(delay) time.Sleep(uuidsQueuePeriod)
for forever() { for forever() {
start := time.Now() start := time.Now()
ctx.queueRound() ctx.queueRound()
time.Sleep(delay - time.Since(start)) time.Sleep(uuidsQueuePeriod - time.Since(start))
} }
}() }()
} }

View File

@ -76,7 +76,7 @@ type queueTestSuite struct {
} }
func (suite *queueTestSuite) SetupSuite() { func (suite *queueTestSuite) SetupSuite() {
delay = 0 uuidsQueuePeriod = 0
} }
func (suite *queueTestSuite) SetupTest() { func (suite *queueTestSuite) SetupTest() {

View File

@ -26,5 +26,5 @@ type ValueNotFound struct {
} }
func (*ValueNotFound) Error() string { func (*ValueNotFound) Error() string {
return "value not found in storage" return "value not found in the storage"
} }