mirror of
https://github.com/elyby/chrly.git
synced 2025-05-31 14:11:51 +05:30
Completely rework mojang textures queue implementation, split it across separate data providers
This commit is contained in:
ErickSkrauch
@@ -1,53 +0,0 @@
|
||||
package queue
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"github.com/elyby/chrly/api/mojang"
|
||||
)
|
||||
|
||||
type broadcastMap struct {
|
||||
lock sync.Mutex
|
||||
listeners map[string][]chan *mojang.SignedTexturesResponse
|
||||
}
|
||||
|
||||
func newBroadcaster() *broadcastMap {
|
||||
return &broadcastMap{
|
||||
listeners: make(map[string][]chan *mojang.SignedTexturesResponse),
|
||||
}
|
||||
}
|
||||
|
||||
// Returns a boolean value, which will be true if the username passed didn't exist before
|
||||
func (c *broadcastMap) AddListener(username string, resultChan chan *mojang.SignedTexturesResponse) bool {
|
||||
c.lock.Lock()
|
||||
defer c.lock.Unlock()
|
||||
|
||||
val, alreadyHasSource := c.listeners[username]
|
||||
if alreadyHasSource {
|
||||
c.listeners[username] = append(val, resultChan)
|
||||
return false
|
||||
}
|
||||
|
||||
c.listeners[username] = []chan *mojang.SignedTexturesResponse{resultChan}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func (c *broadcastMap) BroadcastAndRemove(username string, result *mojang.SignedTexturesResponse) {
|
||||
c.lock.Lock()
|
||||
defer c.lock.Unlock()
|
||||
|
||||
val, ok := c.listeners[username]
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
for _, channel := range val {
|
||||
go func(channel chan *mojang.SignedTexturesResponse) {
|
||||
channel <- result
|
||||
close(channel)
|
||||
}(channel)
|
||||
}
|
||||
|
||||
delete(c.listeners, username)
|
||||
}
|
||||
@@ -1,75 +0,0 @@
|
||||
package queue
|
||||
|
||||
import (
|
||||
"github.com/elyby/chrly/api/mojang"
|
||||
|
||||
testify "github.com/stretchr/testify/assert"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestBroadcastMap_GetOrAppend(t *testing.T) {
|
||||
t.Run("first call when username didn't exist before should return true", func(t *testing.T) {
|
||||
assert := testify.New(t)
|
||||
|
||||
broadcaster := newBroadcaster()
|
||||
channel := make(chan *mojang.SignedTexturesResponse)
|
||||
isFirstListener := broadcaster.AddListener("mock", channel)
|
||||
|
||||
assert.True(isFirstListener)
|
||||
listeners, ok := broadcaster.listeners["mock"]
|
||||
assert.True(ok)
|
||||
assert.Len(listeners, 1)
|
||||
assert.Equal(channel, listeners[0])
|
||||
})
|
||||
|
||||
t.Run("subsequent calls should return false", func(t *testing.T) {
|
||||
assert := testify.New(t)
|
||||
|
||||
broadcaster := newBroadcaster()
|
||||
channel1 := make(chan *mojang.SignedTexturesResponse)
|
||||
isFirstListener := broadcaster.AddListener("mock", channel1)
|
||||
|
||||
assert.True(isFirstListener)
|
||||
|
||||
channel2 := make(chan *mojang.SignedTexturesResponse)
|
||||
isFirstListener = broadcaster.AddListener("mock", channel2)
|
||||
|
||||
assert.False(isFirstListener)
|
||||
|
||||
channel3 := make(chan *mojang.SignedTexturesResponse)
|
||||
isFirstListener = broadcaster.AddListener("mock", channel3)
|
||||
|
||||
assert.False(isFirstListener)
|
||||
})
|
||||
}
|
||||
|
||||
func TestBroadcastMap_BroadcastAndRemove(t *testing.T) {
|
||||
t.Run("should broadcast to all listeners and remove the key", func(t *testing.T) {
|
||||
assert := testify.New(t)
|
||||
|
||||
broadcaster := newBroadcaster()
|
||||
channel1 := make(chan *mojang.SignedTexturesResponse)
|
||||
channel2 := make(chan *mojang.SignedTexturesResponse)
|
||||
broadcaster.AddListener("mock", channel1)
|
||||
broadcaster.AddListener("mock", channel2)
|
||||
|
||||
result := &mojang.SignedTexturesResponse{Id: "mockUuid"}
|
||||
broadcaster.BroadcastAndRemove("mock", result)
|
||||
|
||||
assert.Equal(result, <-channel1)
|
||||
assert.Equal(result, <-channel2)
|
||||
|
||||
channel3 := make(chan *mojang.SignedTexturesResponse)
|
||||
isFirstListener := broadcaster.AddListener("mock", channel3)
|
||||
assert.True(isFirstListener)
|
||||
})
|
||||
|
||||
t.Run("call on not exists username", func(t *testing.T) {
|
||||
assert := testify.New(t)
|
||||
|
||||
assert.NotPanics(func() {
|
||||
broadcaster := newBroadcaster()
|
||||
broadcaster.BroadcastAndRemove("mock", &mojang.SignedTexturesResponse{})
|
||||
})
|
||||
})
|
||||
}
|
||||
@@ -1,112 +0,0 @@
|
||||
package queue
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/elyby/chrly/api/mojang"
|
||||
|
||||
"github.com/tevino/abool"
|
||||
)
|
||||
|
||||
var inMemoryStorageGCPeriod = 10 * time.Second
|
||||
var inMemoryStoragePersistPeriod = time.Minute + 10*time.Second
|
||||
var now = time.Now
|
||||
|
||||
type inMemoryItem struct {
|
||||
textures *mojang.SignedTexturesResponse
|
||||
timestamp int64
|
||||
}
|
||||
|
||||
type inMemoryTexturesStorage struct {
|
||||
lock sync.Mutex
|
||||
data map[string]*inMemoryItem
|
||||
working *abool.AtomicBool
|
||||
}
|
||||
|
||||
func CreateInMemoryTexturesStorage() *inMemoryTexturesStorage {
|
||||
storage := &inMemoryTexturesStorage{
|
||||
data: make(map[string]*inMemoryItem),
|
||||
}
|
||||
|
||||
return storage
|
||||
}
|
||||
|
||||
func (s *inMemoryTexturesStorage) Start() {
|
||||
if s.working == nil {
|
||||
s.working = abool.New()
|
||||
}
|
||||
|
||||
if !s.working.IsSet() {
|
||||
go func() {
|
||||
time.Sleep(inMemoryStorageGCPeriod)
|
||||
// TODO: this can be reimplemented in future with channels, but right now I have no idea how to make it right
|
||||
for s.working.IsSet() {
|
||||
start := time.Now()
|
||||
s.gc()
|
||||
time.Sleep(inMemoryStorageGCPeriod - time.Since(start))
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
s.working.Set()
|
||||
}
|
||||
|
||||
func (s *inMemoryTexturesStorage) Stop() {
|
||||
s.working.UnSet()
|
||||
}
|
||||
|
||||
func (s *inMemoryTexturesStorage) GetTextures(uuid string) (*mojang.SignedTexturesResponse, error) {
|
||||
s.lock.Lock()
|
||||
defer s.lock.Unlock()
|
||||
|
||||
item, exists := s.data[uuid]
|
||||
validRange := getMinimalNotExpiredTimestamp()
|
||||
if !exists || validRange > item.timestamp {
|
||||
return nil, &ValueNotFound{}
|
||||
}
|
||||
|
||||
return item.textures, nil
|
||||
}
|
||||
|
||||
func (s *inMemoryTexturesStorage) StoreTextures(uuid string, textures *mojang.SignedTexturesResponse) {
|
||||
var timestamp int64
|
||||
if textures != nil {
|
||||
decoded := textures.DecodeTextures()
|
||||
if decoded == nil {
|
||||
panic("unable to decode textures")
|
||||
}
|
||||
|
||||
timestamp = decoded.Timestamp
|
||||
} else {
|
||||
timestamp = unixNanoToUnixMicro(now().UnixNano())
|
||||
}
|
||||
|
||||
s.lock.Lock()
|
||||
defer s.lock.Unlock()
|
||||
|
||||
s.data[uuid] = &inMemoryItem{
|
||||
textures: textures,
|
||||
timestamp: timestamp,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *inMemoryTexturesStorage) gc() {
|
||||
s.lock.Lock()
|
||||
defer s.lock.Unlock()
|
||||
|
||||
maxTime := getMinimalNotExpiredTimestamp()
|
||||
for uuid, value := range s.data {
|
||||
if maxTime > value.timestamp {
|
||||
delete(s.data, uuid)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func getMinimalNotExpiredTimestamp() int64 {
|
||||
return unixNanoToUnixMicro(now().Add(inMemoryStoragePersistPeriod * time.Duration(-1)).UnixNano())
|
||||
}
|
||||
|
||||
func unixNanoToUnixMicro(unixNano int64) int64 {
|
||||
return unixNano / 10e5
|
||||
}
|
||||
@@ -1,200 +0,0 @@
|
||||
package queue
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/elyby/chrly/api/mojang"
|
||||
|
||||
testify "github.com/stretchr/testify/assert"
|
||||
"testing"
|
||||
)
|
||||
|
||||
var texturesWithSkin = &mojang.SignedTexturesResponse{
|
||||
Id: "dead24f9a4fa4877b7b04c8c6c72bb46",
|
||||
Name: "mock",
|
||||
Props: []*mojang.Property{
|
||||
{
|
||||
Name: "textures",
|
||||
Value: mojang.EncodeTextures(&mojang.TexturesProp{
|
||||
Timestamp: time.Now().UnixNano() / 10e5,
|
||||
ProfileID: "dead24f9a4fa4877b7b04c8c6c72bb46",
|
||||
ProfileName: "mock",
|
||||
Textures: &mojang.TexturesResponse{
|
||||
Skin: &mojang.SkinTexturesResponse{
|
||||
Url: "http://textures.minecraft.net/texture/74d1e08b0bb7e9f590af27758125bbed1778ac6cef729aedfcb9613e9911ae75",
|
||||
},
|
||||
},
|
||||
}),
|
||||
},
|
||||
},
|
||||
}
|
||||
var texturesWithoutSkin = &mojang.SignedTexturesResponse{
|
||||
Id: "dead24f9a4fa4877b7b04c8c6c72bb46",
|
||||
Name: "mock",
|
||||
Props: []*mojang.Property{
|
||||
{
|
||||
Name: "textures",
|
||||
Value: mojang.EncodeTextures(&mojang.TexturesProp{
|
||||
Timestamp: time.Now().UnixNano() / 10e5,
|
||||
ProfileID: "dead24f9a4fa4877b7b04c8c6c72bb46",
|
||||
ProfileName: "mock",
|
||||
Textures: &mojang.TexturesResponse{},
|
||||
}),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
func TestInMemoryTexturesStorage_GetTextures(t *testing.T) {
|
||||
t.Run("get error when uuid is not exists", func(t *testing.T) {
|
||||
assert := testify.New(t)
|
||||
|
||||
storage := CreateInMemoryTexturesStorage()
|
||||
result, err := storage.GetTextures("b5d58475007d4f9e9ddd1403e2497579")
|
||||
|
||||
assert.Nil(result)
|
||||
assert.Error(err, "value not found in the storage")
|
||||
})
|
||||
|
||||
t.Run("get textures object, when uuid is stored in the storage", func(t *testing.T) {
|
||||
assert := testify.New(t)
|
||||
|
||||
storage := CreateInMemoryTexturesStorage()
|
||||
storage.StoreTextures("dead24f9a4fa4877b7b04c8c6c72bb46", texturesWithSkin)
|
||||
result, err := storage.GetTextures("dead24f9a4fa4877b7b04c8c6c72bb46")
|
||||
|
||||
assert.Equal(texturesWithSkin, result)
|
||||
assert.Nil(err)
|
||||
})
|
||||
|
||||
t.Run("get error when uuid is exists, but textures are expired", func(t *testing.T) {
|
||||
assert := testify.New(t)
|
||||
|
||||
storage := CreateInMemoryTexturesStorage()
|
||||
storage.StoreTextures("dead24f9a4fa4877b7b04c8c6c72bb46", texturesWithSkin)
|
||||
|
||||
now = func() time.Time {
|
||||
return time.Now().Add(time.Minute * 2)
|
||||
}
|
||||
|
||||
result, err := storage.GetTextures("dead24f9a4fa4877b7b04c8c6c72bb46")
|
||||
|
||||
assert.Nil(result)
|
||||
assert.Error(err, "value not found in the storage")
|
||||
|
||||
now = time.Now
|
||||
})
|
||||
}
|
||||
|
||||
func TestInMemoryTexturesStorage_StoreTextures(t *testing.T) {
|
||||
t.Run("store textures for previously not existed uuid", func(t *testing.T) {
|
||||
assert := testify.New(t)
|
||||
|
||||
storage := CreateInMemoryTexturesStorage()
|
||||
storage.StoreTextures("dead24f9a4fa4877b7b04c8c6c72bb46", texturesWithSkin)
|
||||
result, err := storage.GetTextures("dead24f9a4fa4877b7b04c8c6c72bb46")
|
||||
|
||||
assert.Equal(texturesWithSkin, result)
|
||||
assert.Nil(err)
|
||||
})
|
||||
|
||||
t.Run("override already existed textures for uuid", func(t *testing.T) {
|
||||
assert := testify.New(t)
|
||||
|
||||
storage := CreateInMemoryTexturesStorage()
|
||||
storage.StoreTextures("dead24f9a4fa4877b7b04c8c6c72bb46", texturesWithoutSkin)
|
||||
storage.StoreTextures("dead24f9a4fa4877b7b04c8c6c72bb46", texturesWithSkin)
|
||||
result, err := storage.GetTextures("dead24f9a4fa4877b7b04c8c6c72bb46")
|
||||
|
||||
assert.NotEqual(texturesWithoutSkin, result)
|
||||
assert.Equal(texturesWithSkin, result)
|
||||
assert.Nil(err)
|
||||
})
|
||||
|
||||
t.Run("store nil textures", func(t *testing.T) {
|
||||
assert := testify.New(t)
|
||||
|
||||
storage := CreateInMemoryTexturesStorage()
|
||||
storage.StoreTextures("dead24f9a4fa4877b7b04c8c6c72bb46", nil)
|
||||
result, err := storage.GetTextures("dead24f9a4fa4877b7b04c8c6c72bb46")
|
||||
|
||||
assert.Nil(result)
|
||||
assert.Nil(err)
|
||||
})
|
||||
|
||||
t.Run("should panic if textures prop is not decoded", func(t *testing.T) {
|
||||
assert := testify.New(t)
|
||||
|
||||
toStore := &mojang.SignedTexturesResponse{
|
||||
Id: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
|
||||
Name: "mock",
|
||||
Props: []*mojang.Property{},
|
||||
}
|
||||
|
||||
assert.PanicsWithValue("unable to decode textures", func() {
|
||||
storage := CreateInMemoryTexturesStorage()
|
||||
storage.StoreTextures("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", toStore)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func TestInMemoryTexturesStorage_GarbageCollection(t *testing.T) {
|
||||
assert := testify.New(t)
|
||||
|
||||
inMemoryStorageGCPeriod = 10 * time.Millisecond
|
||||
inMemoryStoragePersistPeriod = 10 * time.Millisecond
|
||||
|
||||
textures1 := &mojang.SignedTexturesResponse{
|
||||
Id: "dead24f9a4fa4877b7b04c8c6c72bb46",
|
||||
Name: "mock1",
|
||||
Props: []*mojang.Property{
|
||||
{
|
||||
Name: "textures",
|
||||
Value: mojang.EncodeTextures(&mojang.TexturesProp{
|
||||
Timestamp: time.Now().Add(inMemoryStorageGCPeriod-time.Millisecond*time.Duration(5)).UnixNano() / 10e5,
|
||||
ProfileID: "dead24f9a4fa4877b7b04c8c6c72bb46",
|
||||
ProfileName: "mock1",
|
||||
Textures: &mojang.TexturesResponse{},
|
||||
}),
|
||||
},
|
||||
},
|
||||
}
|
||||
textures2 := &mojang.SignedTexturesResponse{
|
||||
Id: "b5d58475007d4f9e9ddd1403e2497579",
|
||||
Name: "mock2",
|
||||
Props: []*mojang.Property{
|
||||
{
|
||||
Name: "textures",
|
||||
Value: mojang.EncodeTextures(&mojang.TexturesProp{
|
||||
Timestamp: time.Now().Add(inMemoryStorageGCPeriod-time.Millisecond*time.Duration(15)).UnixNano() / 10e5,
|
||||
ProfileID: "b5d58475007d4f9e9ddd1403e2497579",
|
||||
ProfileName: "mock2",
|
||||
Textures: &mojang.TexturesResponse{},
|
||||
}),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
storage := CreateInMemoryTexturesStorage()
|
||||
storage.StoreTextures("dead24f9a4fa4877b7b04c8c6c72bb46", textures1)
|
||||
storage.StoreTextures("b5d58475007d4f9e9ddd1403e2497579", textures2)
|
||||
|
||||
storage.Start()
|
||||
|
||||
time.Sleep(inMemoryStorageGCPeriod + time.Millisecond) // Let it start first iteration
|
||||
|
||||
_, textures1Err := storage.GetTextures("dead24f9a4fa4877b7b04c8c6c72bb46")
|
||||
_, textures2Err := storage.GetTextures("b5d58475007d4f9e9ddd1403e2497579")
|
||||
|
||||
assert.Nil(textures1Err)
|
||||
assert.Error(textures2Err)
|
||||
|
||||
time.Sleep(inMemoryStorageGCPeriod + time.Millisecond) // Let another iteration happen
|
||||
|
||||
_, textures1Err = storage.GetTextures("dead24f9a4fa4877b7b04c8c6c72bb46")
|
||||
_, textures2Err = storage.GetTextures("b5d58475007d4f9e9ddd1403e2497579")
|
||||
|
||||
assert.Error(textures1Err)
|
||||
assert.Error(textures2Err)
|
||||
|
||||
storage.Stop()
|
||||
}
|
||||
@@ -1,56 +0,0 @@
|
||||
// Based on the implementation from https://flaviocopes.com/golang-data-structure-queue/
|
||||
|
||||
package queue
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"github.com/elyby/chrly/api/mojang"
|
||||
)
|
||||
|
||||
type jobItem struct {
|
||||
Username string
|
||||
RespondTo chan *mojang.SignedTexturesResponse
|
||||
}
|
||||
|
||||
type jobsQueue struct {
|
||||
lock sync.Mutex
|
||||
items []*jobItem
|
||||
}
|
||||
|
||||
func (s *jobsQueue) New() *jobsQueue {
|
||||
s.items = []*jobItem{}
|
||||
return s
|
||||
}
|
||||
|
||||
func (s *jobsQueue) Enqueue(t *jobItem) {
|
||||
s.lock.Lock()
|
||||
defer s.lock.Unlock()
|
||||
|
||||
s.items = append(s.items, t)
|
||||
}
|
||||
|
||||
func (s *jobsQueue) Dequeue(n int) []*jobItem {
|
||||
s.lock.Lock()
|
||||
defer s.lock.Unlock()
|
||||
|
||||
if n > s.Size() {
|
||||
n = s.Size()
|
||||
}
|
||||
|
||||
items := s.items[0:n]
|
||||
s.items = s.items[n:len(s.items)]
|
||||
|
||||
return items
|
||||
}
|
||||
|
||||
func (s *jobsQueue) IsEmpty() bool {
|
||||
s.lock.Lock()
|
||||
defer s.lock.Unlock()
|
||||
|
||||
return len(s.items) == 0
|
||||
}
|
||||
|
||||
func (s *jobsQueue) Size() int {
|
||||
return len(s.items)
|
||||
}
|
||||
@@ -1,47 +0,0 @@
|
||||
package queue
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
testify "github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestEnqueue(t *testing.T) {
|
||||
assert := testify.New(t)
|
||||
|
||||
s := createQueue()
|
||||
s.Enqueue(&jobItem{Username: "username1"})
|
||||
s.Enqueue(&jobItem{Username: "username2"})
|
||||
s.Enqueue(&jobItem{Username: "username3"})
|
||||
|
||||
assert.Equal(3, s.Size())
|
||||
}
|
||||
|
||||
func TestDequeueN(t *testing.T) {
|
||||
assert := testify.New(t)
|
||||
|
||||
s := createQueue()
|
||||
s.Enqueue(&jobItem{Username: "username1"})
|
||||
s.Enqueue(&jobItem{Username: "username2"})
|
||||
s.Enqueue(&jobItem{Username: "username3"})
|
||||
s.Enqueue(&jobItem{Username: "username4"})
|
||||
|
||||
items := s.Dequeue(2)
|
||||
assert.Len(items, 2)
|
||||
assert.Equal("username1", items[0].Username)
|
||||
assert.Equal("username2", items[1].Username)
|
||||
assert.Equal(2, s.Size())
|
||||
|
||||
items = s.Dequeue(40)
|
||||
assert.Len(items, 2)
|
||||
assert.Equal("username3", items[0].Username)
|
||||
assert.Equal("username4", items[1].Username)
|
||||
assert.True(s.IsEmpty())
|
||||
}
|
||||
|
||||
func createQueue() *jobsQueue {
|
||||
queue := &jobsQueue{}
|
||||
queue.New()
|
||||
|
||||
return queue
|
||||
}
|
||||
@@ -1,221 +0,0 @@
|
||||
package queue
|
||||
|
||||
import (
|
||||
"net"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/mono83/slf/wd"
|
||||
|
||||
"github.com/elyby/chrly/api/mojang"
|
||||
)
|
||||
|
||||
var UuidsQueueIterationDelay = 2*time.Second + 500*time.Millisecond
|
||||
|
||||
var usernamesToUuids = mojang.UsernamesToUuids
|
||||
var uuidToTextures = mojang.UuidToTextures
|
||||
var forever = func() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// https://help.mojang.com/customer/portal/articles/928638
|
||||
var allowedUsernamesRegex = regexp.MustCompile(`^[\w_]{3,16}$`)
|
||||
|
||||
type JobsQueue struct {
|
||||
Storage Storage
|
||||
Logger wd.Watchdog
|
||||
|
||||
onFirstCall sync.Once
|
||||
queue jobsQueue
|
||||
broadcast *broadcastMap
|
||||
}
|
||||
|
||||
func (ctx *JobsQueue) GetTexturesForUsername(username string) chan *mojang.SignedTexturesResponse {
|
||||
// TODO: convert username to lower case
|
||||
ctx.onFirstCall.Do(func() {
|
||||
ctx.queue.New()
|
||||
ctx.broadcast = newBroadcaster()
|
||||
ctx.startQueue()
|
||||
})
|
||||
|
||||
responseChan := make(chan *mojang.SignedTexturesResponse)
|
||||
if !allowedUsernamesRegex.MatchString(username) {
|
||||
ctx.Logger.IncCounter("mojang_textures.invalid_username", 1)
|
||||
go func() {
|
||||
responseChan <- nil
|
||||
close(responseChan)
|
||||
}()
|
||||
|
||||
return responseChan
|
||||
}
|
||||
|
||||
ctx.Logger.IncCounter("mojang_textures.request", 1)
|
||||
|
||||
uuid, err := ctx.Storage.GetUuid(username)
|
||||
if err == nil && uuid == "" {
|
||||
ctx.Logger.IncCounter("mojang_textures.usernames.cache_hit_nil", 1)
|
||||
|
||||
go func() {
|
||||
responseChan <- nil
|
||||
close(responseChan)
|
||||
}()
|
||||
|
||||
return responseChan
|
||||
}
|
||||
|
||||
isFirstListener := ctx.broadcast.AddListener(username, responseChan)
|
||||
if isFirstListener {
|
||||
start := time.Now()
|
||||
// TODO: respond nil if processing takes more than 5 seconds
|
||||
|
||||
resultChan := make(chan *mojang.SignedTexturesResponse)
|
||||
if uuid == "" {
|
||||
ctx.Logger.IncCounter("mojang_textures.usernames.queued", 1)
|
||||
ctx.queue.Enqueue(&jobItem{username, resultChan})
|
||||
} else {
|
||||
ctx.Logger.IncCounter("mojang_textures.usernames.cache_hit", 1)
|
||||
go func() {
|
||||
resultChan <- ctx.getTextures(uuid)
|
||||
}()
|
||||
}
|
||||
|
||||
go func() {
|
||||
result := <-resultChan
|
||||
close(resultChan)
|
||||
ctx.broadcast.BroadcastAndRemove(username, result)
|
||||
ctx.Logger.RecordTimer("mojang_textures.result_time", time.Since(start))
|
||||
}()
|
||||
} else {
|
||||
ctx.Logger.IncCounter("mojang_textures.already_in_queue", 1)
|
||||
}
|
||||
|
||||
return responseChan
|
||||
}
|
||||
|
||||
func (ctx *JobsQueue) startQueue() {
|
||||
go func() {
|
||||
time.Sleep(UuidsQueueIterationDelay)
|
||||
for forever() {
|
||||
start := time.Now()
|
||||
ctx.queueRound()
|
||||
elapsed := time.Since(start)
|
||||
ctx.Logger.RecordTimer("mojang_textures.usernames.round_time", elapsed)
|
||||
time.Sleep(UuidsQueueIterationDelay)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func (ctx *JobsQueue) queueRound() {
|
||||
if ctx.queue.IsEmpty() {
|
||||
return
|
||||
}
|
||||
|
||||
queueSize := ctx.queue.Size()
|
||||
jobs := ctx.queue.Dequeue(10)
|
||||
ctx.Logger.UpdateGauge("mojang_textures.usernames.iteration_size", int64(len(jobs)))
|
||||
ctx.Logger.UpdateGauge("mojang_textures.usernames.queue_size", int64(queueSize-len(jobs)))
|
||||
var usernames []string
|
||||
for _, job := range jobs {
|
||||
usernames = append(usernames, job.Username)
|
||||
}
|
||||
|
||||
profiles, err := usernamesToUuids(usernames)
|
||||
if err != nil {
|
||||
ctx.handleResponseError(err, "usernames")
|
||||
for _, job := range jobs {
|
||||
job.RespondTo <- nil
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
for _, job := range jobs {
|
||||
go func(job *jobItem) {
|
||||
var uuid string
|
||||
// The profiles in the response are not ordered, so we must search each username over full array
|
||||
for _, profile := range profiles {
|
||||
if strings.EqualFold(job.Username, profile.Name) {
|
||||
uuid = profile.Id
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
_ = ctx.Storage.StoreUuid(job.Username, uuid)
|
||||
if uuid == "" {
|
||||
job.RespondTo <- nil
|
||||
ctx.Logger.IncCounter("mojang_textures.usernames.uuid_miss", 1)
|
||||
} else {
|
||||
job.RespondTo <- ctx.getTextures(uuid)
|
||||
ctx.Logger.IncCounter("mojang_textures.usernames.uuid_hit", 1)
|
||||
}
|
||||
}(job)
|
||||
}
|
||||
}
|
||||
|
||||
func (ctx *JobsQueue) getTextures(uuid string) *mojang.SignedTexturesResponse {
|
||||
existsTextures, err := ctx.Storage.GetTextures(uuid)
|
||||
if err == nil {
|
||||
ctx.Logger.IncCounter("mojang_textures.textures.cache_hit", 1)
|
||||
return existsTextures
|
||||
}
|
||||
|
||||
ctx.Logger.IncCounter("mojang_textures.textures.request", 1)
|
||||
|
||||
start := time.Now()
|
||||
result, err := uuidToTextures(uuid, true)
|
||||
ctx.Logger.RecordTimer("mojang_textures.textures.request_time", time.Since(start))
|
||||
if err != nil {
|
||||
ctx.handleResponseError(err, "textures")
|
||||
}
|
||||
|
||||
// Mojang can respond with an error, but count it as a hit, so store result even if the textures is nil
|
||||
ctx.Storage.StoreTextures(uuid, result)
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
func (ctx *JobsQueue) handleResponseError(err error, threadName string) {
|
||||
ctx.Logger.Debug(":name: Got response error :err", wd.NameParam(threadName), wd.ErrParam(err))
|
||||
|
||||
switch err.(type) {
|
||||
case mojang.ResponseError:
|
||||
if _, ok := err.(*mojang.BadRequestError); ok {
|
||||
ctx.Logger.Warning(":name: Got 400 Bad Request :err", wd.NameParam(threadName), wd.ErrParam(err))
|
||||
return
|
||||
}
|
||||
|
||||
if _, ok := err.(*mojang.ForbiddenError); ok {
|
||||
ctx.Logger.Warning(":name: Got 403 Forbidden :err", wd.NameParam(threadName), wd.ErrParam(err))
|
||||
return
|
||||
}
|
||||
|
||||
if _, ok := err.(*mojang.TooManyRequestsError); ok {
|
||||
ctx.Logger.Warning(":name: Got 429 Too Many Requests :err", wd.NameParam(threadName), wd.ErrParam(err))
|
||||
return
|
||||
}
|
||||
|
||||
return
|
||||
case net.Error:
|
||||
if err.(net.Error).Timeout() {
|
||||
return
|
||||
}
|
||||
|
||||
if _, ok := err.(*url.Error); ok {
|
||||
return
|
||||
}
|
||||
|
||||
if opErr, ok := err.(*net.OpError); ok && (opErr.Op == "dial" || opErr.Op == "read") {
|
||||
return
|
||||
}
|
||||
|
||||
if err == syscall.ECONNREFUSED {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
ctx.Logger.Emergency(":name: Unknown Mojang response error: :err", wd.NameParam(threadName), wd.ErrParam(err))
|
||||
}
|
||||
@@ -1,525 +0,0 @@
|
||||
package queue
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"net"
|
||||
"net/url"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/elyby/chrly/api/mojang"
|
||||
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/mock"
|
||||
"github.com/stretchr/testify/suite"
|
||||
|
||||
mocks "github.com/elyby/chrly/tests"
|
||||
)
|
||||
|
||||
type mojangApiMocks struct {
|
||||
mock.Mock
|
||||
}
|
||||
|
||||
func (o *mojangApiMocks) UsernamesToUuids(usernames []string) ([]*mojang.ProfileInfo, error) {
|
||||
args := o.Called(usernames)
|
||||
var result []*mojang.ProfileInfo
|
||||
if casted, ok := args.Get(0).([]*mojang.ProfileInfo); ok {
|
||||
result = casted
|
||||
}
|
||||
|
||||
return result, args.Error(1)
|
||||
}
|
||||
|
||||
func (o *mojangApiMocks) UuidToTextures(uuid string, signed bool) (*mojang.SignedTexturesResponse, error) {
|
||||
args := o.Called(uuid, signed)
|
||||
var result *mojang.SignedTexturesResponse
|
||||
if casted, ok := args.Get(0).(*mojang.SignedTexturesResponse); ok {
|
||||
result = casted
|
||||
}
|
||||
|
||||
return result, args.Error(1)
|
||||
}
|
||||
|
||||
type mockStorage struct {
|
||||
mock.Mock
|
||||
}
|
||||
|
||||
func (m *mockStorage) GetUuid(username string) (string, error) {
|
||||
args := m.Called(username)
|
||||
return args.String(0), args.Error(1)
|
||||
}
|
||||
|
||||
func (m *mockStorage) StoreUuid(username string, uuid string) error {
|
||||
args := m.Called(username, uuid)
|
||||
return args.Error(0)
|
||||
}
|
||||
|
||||
func (m *mockStorage) GetTextures(uuid string) (*mojang.SignedTexturesResponse, error) {
|
||||
args := m.Called(uuid)
|
||||
var result *mojang.SignedTexturesResponse
|
||||
if casted, ok := args.Get(0).(*mojang.SignedTexturesResponse); ok {
|
||||
result = casted
|
||||
}
|
||||
|
||||
return result, args.Error(1)
|
||||
}
|
||||
|
||||
func (m *mockStorage) StoreTextures(uuid string, textures *mojang.SignedTexturesResponse) {
|
||||
m.Called(uuid, textures)
|
||||
}
|
||||
|
||||
type queueTestSuite struct {
|
||||
suite.Suite
|
||||
Queue *JobsQueue
|
||||
Storage *mockStorage
|
||||
MojangApi *mojangApiMocks
|
||||
Logger *mocks.WdMock
|
||||
Iterate func()
|
||||
|
||||
iterateChan chan bool
|
||||
done func()
|
||||
}
|
||||
|
||||
func (suite *queueTestSuite) SetupSuite() {
|
||||
UuidsQueueIterationDelay = 0
|
||||
}
|
||||
|
||||
func (suite *queueTestSuite) SetupTest() {
|
||||
suite.Storage = &mockStorage{}
|
||||
suite.Logger = &mocks.WdMock{}
|
||||
|
||||
suite.Queue = &JobsQueue{Storage: suite.Storage, Logger: suite.Logger}
|
||||
|
||||
suite.iterateChan = make(chan bool)
|
||||
forever = func() bool {
|
||||
return <-suite.iterateChan
|
||||
}
|
||||
|
||||
suite.Iterate = func() {
|
||||
suite.iterateChan <- true
|
||||
}
|
||||
|
||||
suite.done = func() {
|
||||
suite.iterateChan <- false
|
||||
}
|
||||
|
||||
suite.MojangApi = new(mojangApiMocks)
|
||||
usernamesToUuids = suite.MojangApi.UsernamesToUuids
|
||||
uuidToTextures = suite.MojangApi.UuidToTextures
|
||||
}
|
||||
|
||||
func (suite *queueTestSuite) TearDownTest() {
|
||||
suite.done()
|
||||
time.Sleep(10 * time.Millisecond) // Add delay to let finish all goroutines before assert mocks calls
|
||||
suite.MojangApi.AssertExpectations(suite.T())
|
||||
suite.Storage.AssertExpectations(suite.T())
|
||||
suite.Logger.AssertExpectations(suite.T())
|
||||
}
|
||||
|
||||
func (suite *queueTestSuite) TestReceiveTexturesForOneUsernameWithoutAnyCache() {
|
||||
expectedResult := &mojang.SignedTexturesResponse{Id: "0d252b7218b648bfb86c2ae476954d32", Name: "maksimkurb"}
|
||||
|
||||
suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Once()
|
||||
suite.Logger.On("IncCounter", "mojang_textures.usernames.queued", int64(1)).Once()
|
||||
suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(1)).Once()
|
||||
suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(0)).Once()
|
||||
suite.Logger.On("RecordTimer", "mojang_textures.usernames.round_time", mock.Anything)
|
||||
suite.Logger.On("IncCounter", "mojang_textures.textures.request", int64(1)).Once()
|
||||
suite.Logger.On("RecordTimer", "mojang_textures.textures.request_time", mock.Anything).Once()
|
||||
suite.Logger.On("IncCounter", "mojang_textures.usernames.uuid_hit", int64(1)).Once()
|
||||
suite.Logger.On("RecordTimer", "mojang_textures.result_time", mock.Anything).Once()
|
||||
|
||||
suite.Storage.On("GetUuid", "maksimkurb").Once().Return("", &ValueNotFound{})
|
||||
suite.Storage.On("StoreUuid", "maksimkurb", "0d252b7218b648bfb86c2ae476954d32").Once().Return(nil)
|
||||
suite.Storage.On("GetTextures", "0d252b7218b648bfb86c2ae476954d32").Once().Return(nil, &ValueNotFound{})
|
||||
suite.Storage.On("StoreTextures", "0d252b7218b648bfb86c2ae476954d32", expectedResult).Once()
|
||||
|
||||
suite.MojangApi.On("UsernamesToUuids", []string{"maksimkurb"}).Once().Return([]*mojang.ProfileInfo{
|
||||
{Id: "0d252b7218b648bfb86c2ae476954d32", Name: "maksimkurb"},
|
||||
}, nil)
|
||||
suite.MojangApi.On("UuidToTextures", "0d252b7218b648bfb86c2ae476954d32", true).Once().Return(expectedResult, nil)
|
||||
|
||||
resultChan := suite.Queue.GetTexturesForUsername("maksimkurb")
|
||||
|
||||
suite.Iterate()
|
||||
|
||||
result := <-resultChan
|
||||
suite.Assert().Equal(expectedResult, result)
|
||||
}
|
||||
|
||||
func (suite *queueTestSuite) TestReceiveTexturesForFewUsernamesWithoutAnyCache() {
|
||||
expectedResult1 := &mojang.SignedTexturesResponse{Id: "0d252b7218b648bfb86c2ae476954d32", Name: "maksimkurb"}
|
||||
expectedResult2 := &mojang.SignedTexturesResponse{Id: "4566e69fc90748ee8d71d7ba5aa00d20", Name: "Thinkofdeath"}
|
||||
|
||||
suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Twice()
|
||||
suite.Logger.On("IncCounter", "mojang_textures.usernames.queued", int64(1)).Twice()
|
||||
suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(2)).Once()
|
||||
suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(0)).Once()
|
||||
suite.Logger.On("RecordTimer", "mojang_textures.usernames.round_time", mock.Anything)
|
||||
suite.Logger.On("IncCounter", "mojang_textures.textures.request", int64(1)).Twice()
|
||||
suite.Logger.On("RecordTimer", "mojang_textures.textures.request_time", mock.Anything).Twice()
|
||||
suite.Logger.On("IncCounter", "mojang_textures.usernames.uuid_hit", int64(1)).Twice()
|
||||
suite.Logger.On("RecordTimer", "mojang_textures.result_time", mock.Anything).Twice()
|
||||
|
||||
suite.Storage.On("GetUuid", "maksimkurb").Once().Return("", &ValueNotFound{})
|
||||
suite.Storage.On("GetUuid", "Thinkofdeath").Once().Return("", &ValueNotFound{})
|
||||
suite.Storage.On("StoreUuid", "maksimkurb", "0d252b7218b648bfb86c2ae476954d32").Once().Return(nil)
|
||||
suite.Storage.On("StoreUuid", "Thinkofdeath", "4566e69fc90748ee8d71d7ba5aa00d20").Once().Return(nil)
|
||||
suite.Storage.On("GetTextures", "0d252b7218b648bfb86c2ae476954d32").Once().Return(nil, &ValueNotFound{})
|
||||
suite.Storage.On("GetTextures", "4566e69fc90748ee8d71d7ba5aa00d20").Once().Return(nil, &ValueNotFound{})
|
||||
suite.Storage.On("StoreTextures", "0d252b7218b648bfb86c2ae476954d32", expectedResult1).Once()
|
||||
suite.Storage.On("StoreTextures", "4566e69fc90748ee8d71d7ba5aa00d20", expectedResult2).Once()
|
||||
|
||||
suite.MojangApi.On("UsernamesToUuids", []string{"maksimkurb", "Thinkofdeath"}).Once().Return([]*mojang.ProfileInfo{
|
||||
{Id: "0d252b7218b648bfb86c2ae476954d32", Name: "maksimkurb"},
|
||||
{Id: "4566e69fc90748ee8d71d7ba5aa00d20", Name: "Thinkofdeath"},
|
||||
}, nil)
|
||||
suite.MojangApi.On("UuidToTextures", "0d252b7218b648bfb86c2ae476954d32", true).Once().Return(expectedResult1, nil)
|
||||
suite.MojangApi.On("UuidToTextures", "4566e69fc90748ee8d71d7ba5aa00d20", true).Once().Return(expectedResult2, nil)
|
||||
|
||||
resultChan1 := suite.Queue.GetTexturesForUsername("maksimkurb")
|
||||
resultChan2 := suite.Queue.GetTexturesForUsername("Thinkofdeath")
|
||||
|
||||
suite.Iterate()
|
||||
|
||||
suite.Assert().Equal(expectedResult1, <-resultChan1)
|
||||
suite.Assert().Equal(expectedResult2, <-resultChan2)
|
||||
}
|
||||
|
||||
func (suite *queueTestSuite) TestReceiveTexturesForUsernameWithCachedUuid() {
|
||||
expectedResult := &mojang.SignedTexturesResponse{Id: "0d252b7218b648bfb86c2ae476954d32", Name: "maksimkurb"}
|
||||
|
||||
suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Once()
|
||||
suite.Logger.On("IncCounter", "mojang_textures.usernames.cache_hit", int64(1)).Once()
|
||||
suite.Logger.On("IncCounter", "mojang_textures.textures.request", int64(1)).Once()
|
||||
suite.Logger.On("RecordTimer", "mojang_textures.textures.request_time", mock.Anything).Once()
|
||||
suite.Logger.On("RecordTimer", "mojang_textures.result_time", mock.Anything).Once()
|
||||
|
||||
suite.Storage.On("GetUuid", "maksimkurb").Once().Return("0d252b7218b648bfb86c2ae476954d32", nil)
|
||||
// Storage.StoreUuid shouldn't be called
|
||||
suite.Storage.On("GetTextures", "0d252b7218b648bfb86c2ae476954d32").Once().Return(nil, &ValueNotFound{})
|
||||
suite.Storage.On("StoreTextures", "0d252b7218b648bfb86c2ae476954d32", expectedResult).Once()
|
||||
|
||||
// MojangApi.UsernamesToUuids shouldn't be called
|
||||
suite.MojangApi.On("UuidToTextures", "0d252b7218b648bfb86c2ae476954d32", true).Once().Return(expectedResult, nil)
|
||||
|
||||
resultChan := suite.Queue.GetTexturesForUsername("maksimkurb")
|
||||
|
||||
// Note that there is no iteration
|
||||
|
||||
result := <-resultChan
|
||||
suite.Assert().Equal(expectedResult, result)
|
||||
}
|
||||
|
||||
func (suite *queueTestSuite) TestReceiveTexturesForUsernameWithFullyCachedResult() {
|
||||
expectedResult := &mojang.SignedTexturesResponse{Id: "0d252b7218b648bfb86c2ae476954d32", Name: "maksimkurb"}
|
||||
|
||||
suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Once()
|
||||
suite.Logger.On("IncCounter", "mojang_textures.usernames.cache_hit", int64(1)).Once()
|
||||
suite.Logger.On("IncCounter", "mojang_textures.textures.cache_hit", int64(1)).Once()
|
||||
suite.Logger.On("RecordTimer", "mojang_textures.result_time", mock.Anything).Once()
|
||||
|
||||
suite.Storage.On("GetUuid", "maksimkurb").Once().Return("0d252b7218b648bfb86c2ae476954d32", nil)
|
||||
// Storage.StoreUuid shouldn't be called
|
||||
suite.Storage.On("GetTextures", "0d252b7218b648bfb86c2ae476954d32").Once().Return(expectedResult, nil)
|
||||
// Storage.StoreTextures shouldn't be called
|
||||
|
||||
// MojangApi.UsernamesToUuids shouldn't be called
|
||||
// MojangApi.UuidToTextures shouldn't be called
|
||||
|
||||
resultChan := suite.Queue.GetTexturesForUsername("maksimkurb")
|
||||
|
||||
// Note that there is no iteration
|
||||
|
||||
result := <-resultChan
|
||||
suite.Assert().Equal(expectedResult, result)
|
||||
}
|
||||
|
||||
func (suite *queueTestSuite) TestReceiveTexturesForUsernameWithCachedUnknownUuid() {
|
||||
suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Once()
|
||||
suite.Logger.On("IncCounter", "mojang_textures.usernames.cache_hit_nil", int64(1)).Once()
|
||||
|
||||
suite.Storage.On("GetUuid", "maksimkurb").Once().Return("", nil)
|
||||
// Storage.StoreUuid shouldn't be called
|
||||
// Storage.GetTextures shouldn't be called
|
||||
// Storage.StoreTextures shouldn't be called
|
||||
|
||||
// MojangApi.UsernamesToUuids shouldn't be called
|
||||
// MojangApi.UuidToTextures shouldn't be called
|
||||
|
||||
resultChan := suite.Queue.GetTexturesForUsername("maksimkurb")
|
||||
|
||||
// Note that there is no iteration
|
||||
|
||||
suite.Assert().Nil(<-resultChan)
|
||||
}
|
||||
|
||||
func (suite *queueTestSuite) TestReceiveTexturesForMoreThan10Usernames() {
|
||||
usernames := make([]string, 12)
|
||||
for i := 0; i < cap(usernames); i++ {
|
||||
usernames[i] = randStr(8)
|
||||
}
|
||||
|
||||
suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Times(12)
|
||||
suite.Logger.On("IncCounter", "mojang_textures.usernames.queued", int64(1)).Times(12)
|
||||
suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(10)).Once()
|
||||
suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(2)).Once()
|
||||
suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(2)).Once()
|
||||
suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(0)).Once()
|
||||
suite.Logger.On("RecordTimer", "mojang_textures.usernames.round_time", mock.Anything).Twice()
|
||||
suite.Logger.On("IncCounter", "mojang_textures.usernames.uuid_miss", int64(1)).Times(12)
|
||||
suite.Logger.On("RecordTimer", "mojang_textures.result_time", mock.Anything).Times(12)
|
||||
|
||||
suite.Storage.On("GetUuid", mock.Anything).Times(12).Return("", &ValueNotFound{})
|
||||
suite.Storage.On("StoreUuid", mock.Anything, "").Times(12).Return(nil) // should be called with "" if username is not compared to uuid
|
||||
// Storage.GetTextures and Storage.SetTextures shouldn't be called
|
||||
|
||||
suite.MojangApi.On("UsernamesToUuids", usernames[0:10]).Once().Return([]*mojang.ProfileInfo{}, nil)
|
||||
suite.MojangApi.On("UsernamesToUuids", usernames[10:12]).Once().Return([]*mojang.ProfileInfo{}, nil)
|
||||
|
||||
channels := make([]chan *mojang.SignedTexturesResponse, 12)
|
||||
for i, username := range usernames {
|
||||
channels[i] = suite.Queue.GetTexturesForUsername(username)
|
||||
}
|
||||
|
||||
suite.Iterate()
|
||||
suite.Iterate()
|
||||
|
||||
for _, channel := range channels {
|
||||
<-channel
|
||||
}
|
||||
}
|
||||
|
||||
func (suite *queueTestSuite) TestReceiveTexturesForTheSameUsernames() {
|
||||
expectedResult := &mojang.SignedTexturesResponse{Id: "0d252b7218b648bfb86c2ae476954d32", Name: "maksimkurb"}
|
||||
|
||||
suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Twice()
|
||||
suite.Logger.On("IncCounter", "mojang_textures.usernames.queued", int64(1)).Once()
|
||||
suite.Logger.On("IncCounter", "mojang_textures.already_in_queue", int64(1)).Once()
|
||||
suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(1)).Once()
|
||||
suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(0)).Once()
|
||||
suite.Logger.On("RecordTimer", "mojang_textures.usernames.round_time", mock.Anything)
|
||||
suite.Logger.On("IncCounter", "mojang_textures.textures.request", int64(1)).Once()
|
||||
suite.Logger.On("RecordTimer", "mojang_textures.textures.request_time", mock.Anything).Once()
|
||||
suite.Logger.On("IncCounter", "mojang_textures.usernames.uuid_hit", int64(1)).Once()
|
||||
suite.Logger.On("RecordTimer", "mojang_textures.result_time", mock.Anything).Once()
|
||||
|
||||
suite.Storage.On("GetUuid", "maksimkurb").Twice().Return("", &ValueNotFound{})
|
||||
suite.Storage.On("StoreUuid", "maksimkurb", "0d252b7218b648bfb86c2ae476954d32").Once().Return(nil)
|
||||
suite.Storage.On("GetTextures", "0d252b7218b648bfb86c2ae476954d32").Once().Return(nil, &ValueNotFound{})
|
||||
suite.Storage.On("StoreTextures", "0d252b7218b648bfb86c2ae476954d32", expectedResult).Once()
|
||||
suite.MojangApi.On("UsernamesToUuids", []string{"maksimkurb"}).Once().Return([]*mojang.ProfileInfo{
|
||||
{Id: "0d252b7218b648bfb86c2ae476954d32", Name: "maksimkurb"},
|
||||
}, nil)
|
||||
suite.MojangApi.On("UuidToTextures", "0d252b7218b648bfb86c2ae476954d32", true).Once().Return(expectedResult, nil)
|
||||
|
||||
resultChan1 := suite.Queue.GetTexturesForUsername("maksimkurb")
|
||||
resultChan2 := suite.Queue.GetTexturesForUsername("maksimkurb")
|
||||
|
||||
suite.Iterate()
|
||||
|
||||
suite.Assert().Equal(expectedResult, <-resultChan1)
|
||||
suite.Assert().Equal(expectedResult, <-resultChan2)
|
||||
}
|
||||
|
||||
func (suite *queueTestSuite) TestReceiveTexturesForUsernameThatAlreadyProcessing() {
|
||||
expectedResult := &mojang.SignedTexturesResponse{Id: "0d252b7218b648bfb86c2ae476954d32", Name: "maksimkurb"}
|
||||
|
||||
suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Twice()
|
||||
suite.Logger.On("IncCounter", "mojang_textures.usernames.queued", int64(1)).Once()
|
||||
suite.Logger.On("IncCounter", "mojang_textures.already_in_queue", int64(1)).Once()
|
||||
suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(1)).Once()
|
||||
suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(0)).Once()
|
||||
suite.Logger.On("RecordTimer", "mojang_textures.usernames.round_time", mock.Anything)
|
||||
suite.Logger.On("IncCounter", "mojang_textures.textures.request", int64(1)).Once()
|
||||
suite.Logger.On("RecordTimer", "mojang_textures.textures.request_time", mock.Anything).Once()
|
||||
suite.Logger.On("IncCounter", "mojang_textures.usernames.uuid_hit", int64(1)).Once()
|
||||
suite.Logger.On("RecordTimer", "mojang_textures.result_time", mock.Anything).Once()
|
||||
|
||||
suite.Storage.On("GetUuid", "maksimkurb").Twice().Return("", &ValueNotFound{})
|
||||
suite.Storage.On("StoreUuid", "maksimkurb", "0d252b7218b648bfb86c2ae476954d32").Once().Return(nil)
|
||||
suite.Storage.On("GetTextures", "0d252b7218b648bfb86c2ae476954d32").Once().Return(nil, &ValueNotFound{})
|
||||
suite.Storage.On("StoreTextures", "0d252b7218b648bfb86c2ae476954d32", expectedResult).Once()
|
||||
|
||||
suite.MojangApi.On("UsernamesToUuids", []string{"maksimkurb"}).Once().Return([]*mojang.ProfileInfo{
|
||||
{Id: "0d252b7218b648bfb86c2ae476954d32", Name: "maksimkurb"},
|
||||
}, nil)
|
||||
suite.MojangApi.On("UuidToTextures", "0d252b7218b648bfb86c2ae476954d32", true).
|
||||
Once().
|
||||
After(10*time.Millisecond). // Simulate long round trip
|
||||
Return(expectedResult, nil)
|
||||
|
||||
resultChan1 := suite.Queue.GetTexturesForUsername("maksimkurb")
|
||||
|
||||
// Note that for entire test there is only one iteration
|
||||
suite.Iterate()
|
||||
|
||||
// Let it meet delayed UuidToTextures request
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
|
||||
resultChan2 := suite.Queue.GetTexturesForUsername("maksimkurb")
|
||||
|
||||
suite.Assert().Equal(expectedResult, <-resultChan1)
|
||||
suite.Assert().Equal(expectedResult, <-resultChan2)
|
||||
}
|
||||
|
||||
func (suite *queueTestSuite) TestDoNothingWhenNoTasks() {
|
||||
suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Once()
|
||||
suite.Logger.On("IncCounter", "mojang_textures.usernames.queued", int64(1)).Once()
|
||||
suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(1)).Once()
|
||||
suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(0)).Once()
|
||||
suite.Logger.On("RecordTimer", "mojang_textures.usernames.round_time", mock.Anything)
|
||||
suite.Logger.On("IncCounter", "mojang_textures.usernames.uuid_miss", int64(1)).Once()
|
||||
suite.Logger.On("RecordTimer", "mojang_textures.result_time", mock.Anything).Once()
|
||||
|
||||
suite.Storage.On("GetUuid", "maksimkurb").Once().Return("", &ValueNotFound{})
|
||||
suite.Storage.On("StoreUuid", "maksimkurb", "").Once().Return(nil)
|
||||
// Storage.GetTextures and Storage.StoreTextures shouldn't be called
|
||||
|
||||
suite.MojangApi.On("UsernamesToUuids", []string{"maksimkurb"}).Once().Return([]*mojang.ProfileInfo{}, nil)
|
||||
|
||||
// Perform first iteration and await it finish
|
||||
resultChan := suite.Queue.GetTexturesForUsername("maksimkurb")
|
||||
|
||||
suite.Iterate()
|
||||
|
||||
suite.Assert().Nil(<-resultChan)
|
||||
|
||||
// Let it to perform a few more iterations to ensure, that there is no calls to external APIs
|
||||
suite.Iterate()
|
||||
suite.Iterate()
|
||||
}
|
||||
|
||||
type timeoutError struct {
|
||||
}
|
||||
|
||||
func (*timeoutError) Error() string { return "timeout error" }
|
||||
func (*timeoutError) Timeout() bool { return true }
|
||||
func (*timeoutError) Temporary() bool { return false }
|
||||
|
||||
var expectedErrors = []error{
|
||||
&mojang.BadRequestError{},
|
||||
&mojang.ForbiddenError{},
|
||||
&mojang.TooManyRequestsError{},
|
||||
&mojang.ServerError{},
|
||||
&timeoutError{},
|
||||
&url.Error{Op: "GET", URL: "http://localhost"},
|
||||
&net.OpError{Op: "read"},
|
||||
&net.OpError{Op: "dial"},
|
||||
syscall.ECONNREFUSED,
|
||||
}
|
||||
|
||||
func (suite *queueTestSuite) TestShouldNotLogErrorWhenExpectedErrorReturnedFromUsernameToUuidRequest() {
|
||||
suite.Logger.On("IncCounter", mock.Anything, mock.Anything)
|
||||
suite.Logger.On("UpdateGauge", mock.Anything, mock.Anything)
|
||||
suite.Logger.On("RecordTimer", mock.Anything, mock.Anything)
|
||||
suite.Logger.On("Debug", ":name: Got response error :err", mock.Anything, mock.Anything).Times(len(expectedErrors))
|
||||
suite.Logger.On("Warning", ":name: Got 400 Bad Request :err", mock.Anything, mock.Anything).Once()
|
||||
suite.Logger.On("Warning", ":name: Got 403 Forbidden :err", mock.Anything, mock.Anything).Once()
|
||||
suite.Logger.On("Warning", ":name: Got 429 Too Many Requests :err", mock.Anything, mock.Anything).Once()
|
||||
|
||||
suite.Storage.On("GetUuid", "maksimkurb").Return("", &ValueNotFound{})
|
||||
|
||||
for _, err := range expectedErrors {
|
||||
suite.MojangApi.On("UsernamesToUuids", []string{"maksimkurb"}).Once().Return(nil, err)
|
||||
|
||||
resultChan := suite.Queue.GetTexturesForUsername("maksimkurb")
|
||||
suite.Iterate()
|
||||
suite.Assert().Nil(<-resultChan)
|
||||
suite.MojangApi.AssertExpectations(suite.T())
|
||||
suite.MojangApi.ExpectedCalls = nil // https://github.com/stretchr/testify/issues/558#issuecomment-372112364
|
||||
}
|
||||
}
|
||||
|
||||
func (suite *queueTestSuite) TestShouldLogEmergencyOnUnexpectedErrorReturnedFromUsernameToUuidRequest() {
|
||||
suite.Logger.On("IncCounter", mock.Anything, mock.Anything)
|
||||
suite.Logger.On("UpdateGauge", mock.Anything, mock.Anything)
|
||||
suite.Logger.On("RecordTimer", mock.Anything, mock.Anything)
|
||||
suite.Logger.On("Debug", ":name: Got response error :err", mock.Anything, mock.Anything).Once()
|
||||
suite.Logger.On("Emergency", ":name: Unknown Mojang response error: :err", mock.Anything, mock.Anything).Once()
|
||||
|
||||
suite.Storage.On("GetUuid", "maksimkurb").Return("", &ValueNotFound{})
|
||||
|
||||
suite.MojangApi.On("UsernamesToUuids", []string{"maksimkurb"}).Once().Return(nil, errors.New("unexpected error"))
|
||||
|
||||
resultChan := suite.Queue.GetTexturesForUsername("maksimkurb")
|
||||
suite.Iterate()
|
||||
suite.Assert().Nil(<-resultChan)
|
||||
}
|
||||
|
||||
func (suite *queueTestSuite) TestShouldNotLogErrorWhenExpectedErrorReturnedFromUuidToTexturesRequest() {
|
||||
suite.Logger.On("IncCounter", mock.Anything, mock.Anything)
|
||||
suite.Logger.On("UpdateGauge", mock.Anything, mock.Anything)
|
||||
suite.Logger.On("RecordTimer", mock.Anything, mock.Anything)
|
||||
suite.Logger.On("Debug", ":name: Got response error :err", mock.Anything, mock.Anything).Times(len(expectedErrors))
|
||||
suite.Logger.On("Warning", ":name: Got 400 Bad Request :err", mock.Anything, mock.Anything).Once()
|
||||
suite.Logger.On("Warning", ":name: Got 403 Forbidden :err", mock.Anything, mock.Anything).Once()
|
||||
suite.Logger.On("Warning", ":name: Got 429 Too Many Requests :err", mock.Anything, mock.Anything).Once()
|
||||
|
||||
suite.Storage.On("GetUuid", "maksimkurb").Return("", &ValueNotFound{})
|
||||
suite.Storage.On("StoreUuid", "maksimkurb", "0d252b7218b648bfb86c2ae476954d32").Return(nil)
|
||||
suite.Storage.On("GetTextures", "0d252b7218b648bfb86c2ae476954d32").Return(nil, &ValueNotFound{})
|
||||
suite.Storage.On("StoreTextures", "0d252b7218b648bfb86c2ae476954d32", (*mojang.SignedTexturesResponse)(nil))
|
||||
|
||||
for _, err := range expectedErrors {
|
||||
suite.MojangApi.On("UsernamesToUuids", []string{"maksimkurb"}).Once().Return([]*mojang.ProfileInfo{
|
||||
{Id: "0d252b7218b648bfb86c2ae476954d32", Name: "maksimkurb"},
|
||||
}, nil)
|
||||
suite.MojangApi.On("UuidToTextures", "0d252b7218b648bfb86c2ae476954d32", true).Once().Return(nil, err)
|
||||
|
||||
resultChan := suite.Queue.GetTexturesForUsername("maksimkurb")
|
||||
suite.Iterate()
|
||||
suite.Assert().Nil(<-resultChan)
|
||||
suite.MojangApi.AssertExpectations(suite.T())
|
||||
suite.MojangApi.ExpectedCalls = nil // https://github.com/stretchr/testify/issues/558#issuecomment-372112364
|
||||
}
|
||||
}
|
||||
|
||||
func (suite *queueTestSuite) TestShouldLogEmergencyOnUnexpectedErrorReturnedFromUuidToTexturesRequest() {
|
||||
suite.Logger.On("IncCounter", mock.Anything, mock.Anything)
|
||||
suite.Logger.On("UpdateGauge", mock.Anything, mock.Anything)
|
||||
suite.Logger.On("RecordTimer", mock.Anything, mock.Anything)
|
||||
suite.Logger.On("Debug", ":name: Got response error :err", mock.Anything, mock.Anything).Once()
|
||||
suite.Logger.On("Emergency", ":name: Unknown Mojang response error: :err", mock.Anything, mock.Anything).Once()
|
||||
|
||||
suite.Storage.On("GetUuid", "maksimkurb").Return("", &ValueNotFound{})
|
||||
suite.Storage.On("StoreUuid", "maksimkurb", "0d252b7218b648bfb86c2ae476954d32").Return(nil)
|
||||
suite.Storage.On("GetTextures", "0d252b7218b648bfb86c2ae476954d32").Return(nil, &ValueNotFound{})
|
||||
suite.Storage.On("StoreTextures", "0d252b7218b648bfb86c2ae476954d32", (*mojang.SignedTexturesResponse)(nil))
|
||||
|
||||
suite.MojangApi.On("UsernamesToUuids", []string{"maksimkurb"}).Once().Return([]*mojang.ProfileInfo{
|
||||
{Id: "0d252b7218b648bfb86c2ae476954d32", Name: "maksimkurb"},
|
||||
}, nil)
|
||||
suite.MojangApi.On("UuidToTextures", "0d252b7218b648bfb86c2ae476954d32", true).Once().Return(nil, errors.New("unexpected error"))
|
||||
|
||||
resultChan := suite.Queue.GetTexturesForUsername("maksimkurb")
|
||||
suite.Iterate()
|
||||
suite.Assert().Nil(<-resultChan)
|
||||
}
|
||||
|
||||
func (suite *queueTestSuite) TestReceiveTexturesForNotAllowedMojangUsername() {
|
||||
suite.Logger.On("IncCounter", "mojang_textures.invalid_username", int64(1)).Once()
|
||||
|
||||
resultChan := suite.Queue.GetTexturesForUsername("Not allowed")
|
||||
suite.Assert().Nil(<-resultChan)
|
||||
}
|
||||
|
||||
func TestJobsQueueSuite(t *testing.T) {
|
||||
suite.Run(t, new(queueTestSuite))
|
||||
}
|
||||
|
||||
var replacer = strings.NewReplacer("-", "_", "=", "")
|
||||
|
||||
// https://stackoverflow.com/a/50581165
|
||||
func randStr(len int) string {
|
||||
buff := make([]byte, len)
|
||||
_, _ = rand.Read(buff)
|
||||
str := replacer.Replace(base64.URLEncoding.EncodeToString(buff))
|
||||
|
||||
// Base 64 can be longer than len
|
||||
return str[:len]
|
||||
}
|
||||
@@ -1,53 +0,0 @@
|
||||
package queue
|
||||
|
||||
import "github.com/elyby/chrly/api/mojang"
|
||||
|
||||
type UuidsStorage interface {
|
||||
GetUuid(username string) (string, error)
|
||||
StoreUuid(username string, uuid string) error
|
||||
}
|
||||
|
||||
// nil value can be passed to the storage to indicate that there is no textures
|
||||
// for uuid and we know about it. Return err only in case, when storage completely
|
||||
// unable to load any information about textures
|
||||
type TexturesStorage interface {
|
||||
GetTextures(uuid string) (*mojang.SignedTexturesResponse, error)
|
||||
StoreTextures(uuid string, textures *mojang.SignedTexturesResponse)
|
||||
}
|
||||
|
||||
type Storage interface {
|
||||
UuidsStorage
|
||||
TexturesStorage
|
||||
}
|
||||
|
||||
// SplittedStorage allows you to use separate storage engines to satisfy
|
||||
// the Storage interface
|
||||
type SplittedStorage struct {
|
||||
UuidsStorage
|
||||
TexturesStorage
|
||||
}
|
||||
|
||||
func (s *SplittedStorage) GetUuid(username string) (string, error) {
|
||||
return s.UuidsStorage.GetUuid(username)
|
||||
}
|
||||
|
||||
func (s *SplittedStorage) StoreUuid(username string, uuid string) error {
|
||||
return s.UuidsStorage.StoreUuid(username, uuid)
|
||||
}
|
||||
|
||||
func (s *SplittedStorage) GetTextures(uuid string) (*mojang.SignedTexturesResponse, error) {
|
||||
return s.TexturesStorage.GetTextures(uuid)
|
||||
}
|
||||
|
||||
func (s *SplittedStorage) StoreTextures(uuid string, textures *mojang.SignedTexturesResponse) {
|
||||
s.TexturesStorage.StoreTextures(uuid, textures)
|
||||
}
|
||||
|
||||
// This error can be used to indicate, that requested
|
||||
// value doesn't exists in the storage
|
||||
type ValueNotFound struct {
|
||||
}
|
||||
|
||||
func (*ValueNotFound) Error() string {
|
||||
return "value not found in the storage"
|
||||
}
|
||||
@@ -1,89 +0,0 @@
|
||||
package queue
|
||||
|
||||
import (
|
||||
"github.com/elyby/chrly/api/mojang"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/mock"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type uuidsStorageMock struct {
|
||||
mock.Mock
|
||||
}
|
||||
|
||||
func (m *uuidsStorageMock) GetUuid(username string) (string, error) {
|
||||
args := m.Called(username)
|
||||
return args.String(0), args.Error(1)
|
||||
}
|
||||
|
||||
func (m *uuidsStorageMock) StoreUuid(username string, uuid string) error {
|
||||
m.Called(username, uuid)
|
||||
return nil
|
||||
}
|
||||
|
||||
type texturesStorageMock struct {
|
||||
mock.Mock
|
||||
}
|
||||
|
||||
func (m *texturesStorageMock) GetTextures(uuid string) (*mojang.SignedTexturesResponse, error) {
|
||||
args := m.Called(uuid)
|
||||
var result *mojang.SignedTexturesResponse
|
||||
if casted, ok := args.Get(0).(*mojang.SignedTexturesResponse); ok {
|
||||
result = casted
|
||||
}
|
||||
|
||||
return result, args.Error(1)
|
||||
}
|
||||
|
||||
func (m *texturesStorageMock) StoreTextures(uuid string, textures *mojang.SignedTexturesResponse) {
|
||||
m.Called(uuid, textures)
|
||||
}
|
||||
|
||||
func TestSplittedStorage(t *testing.T) {
|
||||
createMockedStorage := func() (*SplittedStorage, *uuidsStorageMock, *texturesStorageMock) {
|
||||
uuidsStorage := &uuidsStorageMock{}
|
||||
texturesStorage := &texturesStorageMock{}
|
||||
|
||||
return &SplittedStorage{uuidsStorage, texturesStorage}, uuidsStorage, texturesStorage
|
||||
}
|
||||
|
||||
t.Run("GetUuid", func(t *testing.T) {
|
||||
storage, uuidsMock, _ := createMockedStorage()
|
||||
uuidsMock.On("GetUuid", "username").Once().Return("find me", nil)
|
||||
result, err := storage.GetUuid("username")
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, "find me", result)
|
||||
uuidsMock.AssertExpectations(t)
|
||||
})
|
||||
|
||||
t.Run("StoreUuid", func(t *testing.T) {
|
||||
storage, uuidsMock, _ := createMockedStorage()
|
||||
uuidsMock.On("StoreUuid", "username", "result").Once()
|
||||
_ = storage.StoreUuid("username", "result")
|
||||
uuidsMock.AssertExpectations(t)
|
||||
})
|
||||
|
||||
t.Run("GetTextures", func(t *testing.T) {
|
||||
result := &mojang.SignedTexturesResponse{Id: "mock id"}
|
||||
storage, _, texturesMock := createMockedStorage()
|
||||
texturesMock.On("GetTextures", "uuid").Once().Return(result, nil)
|
||||
returned, err := storage.GetTextures("uuid")
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, result, returned)
|
||||
texturesMock.AssertExpectations(t)
|
||||
})
|
||||
|
||||
t.Run("StoreTextures", func(t *testing.T) {
|
||||
toStore := &mojang.SignedTexturesResponse{}
|
||||
storage, _, texturesMock := createMockedStorage()
|
||||
texturesMock.On("StoreTextures", "mock id", toStore).Once()
|
||||
storage.StoreTextures("mock id", toStore)
|
||||
texturesMock.AssertExpectations(t)
|
||||
})
|
||||
}
|
||||
|
||||
func TestValueNotFound_Error(t *testing.T) {
|
||||
err := &ValueNotFound{}
|
||||
assert.Equal(t, "value not found in the storage", err.Error())
|
||||
}
|
||||
Reference in New Issue
Block a user