ElyBy-Mirror/chrly
1
0
Fork 0
mirror of https://github.com/elyby/chrly.git synced 2025-05-31 14:11:51 +05:30
Code Issues Packages Projects Releases Wiki Activity

Completely rework mojang textures queue implementation, split it across separate data providers

Browse Source
This commit is contained in:
ErickSkrauch
2019-11-21 01:33:13 +03:00
parent 6fd88e077e
commit 0644dfe021
34 changed files with 1405 additions and 1180 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
CHANGELOG.md
View File

@@ -5,14 +5,26 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [Unreleased] - xxxx-xx-xx ## [Unreleased] - xxxx-xx-xx
### Added
- New StatsD metrics:
- Counters:
- `ely.skinsystem.{hostname}.app.mojang_textures.usernames.textures_hit`
- `ely.skinsystem.{hostname}.app.mojang_textures.usernames.textures_miss`
### Fixed
- `ely.skinsystem.{hostname}.app.mojang_textures.usernames.iteration_size` and
`ely.skinsystem.{hostname}.app.mojang_textures.usernames.queue_size` are now updated even if the queue is empty.
### Changed
- Event `ely.skinsystem.{hostname}.app.mojang_textures.already_in_queue` has been renamed into `ely.skinsystem.{hostname}.app.mojang_textures.already_scheduled`.
## [4.3.0] - 2019-11-08 ## [4.3.0] - 2019-11-08
### Added ### Added
- 403 Forbidden errors from the Mojang's API are now logged - 403 Forbidden errors from the Mojang's API are now logged.
- `QUEUE_LOOP_DELAY` configuration param to adjust Mojang's textures queue performance - `QUEUE_LOOP_DELAY` configuration param to adjust Mojang's textures queue performance.
### Changed ### Changed
- Mojang's textures queue loop is now has an iteration delay of 2.5 seconds (was 1) - Mojang's textures queue loop is now has an iteration delay of 2.5 seconds (was 1).
- Bumped Go version to 1.13. - Bumped Go version to 1.13.
## [4.2.3] - 2019-10-03 ## [4.2.3] - 2019-10-03

53
api/mojang/queue/broadcast.go
View File

@@ -1,53 +0,0 @@
package queue
import (
"sync"
"github.com/elyby/chrly/api/mojang"
)
type broadcastMap struct {
lock sync.Mutex
listeners map[string][]chan *mojang.SignedTexturesResponse
}
func newBroadcaster() *broadcastMap {
return &broadcastMap{
listeners: make(map[string][]chan *mojang.SignedTexturesResponse),
}
}
// Returns a boolean value, which will be true if the username passed didn't exist before
func (c *broadcastMap) AddListener(username string, resultChan chan *mojang.SignedTexturesResponse) bool {
c.lock.Lock()
defer c.lock.Unlock()
val, alreadyHasSource := c.listeners[username]
if alreadyHasSource {
c.listeners[username] = append(val, resultChan)
return false
}
c.listeners[username] = []chan *mojang.SignedTexturesResponse{resultChan}
return true
}
func (c *broadcastMap) BroadcastAndRemove(username string, result *mojang.SignedTexturesResponse) {
c.lock.Lock()
defer c.lock.Unlock()
val, ok := c.listeners[username]
if !ok {
return
}
for _, channel := range val {
go func(channel chan *mojang.SignedTexturesResponse) {
channel <- result
close(channel)
}(channel)
}
delete(c.listeners, username)
}

75
api/mojang/queue/broadcast_test.go
View File

@@ -1,75 +0,0 @@
package queue
import (
"github.com/elyby/chrly/api/mojang"
testify "github.com/stretchr/testify/assert"
"testing"
)
func TestBroadcastMap_GetOrAppend(t *testing.T) {
t.Run("first call when username didn't exist before should return true", func(t *testing.T) {
assert := testify.New(t)
broadcaster := newBroadcaster()
channel := make(chan *mojang.SignedTexturesResponse)
isFirstListener := broadcaster.AddListener("mock", channel)
assert.True(isFirstListener)
listeners, ok := broadcaster.listeners["mock"]
assert.True(ok)
assert.Len(listeners, 1)
assert.Equal(channel, listeners[0])
})
t.Run("subsequent calls should return false", func(t *testing.T) {
assert := testify.New(t)
broadcaster := newBroadcaster()
channel1 := make(chan *mojang.SignedTexturesResponse)
isFirstListener := broadcaster.AddListener("mock", channel1)
assert.True(isFirstListener)
channel2 := make(chan *mojang.SignedTexturesResponse)
isFirstListener = broadcaster.AddListener("mock", channel2)
assert.False(isFirstListener)
channel3 := make(chan *mojang.SignedTexturesResponse)
isFirstListener = broadcaster.AddListener("mock", channel3)
assert.False(isFirstListener)
})
}
func TestBroadcastMap_BroadcastAndRemove(t *testing.T) {
t.Run("should broadcast to all listeners and remove the key", func(t *testing.T) {
assert := testify.New(t)
broadcaster := newBroadcaster()
channel1 := make(chan *mojang.SignedTexturesResponse)
channel2 := make(chan *mojang.SignedTexturesResponse)
broadcaster.AddListener("mock", channel1)
broadcaster.AddListener("mock", channel2)
result := &mojang.SignedTexturesResponse{Id: "mockUuid"}
broadcaster.BroadcastAndRemove("mock", result)
assert.Equal(result, <-channel1)
assert.Equal(result, <-channel2)
channel3 := make(chan *mojang.SignedTexturesResponse)
isFirstListener := broadcaster.AddListener("mock", channel3)
assert.True(isFirstListener)
})
t.Run("call on not exists username", func(t *testing.T) {
assert := testify.New(t)
assert.NotPanics(func() {
broadcaster := newBroadcaster()
broadcaster.BroadcastAndRemove("mock", &mojang.SignedTexturesResponse{})
})
})
}

56
api/mojang/queue/jobs_structure.go
View File

@@ -1,56 +0,0 @@
// Based on the implementation from https://flaviocopes.com/golang-data-structure-queue/
package queue
import (
"sync"
"github.com/elyby/chrly/api/mojang"
)
type jobItem struct {
Username string
RespondTo chan *mojang.SignedTexturesResponse
}
type jobsQueue struct {
lock sync.Mutex
items []*jobItem
}
func (s *jobsQueue) New() *jobsQueue {
s.items = []*jobItem{}
return s
}
func (s *jobsQueue) Enqueue(t *jobItem) {
s.lock.Lock()
defer s.lock.Unlock()
s.items = append(s.items, t)
}
func (s *jobsQueue) Dequeue(n int) []*jobItem {
s.lock.Lock()
defer s.lock.Unlock()
if n > s.Size() {
n = s.Size()
}
items := s.items[0:n]
s.items = s.items[n:len(s.items)]
return items
}
func (s *jobsQueue) IsEmpty() bool {
s.lock.Lock()
defer s.lock.Unlock()
return len(s.items) == 0
}
func (s *jobsQueue) Size() int {
return len(s.items)
}

47
api/mojang/queue/jobs_structure_test.go
View File

@@ -1,47 +0,0 @@
package queue
import (
"testing"
testify "github.com/stretchr/testify/assert"
)
func TestEnqueue(t *testing.T) {
assert := testify.New(t)
s := createQueue()
s.Enqueue(&jobItem{Username: "username1"})
s.Enqueue(&jobItem{Username: "username2"})
s.Enqueue(&jobItem{Username: "username3"})
assert.Equal(3, s.Size())
}
func TestDequeueN(t *testing.T) {
assert := testify.New(t)
s := createQueue()
s.Enqueue(&jobItem{Username: "username1"})
s.Enqueue(&jobItem{Username: "username2"})
s.Enqueue(&jobItem{Username: "username3"})
s.Enqueue(&jobItem{Username: "username4"})
items := s.Dequeue(2)
assert.Len(items, 2)
assert.Equal("username1", items[0].Username)
assert.Equal("username2", items[1].Username)
assert.Equal(2, s.Size())
items = s.Dequeue(40)
assert.Len(items, 2)
assert.Equal("username3", items[0].Username)
assert.Equal("username4", items[1].Username)
assert.True(s.IsEmpty())
}
func createQueue() *jobsQueue {
queue := &jobsQueue{}
queue.New()
return queue
}

221
api/mojang/queue/queue.go
View File

@@ -1,221 +0,0 @@
package queue
import (
"net"
"net/url"
"regexp"
"strings"
"sync"
"syscall"
"time"
"github.com/mono83/slf/wd"
"github.com/elyby/chrly/api/mojang"
)
var UuidsQueueIterationDelay = 2*time.Second + 500*time.Millisecond
var usernamesToUuids = mojang.UsernamesToUuids
var uuidToTextures = mojang.UuidToTextures
var forever = func() bool {
return true
}
// https://help.mojang.com/customer/portal/articles/928638
var allowedUsernamesRegex = regexp.MustCompile(`^[\w_]{3,16}$`)
type JobsQueue struct {
Storage Storage
Logger wd.Watchdog
onFirstCall sync.Once
queue jobsQueue
broadcast *broadcastMap
}
func (ctx *JobsQueue) GetTexturesForUsername(username string) chan *mojang.SignedTexturesResponse {
// TODO: convert username to lower case
ctx.onFirstCall.Do(func() {
ctx.queue.New()
ctx.broadcast = newBroadcaster()
ctx.startQueue()
})
responseChan := make(chan *mojang.SignedTexturesResponse)
if !allowedUsernamesRegex.MatchString(username) {
ctx.Logger.IncCounter("mojang_textures.invalid_username", 1)
go func() {
responseChan <- nil
close(responseChan)
}()
return responseChan
}
ctx.Logger.IncCounter("mojang_textures.request", 1)
uuid, err := ctx.Storage.GetUuid(username)
if err == nil && uuid == "" {
ctx.Logger.IncCounter("mojang_textures.usernames.cache_hit_nil", 1)
go func() {
responseChan <- nil
close(responseChan)
}()
return responseChan
}
isFirstListener := ctx.broadcast.AddListener(username, responseChan)
if isFirstListener {
start := time.Now()
// TODO: respond nil if processing takes more than 5 seconds
resultChan := make(chan *mojang.SignedTexturesResponse)
if uuid == "" {
ctx.Logger.IncCounter("mojang_textures.usernames.queued", 1)
ctx.queue.Enqueue(&jobItem{username, resultChan})
} else {
ctx.Logger.IncCounter("mojang_textures.usernames.cache_hit", 1)
go func() {
resultChan <- ctx.getTextures(uuid)
}()
}
go func() {
result := <-resultChan
close(resultChan)
ctx.broadcast.BroadcastAndRemove(username, result)
ctx.Logger.RecordTimer("mojang_textures.result_time", time.Since(start))
}()
} else {
ctx.Logger.IncCounter("mojang_textures.already_in_queue", 1)
}
return responseChan
}
func (ctx *JobsQueue) startQueue() {
go func() {
time.Sleep(UuidsQueueIterationDelay)
for forever() {
start := time.Now()
ctx.queueRound()
elapsed := time.Since(start)
ctx.Logger.RecordTimer("mojang_textures.usernames.round_time", elapsed)
time.Sleep(UuidsQueueIterationDelay)
}
}()
}
func (ctx *JobsQueue) queueRound() {
if ctx.queue.IsEmpty() {
return
}
queueSize := ctx.queue.Size()
jobs := ctx.queue.Dequeue(10)
ctx.Logger.UpdateGauge("mojang_textures.usernames.iteration_size", int64(len(jobs)))
ctx.Logger.UpdateGauge("mojang_textures.usernames.queue_size", int64(queueSize-len(jobs)))
var usernames []string
for _, job := range jobs {
usernames = append(usernames, job.Username)
}
profiles, err := usernamesToUuids(usernames)
if err != nil {
ctx.handleResponseError(err, "usernames")
for _, job := range jobs {
job.RespondTo <- nil
}
return
}
for _, job := range jobs {
go func(job *jobItem) {
var uuid string
// The profiles in the response are not ordered, so we must search each username over full array
for _, profile := range profiles {
if strings.EqualFold(job.Username, profile.Name) {
uuid = profile.Id
break
}
}
_ = ctx.Storage.StoreUuid(job.Username, uuid)
if uuid == "" {
job.RespondTo <- nil
ctx.Logger.IncCounter("mojang_textures.usernames.uuid_miss", 1)
} else {
job.RespondTo <- ctx.getTextures(uuid)
ctx.Logger.IncCounter("mojang_textures.usernames.uuid_hit", 1)
}
}(job)
}
}
func (ctx *JobsQueue) getTextures(uuid string) *mojang.SignedTexturesResponse {
existsTextures, err := ctx.Storage.GetTextures(uuid)
if err == nil {
ctx.Logger.IncCounter("mojang_textures.textures.cache_hit", 1)
return existsTextures
}
ctx.Logger.IncCounter("mojang_textures.textures.request", 1)
start := time.Now()
result, err := uuidToTextures(uuid, true)
ctx.Logger.RecordTimer("mojang_textures.textures.request_time", time.Since(start))
if err != nil {
ctx.handleResponseError(err, "textures")
}
// Mojang can respond with an error, but count it as a hit, so store result even if the textures is nil
ctx.Storage.StoreTextures(uuid, result)
return result
}
func (ctx *JobsQueue) handleResponseError(err error, threadName string) {
ctx.Logger.Debug(":name: Got response error :err", wd.NameParam(threadName), wd.ErrParam(err))
switch err.(type) {
case mojang.ResponseError:
if _, ok := err.(*mojang.BadRequestError); ok {
ctx.Logger.Warning(":name: Got 400 Bad Request :err", wd.NameParam(threadName), wd.ErrParam(err))
return
}
if _, ok := err.(*mojang.ForbiddenError); ok {
ctx.Logger.Warning(":name: Got 403 Forbidden :err", wd.NameParam(threadName), wd.ErrParam(err))
return
}
if _, ok := err.(*mojang.TooManyRequestsError); ok {
ctx.Logger.Warning(":name: Got 429 Too Many Requests :err", wd.NameParam(threadName), wd.ErrParam(err))
return
}
return
case net.Error:
if err.(net.Error).Timeout() {
return
}
if _, ok := err.(*url.Error); ok {
return
}
if opErr, ok := err.(*net.OpError); ok && (opErr.Op == "dial" || opErr.Op == "read") {
return
}
if err == syscall.ECONNREFUSED {
return
}
}
ctx.Logger.Emergency(":name: Unknown Mojang response error: :err", wd.NameParam(threadName), wd.ErrParam(err))
}

525
api/mojang/queue/queue_test.go
View File

@@ -1,525 +0,0 @@
package queue
import (
"crypto/rand"
"encoding/base64"
"errors"
"net"
"net/url"
"strings"
"syscall"
"time"
"github.com/elyby/chrly/api/mojang"
"testing"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/suite"
mocks "github.com/elyby/chrly/tests"
)
type mojangApiMocks struct {
mock.Mock
}
func (o *mojangApiMocks) UsernamesToUuids(usernames []string) ([]*mojang.ProfileInfo, error) {
args := o.Called(usernames)
var result []*mojang.ProfileInfo
if casted, ok := args.Get(0).([]*mojang.ProfileInfo); ok {
result = casted
}
return result, args.Error(1)
}
func (o *mojangApiMocks) UuidToTextures(uuid string, signed bool) (*mojang.SignedTexturesResponse, error) {
args := o.Called(uuid, signed)
var result *mojang.SignedTexturesResponse
if casted, ok := args.Get(0).(*mojang.SignedTexturesResponse); ok {
result = casted
}
return result, args.Error(1)
}
type mockStorage struct {
mock.Mock
}
func (m *mockStorage) GetUuid(username string) (string, error) {
args := m.Called(username)
return args.String(0), args.Error(1)
}
func (m *mockStorage) StoreUuid(username string, uuid string) error {
args := m.Called(username, uuid)
return args.Error(0)
}
func (m *mockStorage) GetTextures(uuid string) (*mojang.SignedTexturesResponse, error) {
args := m.Called(uuid)
var result *mojang.SignedTexturesResponse
if casted, ok := args.Get(0).(*mojang.SignedTexturesResponse); ok {
result = casted
}
return result, args.Error(1)
}
func (m *mockStorage) StoreTextures(uuid string, textures *mojang.SignedTexturesResponse) {
m.Called(uuid, textures)
}
type queueTestSuite struct {
suite.Suite
Queue *JobsQueue
Storage *mockStorage
MojangApi *mojangApiMocks
Logger *mocks.WdMock
Iterate func()
iterateChan chan bool
done func()
}
func (suite *queueTestSuite) SetupSuite() {
UuidsQueueIterationDelay = 0
}
func (suite *queueTestSuite) SetupTest() {
suite.Storage = &mockStorage{}
suite.Logger = &mocks.WdMock{}
suite.Queue = &JobsQueue{Storage: suite.Storage, Logger: suite.Logger}
suite.iterateChan = make(chan bool)
forever = func() bool {
return <-suite.iterateChan
}
suite.Iterate = func() {
suite.iterateChan <- true
}
suite.done = func() {
suite.iterateChan <- false
}
suite.MojangApi = new(mojangApiMocks)
usernamesToUuids = suite.MojangApi.UsernamesToUuids
uuidToTextures = suite.MojangApi.UuidToTextures
}
func (suite *queueTestSuite) TearDownTest() {
suite.done()
time.Sleep(10 * time.Millisecond) // Add delay to let finish all goroutines before assert mocks calls
suite.MojangApi.AssertExpectations(suite.T())
suite.Storage.AssertExpectations(suite.T())
suite.Logger.AssertExpectations(suite.T())
}
func (suite *queueTestSuite) TestReceiveTexturesForOneUsernameWithoutAnyCache() {
expectedResult := &mojang.SignedTexturesResponse{Id: "0d252b7218b648bfb86c2ae476954d32", Name: "maksimkurb"}
suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Once()
suite.Logger.On("IncCounter", "mojang_textures.usernames.queued", int64(1)).Once()
suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(1)).Once()
suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(0)).Once()
suite.Logger.On("RecordTimer", "mojang_textures.usernames.round_time", mock.Anything)
suite.Logger.On("IncCounter", "mojang_textures.textures.request", int64(1)).Once()
suite.Logger.On("RecordTimer", "mojang_textures.textures.request_time", mock.Anything).Once()
suite.Logger.On("IncCounter", "mojang_textures.usernames.uuid_hit", int64(1)).Once()
suite.Logger.On("RecordTimer", "mojang_textures.result_time", mock.Anything).Once()
suite.Storage.On("GetUuid", "maksimkurb").Once().Return("", &ValueNotFound{})
suite.Storage.On("StoreUuid", "maksimkurb", "0d252b7218b648bfb86c2ae476954d32").Once().Return(nil)
suite.Storage.On("GetTextures", "0d252b7218b648bfb86c2ae476954d32").Once().Return(nil, &ValueNotFound{})
suite.Storage.On("StoreTextures", "0d252b7218b648bfb86c2ae476954d32", expectedResult).Once()
suite.MojangApi.On("UsernamesToUuids", []string{"maksimkurb"}).Once().Return([]*mojang.ProfileInfo{
{Id: "0d252b7218b648bfb86c2ae476954d32", Name: "maksimkurb"},
}, nil)
suite.MojangApi.On("UuidToTextures", "0d252b7218b648bfb86c2ae476954d32", true).Once().Return(expectedResult, nil)
resultChan := suite.Queue.GetTexturesForUsername("maksimkurb")
suite.Iterate()
result := <-resultChan
suite.Assert().Equal(expectedResult, result)
}
func (suite *queueTestSuite) TestReceiveTexturesForFewUsernamesWithoutAnyCache() {
expectedResult1 := &mojang.SignedTexturesResponse{Id: "0d252b7218b648bfb86c2ae476954d32", Name: "maksimkurb"}
expectedResult2 := &mojang.SignedTexturesResponse{Id: "4566e69fc90748ee8d71d7ba5aa00d20", Name: "Thinkofdeath"}
suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Twice()
suite.Logger.On("IncCounter", "mojang_textures.usernames.queued", int64(1)).Twice()
suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(2)).Once()
suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(0)).Once()
suite.Logger.On("RecordTimer", "mojang_textures.usernames.round_time", mock.Anything)
suite.Logger.On("IncCounter", "mojang_textures.textures.request", int64(1)).Twice()
suite.Logger.On("RecordTimer", "mojang_textures.textures.request_time", mock.Anything).Twice()
suite.Logger.On("IncCounter", "mojang_textures.usernames.uuid_hit", int64(1)).Twice()
suite.Logger.On("RecordTimer", "mojang_textures.result_time", mock.Anything).Twice()
suite.Storage.On("GetUuid", "maksimkurb").Once().Return("", &ValueNotFound{})
suite.Storage.On("GetUuid", "Thinkofdeath").Once().Return("", &ValueNotFound{})
suite.Storage.On("StoreUuid", "maksimkurb", "0d252b7218b648bfb86c2ae476954d32").Once().Return(nil)
suite.Storage.On("StoreUuid", "Thinkofdeath", "4566e69fc90748ee8d71d7ba5aa00d20").Once().Return(nil)
suite.Storage.On("GetTextures", "0d252b7218b648bfb86c2ae476954d32").Once().Return(nil, &ValueNotFound{})
suite.Storage.On("GetTextures", "4566e69fc90748ee8d71d7ba5aa00d20").Once().Return(nil, &ValueNotFound{})
suite.Storage.On("StoreTextures", "0d252b7218b648bfb86c2ae476954d32", expectedResult1).Once()
suite.Storage.On("StoreTextures", "4566e69fc90748ee8d71d7ba5aa00d20", expectedResult2).Once()
suite.MojangApi.On("UsernamesToUuids", []string{"maksimkurb", "Thinkofdeath"}).Once().Return([]*mojang.ProfileInfo{
{Id: "0d252b7218b648bfb86c2ae476954d32", Name: "maksimkurb"},
{Id: "4566e69fc90748ee8d71d7ba5aa00d20", Name: "Thinkofdeath"},
}, nil)
suite.MojangApi.On("UuidToTextures", "0d252b7218b648bfb86c2ae476954d32", true).Once().Return(expectedResult1, nil)
suite.MojangApi.On("UuidToTextures", "4566e69fc90748ee8d71d7ba5aa00d20", true).Once().Return(expectedResult2, nil)
resultChan1 := suite.Queue.GetTexturesForUsername("maksimkurb")
resultChan2 := suite.Queue.GetTexturesForUsername("Thinkofdeath")
suite.Iterate()
suite.Assert().Equal(expectedResult1, <-resultChan1)
suite.Assert().Equal(expectedResult2, <-resultChan2)
}
func (suite *queueTestSuite) TestReceiveTexturesForUsernameWithCachedUuid() {
expectedResult := &mojang.SignedTexturesResponse{Id: "0d252b7218b648bfb86c2ae476954d32", Name: "maksimkurb"}
suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Once()
suite.Logger.On("IncCounter", "mojang_textures.usernames.cache_hit", int64(1)).Once()
suite.Logger.On("IncCounter", "mojang_textures.textures.request", int64(1)).Once()
suite.Logger.On("RecordTimer", "mojang_textures.textures.request_time", mock.Anything).Once()
suite.Logger.On("RecordTimer", "mojang_textures.result_time", mock.Anything).Once()
suite.Storage.On("GetUuid", "maksimkurb").Once().Return("0d252b7218b648bfb86c2ae476954d32", nil)
// Storage.StoreUuid shouldn't be called
suite.Storage.On("GetTextures", "0d252b7218b648bfb86c2ae476954d32").Once().Return(nil, &ValueNotFound{})
suite.Storage.On("StoreTextures", "0d252b7218b648bfb86c2ae476954d32", expectedResult).Once()
// MojangApi.UsernamesToUuids shouldn't be called
suite.MojangApi.On("UuidToTextures", "0d252b7218b648bfb86c2ae476954d32", true).Once().Return(expectedResult, nil)
resultChan := suite.Queue.GetTexturesForUsername("maksimkurb")
// Note that there is no iteration
result := <-resultChan
suite.Assert().Equal(expectedResult, result)
}
func (suite *queueTestSuite) TestReceiveTexturesForUsernameWithFullyCachedResult() {
expectedResult := &mojang.SignedTexturesResponse{Id: "0d252b7218b648bfb86c2ae476954d32", Name: "maksimkurb"}
suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Once()
suite.Logger.On("IncCounter", "mojang_textures.usernames.cache_hit", int64(1)).Once()
suite.Logger.On("IncCounter", "mojang_textures.textures.cache_hit", int64(1)).Once()
suite.Logger.On("RecordTimer", "mojang_textures.result_time", mock.Anything).Once()
suite.Storage.On("GetUuid", "maksimkurb").Once().Return("0d252b7218b648bfb86c2ae476954d32", nil)
// Storage.StoreUuid shouldn't be called
suite.Storage.On("GetTextures", "0d252b7218b648bfb86c2ae476954d32").Once().Return(expectedResult, nil)
// Storage.StoreTextures shouldn't be called
// MojangApi.UsernamesToUuids shouldn't be called
// MojangApi.UuidToTextures shouldn't be called
resultChan := suite.Queue.GetTexturesForUsername("maksimkurb")
// Note that there is no iteration
result := <-resultChan
suite.Assert().Equal(expectedResult, result)
}
func (suite *queueTestSuite) TestReceiveTexturesForUsernameWithCachedUnknownUuid() {
suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Once()
suite.Logger.On("IncCounter", "mojang_textures.usernames.cache_hit_nil", int64(1)).Once()
suite.Storage.On("GetUuid", "maksimkurb").Once().Return("", nil)
// Storage.StoreUuid shouldn't be called
// Storage.GetTextures shouldn't be called
// Storage.StoreTextures shouldn't be called
// MojangApi.UsernamesToUuids shouldn't be called
// MojangApi.UuidToTextures shouldn't be called
resultChan := suite.Queue.GetTexturesForUsername("maksimkurb")
// Note that there is no iteration
suite.Assert().Nil(<-resultChan)
}
func (suite *queueTestSuite) TestReceiveTexturesForMoreThan10Usernames() {
usernames := make([]string, 12)
for i := 0; i < cap(usernames); i++ {
usernames[i] = randStr(8)
}
suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Times(12)
suite.Logger.On("IncCounter", "mojang_textures.usernames.queued", int64(1)).Times(12)
suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(10)).Once()
suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(2)).Once()
suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(2)).Once()
suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(0)).Once()
suite.Logger.On("RecordTimer", "mojang_textures.usernames.round_time", mock.Anything).Twice()
suite.Logger.On("IncCounter", "mojang_textures.usernames.uuid_miss", int64(1)).Times(12)
suite.Logger.On("RecordTimer", "mojang_textures.result_time", mock.Anything).Times(12)
suite.Storage.On("GetUuid", mock.Anything).Times(12).Return("", &ValueNotFound{})
suite.Storage.On("StoreUuid", mock.Anything, "").Times(12).Return(nil) // should be called with "" if username is not compared to uuid
// Storage.GetTextures and Storage.SetTextures shouldn't be called
suite.MojangApi.On("UsernamesToUuids", usernames[0:10]).Once().Return([]*mojang.ProfileInfo{}, nil)
suite.MojangApi.On("UsernamesToUuids", usernames[10:12]).Once().Return([]*mojang.ProfileInfo{}, nil)
channels := make([]chan *mojang.SignedTexturesResponse, 12)
for i, username := range usernames {
channels[i] = suite.Queue.GetTexturesForUsername(username)
}
suite.Iterate()
suite.Iterate()
for _, channel := range channels {
<-channel
}
}
func (suite *queueTestSuite) TestReceiveTexturesForTheSameUsernames() {
expectedResult := &mojang.SignedTexturesResponse{Id: "0d252b7218b648bfb86c2ae476954d32", Name: "maksimkurb"}
suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Twice()
suite.Logger.On("IncCounter", "mojang_textures.usernames.queued", int64(1)).Once()
suite.Logger.On("IncCounter", "mojang_textures.already_in_queue", int64(1)).Once()
suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(1)).Once()
suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(0)).Once()
suite.Logger.On("RecordTimer", "mojang_textures.usernames.round_time", mock.Anything)
suite.Logger.On("IncCounter", "mojang_textures.textures.request", int64(1)).Once()
suite.Logger.On("RecordTimer", "mojang_textures.textures.request_time", mock.Anything).Once()
suite.Logger.On("IncCounter", "mojang_textures.usernames.uuid_hit", int64(1)).Once()
suite.Logger.On("RecordTimer", "mojang_textures.result_time", mock.Anything).Once()
suite.Storage.On("GetUuid", "maksimkurb").Twice().Return("", &ValueNotFound{})
suite.Storage.On("StoreUuid", "maksimkurb", "0d252b7218b648bfb86c2ae476954d32").Once().Return(nil)
suite.Storage.On("GetTextures", "0d252b7218b648bfb86c2ae476954d32").Once().Return(nil, &ValueNotFound{})
suite.Storage.On("StoreTextures", "0d252b7218b648bfb86c2ae476954d32", expectedResult).Once()
suite.MojangApi.On("UsernamesToUuids", []string{"maksimkurb"}).Once().Return([]*mojang.ProfileInfo{
{Id: "0d252b7218b648bfb86c2ae476954d32", Name: "maksimkurb"},
}, nil)
suite.MojangApi.On("UuidToTextures", "0d252b7218b648bfb86c2ae476954d32", true).Once().Return(expectedResult, nil)
resultChan1 := suite.Queue.GetTexturesForUsername("maksimkurb")
resultChan2 := suite.Queue.GetTexturesForUsername("maksimkurb")
suite.Iterate()
suite.Assert().Equal(expectedResult, <-resultChan1)
suite.Assert().Equal(expectedResult, <-resultChan2)
}
func (suite *queueTestSuite) TestReceiveTexturesForUsernameThatAlreadyProcessing() {
expectedResult := &mojang.SignedTexturesResponse{Id: "0d252b7218b648bfb86c2ae476954d32", Name: "maksimkurb"}
suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Twice()
suite.Logger.On("IncCounter", "mojang_textures.usernames.queued", int64(1)).Once()
suite.Logger.On("IncCounter", "mojang_textures.already_in_queue", int64(1)).Once()
suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(1)).Once()
suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(0)).Once()
suite.Logger.On("RecordTimer", "mojang_textures.usernames.round_time", mock.Anything)
suite.Logger.On("IncCounter", "mojang_textures.textures.request", int64(1)).Once()
suite.Logger.On("RecordTimer", "mojang_textures.textures.request_time", mock.Anything).Once()
suite.Logger.On("IncCounter", "mojang_textures.usernames.uuid_hit", int64(1)).Once()
suite.Logger.On("RecordTimer", "mojang_textures.result_time", mock.Anything).Once()
suite.Storage.On("GetUuid", "maksimkurb").Twice().Return("", &ValueNotFound{})
suite.Storage.On("StoreUuid", "maksimkurb", "0d252b7218b648bfb86c2ae476954d32").Once().Return(nil)
suite.Storage.On("GetTextures", "0d252b7218b648bfb86c2ae476954d32").Once().Return(nil, &ValueNotFound{})
suite.Storage.On("StoreTextures", "0d252b7218b648bfb86c2ae476954d32", expectedResult).Once()
suite.MojangApi.On("UsernamesToUuids", []string{"maksimkurb"}).Once().Return([]*mojang.ProfileInfo{
{Id: "0d252b7218b648bfb86c2ae476954d32", Name: "maksimkurb"},
}, nil)
suite.MojangApi.On("UuidToTextures", "0d252b7218b648bfb86c2ae476954d32", true).
Once().
After(10*time.Millisecond). // Simulate long round trip
Return(expectedResult, nil)
resultChan1 := suite.Queue.GetTexturesForUsername("maksimkurb")
// Note that for entire test there is only one iteration
suite.Iterate()
// Let it meet delayed UuidToTextures request
time.Sleep(5 * time.Millisecond)
resultChan2 := suite.Queue.GetTexturesForUsername("maksimkurb")
suite.Assert().Equal(expectedResult, <-resultChan1)
suite.Assert().Equal(expectedResult, <-resultChan2)
}
func (suite *queueTestSuite) TestDoNothingWhenNoTasks() {
suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Once()
suite.Logger.On("IncCounter", "mojang_textures.usernames.queued", int64(1)).Once()
suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(1)).Once()
suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(0)).Once()
suite.Logger.On("RecordTimer", "mojang_textures.usernames.round_time", mock.Anything)
suite.Logger.On("IncCounter", "mojang_textures.usernames.uuid_miss", int64(1)).Once()
suite.Logger.On("RecordTimer", "mojang_textures.result_time", mock.Anything).Once()
suite.Storage.On("GetUuid", "maksimkurb").Once().Return("", &ValueNotFound{})
suite.Storage.On("StoreUuid", "maksimkurb", "").Once().Return(nil)
// Storage.GetTextures and Storage.StoreTextures shouldn't be called
suite.MojangApi.On("UsernamesToUuids", []string{"maksimkurb"}).Once().Return([]*mojang.ProfileInfo{}, nil)
// Perform first iteration and await it finish
resultChan := suite.Queue.GetTexturesForUsername("maksimkurb")
suite.Iterate()
suite.Assert().Nil(<-resultChan)
// Let it to perform a few more iterations to ensure, that there is no calls to external APIs
suite.Iterate()
suite.Iterate()
}
type timeoutError struct {
}
func (*timeoutError) Error() string { return "timeout error" }
func (*timeoutError) Timeout() bool { return true }
func (*timeoutError) Temporary() bool { return false }
var expectedErrors = []error{
&mojang.BadRequestError{},
&mojang.ForbiddenError{},
&mojang.TooManyRequestsError{},
&mojang.ServerError{},
&timeoutError{},
&url.Error{Op: "GET", URL: "http://localhost"},
&net.OpError{Op: "read"},
&net.OpError{Op: "dial"},
syscall.ECONNREFUSED,
}
func (suite *queueTestSuite) TestShouldNotLogErrorWhenExpectedErrorReturnedFromUsernameToUuidRequest() {
suite.Logger.On("IncCounter", mock.Anything, mock.Anything)
suite.Logger.On("UpdateGauge", mock.Anything, mock.Anything)
suite.Logger.On("RecordTimer", mock.Anything, mock.Anything)
suite.Logger.On("Debug", ":name: Got response error :err", mock.Anything, mock.Anything).Times(len(expectedErrors))
suite.Logger.On("Warning", ":name: Got 400 Bad Request :err", mock.Anything, mock.Anything).Once()
suite.Logger.On("Warning", ":name: Got 403 Forbidden :err", mock.Anything, mock.Anything).Once()
suite.Logger.On("Warning", ":name: Got 429 Too Many Requests :err", mock.Anything, mock.Anything).Once()
suite.Storage.On("GetUuid", "maksimkurb").Return("", &ValueNotFound{})
for _, err := range expectedErrors {
suite.MojangApi.On("UsernamesToUuids", []string{"maksimkurb"}).Once().Return(nil, err)
resultChan := suite.Queue.GetTexturesForUsername("maksimkurb")
suite.Iterate()
suite.Assert().Nil(<-resultChan)
suite.MojangApi.AssertExpectations(suite.T())
suite.MojangApi.ExpectedCalls = nil // https://github.com/stretchr/testify/issues/558#issuecomment-372112364
}
}
func (suite *queueTestSuite) TestShouldLogEmergencyOnUnexpectedErrorReturnedFromUsernameToUuidRequest() {
suite.Logger.On("IncCounter", mock.Anything, mock.Anything)
suite.Logger.On("UpdateGauge", mock.Anything, mock.Anything)
suite.Logger.On("RecordTimer", mock.Anything, mock.Anything)
suite.Logger.On("Debug", ":name: Got response error :err", mock.Anything, mock.Anything).Once()
suite.Logger.On("Emergency", ":name: Unknown Mojang response error: :err", mock.Anything, mock.Anything).Once()
suite.Storage.On("GetUuid", "maksimkurb").Return("", &ValueNotFound{})
suite.MojangApi.On("UsernamesToUuids", []string{"maksimkurb"}).Once().Return(nil, errors.New("unexpected error"))
resultChan := suite.Queue.GetTexturesForUsername("maksimkurb")
suite.Iterate()
suite.Assert().Nil(<-resultChan)
}
func (suite *queueTestSuite) TestShouldNotLogErrorWhenExpectedErrorReturnedFromUuidToTexturesRequest() {
suite.Logger.On("IncCounter", mock.Anything, mock.Anything)
suite.Logger.On("UpdateGauge", mock.Anything, mock.Anything)
suite.Logger.On("RecordTimer", mock.Anything, mock.Anything)
suite.Logger.On("Debug", ":name: Got response error :err", mock.Anything, mock.Anything).Times(len(expectedErrors))
suite.Logger.On("Warning", ":name: Got 400 Bad Request :err", mock.Anything, mock.Anything).Once()
suite.Logger.On("Warning", ":name: Got 403 Forbidden :err", mock.Anything, mock.Anything).Once()
suite.Logger.On("Warning", ":name: Got 429 Too Many Requests :err", mock.Anything, mock.Anything).Once()
suite.Storage.On("GetUuid", "maksimkurb").Return("", &ValueNotFound{})
suite.Storage.On("StoreUuid", "maksimkurb", "0d252b7218b648bfb86c2ae476954d32").Return(nil)
suite.Storage.On("GetTextures", "0d252b7218b648bfb86c2ae476954d32").Return(nil, &ValueNotFound{})
suite.Storage.On("StoreTextures", "0d252b7218b648bfb86c2ae476954d32", (*mojang.SignedTexturesResponse)(nil))
for _, err := range expectedErrors {
suite.MojangApi.On("UsernamesToUuids", []string{"maksimkurb"}).Once().Return([]*mojang.ProfileInfo{
{Id: "0d252b7218b648bfb86c2ae476954d32", Name: "maksimkurb"},
}, nil)
suite.MojangApi.On("UuidToTextures", "0d252b7218b648bfb86c2ae476954d32", true).Once().Return(nil, err)
resultChan := suite.Queue.GetTexturesForUsername("maksimkurb")
suite.Iterate()
suite.Assert().Nil(<-resultChan)
suite.MojangApi.AssertExpectations(suite.T())
suite.MojangApi.ExpectedCalls = nil // https://github.com/stretchr/testify/issues/558#issuecomment-372112364
}
}
func (suite *queueTestSuite) TestShouldLogEmergencyOnUnexpectedErrorReturnedFromUuidToTexturesRequest() {
suite.Logger.On("IncCounter", mock.Anything, mock.Anything)
suite.Logger.On("UpdateGauge", mock.Anything, mock.Anything)
suite.Logger.On("RecordTimer", mock.Anything, mock.Anything)
suite.Logger.On("Debug", ":name: Got response error :err", mock.Anything, mock.Anything).Once()
suite.Logger.On("Emergency", ":name: Unknown Mojang response error: :err", mock.Anything, mock.Anything).Once()
suite.Storage.On("GetUuid", "maksimkurb").Return("", &ValueNotFound{})
suite.Storage.On("StoreUuid", "maksimkurb", "0d252b7218b648bfb86c2ae476954d32").Return(nil)
suite.Storage.On("GetTextures", "0d252b7218b648bfb86c2ae476954d32").Return(nil, &ValueNotFound{})
suite.Storage.On("StoreTextures", "0d252b7218b648bfb86c2ae476954d32", (*mojang.SignedTexturesResponse)(nil))
suite.MojangApi.On("UsernamesToUuids", []string{"maksimkurb"}).Once().Return([]*mojang.ProfileInfo{
{Id: "0d252b7218b648bfb86c2ae476954d32", Name: "maksimkurb"},
}, nil)
suite.MojangApi.On("UuidToTextures", "0d252b7218b648bfb86c2ae476954d32", true).Once().Return(nil, errors.New("unexpected error"))
resultChan := suite.Queue.GetTexturesForUsername("maksimkurb")
suite.Iterate()
suite.Assert().Nil(<-resultChan)
}
func (suite *queueTestSuite) TestReceiveTexturesForNotAllowedMojangUsername() {
suite.Logger.On("IncCounter", "mojang_textures.invalid_username", int64(1)).Once()
resultChan := suite.Queue.GetTexturesForUsername("Not allowed")
suite.Assert().Nil(<-resultChan)
}
func TestJobsQueueSuite(t *testing.T) {
suite.Run(t, new(queueTestSuite))
}
var replacer = strings.NewReplacer("-", "_", "=", "")
// https://stackoverflow.com/a/50581165
func randStr(len int) string {
buff := make([]byte, len)
_, _ = rand.Read(buff)
str := replacer.Replace(base64.URLEncoding.EncodeToString(buff))
// Base 64 can be longer than len
return str[:len]
}

53
api/mojang/queue/storage.go
View File

@@ -1,53 +0,0 @@
package queue
import "github.com/elyby/chrly/api/mojang"
type UuidsStorage interface {
GetUuid(username string) (string, error)
StoreUuid(username string, uuid string) error
}
// nil value can be passed to the storage to indicate that there is no textures
// for uuid and we know about it. Return err only in case, when storage completely
// unable to load any information about textures
type TexturesStorage interface {
GetTextures(uuid string) (*mojang.SignedTexturesResponse, error)
StoreTextures(uuid string, textures *mojang.SignedTexturesResponse)
}
type Storage interface {
UuidsStorage
TexturesStorage
}
// SplittedStorage allows you to use separate storage engines to satisfy
// the Storage interface
type SplittedStorage struct {
UuidsStorage
TexturesStorage
}
func (s *SplittedStorage) GetUuid(username string) (string, error) {
return s.UuidsStorage.GetUuid(username)
}
func (s *SplittedStorage) StoreUuid(username string, uuid string) error {
return s.UuidsStorage.StoreUuid(username, uuid)
}
func (s *SplittedStorage) GetTextures(uuid string) (*mojang.SignedTexturesResponse, error) {
return s.TexturesStorage.GetTextures(uuid)
}
func (s *SplittedStorage) StoreTextures(uuid string, textures *mojang.SignedTexturesResponse) {
s.TexturesStorage.StoreTextures(uuid, textures)
}
// This error can be used to indicate, that requested
// value doesn't exists in the storage
type ValueNotFound struct {
}
func (*ValueNotFound) Error() string {
return "value not found in the storage"
}

30
cmd/serve.go
View File

@@ -8,11 +8,11 @@ import (
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/spf13/viper" "github.com/spf13/viper"
"github.com/elyby/chrly/api/mojang/queue"
"github.com/elyby/chrly/auth" "github.com/elyby/chrly/auth"
"github.com/elyby/chrly/bootstrap" "github.com/elyby/chrly/bootstrap"
"github.com/elyby/chrly/db" "github.com/elyby/chrly/db"
"github.com/elyby/chrly/http" "github.com/elyby/chrly/http"
"github.com/elyby/chrly/mojangtextures"
) )
var serveCmd = &cobra.Command{ var serveCmd = &cobra.Command{
@@ -52,12 +52,19 @@ var serveCmd = &cobra.Command{
return return
} }
queue.UuidsQueueIterationDelay = time.Duration(viper.GetInt("queue.loop_delay")) * time.Millisecond texturesStorage := mojangtextures.NewInMemoryTexturesStorage()
texturesStorage := queue.CreateInMemoryTexturesStorage()
texturesStorage.Start() texturesStorage.Start()
mojangTexturesQueue := &queue.JobsQueue{ mojangTexturesProvider := &mojangtextures.Provider{
Logger: logger, Logger: logger,
Storage: &queue.SplittedStorage{ UuidsProvider: &mojangtextures.BatchUuidsProvider{
IterationDelay: time.Duration(viper.GetInt("queue.loop_delay")) * time.Millisecond,
IterationSize: viper.GetInt("queue.batch_size"),
Logger: logger,
},
TexturesProvider: &mojangtextures.MojangApiTexturesProvider{
Logger: logger,
},
Storage: &mojangtextures.SeparatedStorage{
UuidsStorage: mojangUuidsRepository, UuidsStorage: mojangUuidsRepository,
TexturesStorage: texturesStorage, TexturesStorage: texturesStorage,
}, },
@@ -65,12 +72,12 @@ var serveCmd = &cobra.Command{
logger.Info("Mojang's textures queue is successfully initialized") logger.Info("Mojang's textures queue is successfully initialized")
cfg := &http.Config{ cfg := &http.Config{
ListenSpec: fmt.Sprintf("%s:%d", viper.GetString("server.host"), viper.GetInt("server.port")), ListenSpec: fmt.Sprintf("%s:%d", viper.GetString("server.host"), viper.GetInt("server.port")),
SkinsRepo: skinsRepo, SkinsRepo: skinsRepo,
CapesRepo: capesRepo, CapesRepo: capesRepo,
MojangTexturesQueue: mojangTexturesQueue, MojangTexturesProvider: mojangTexturesProvider,
Logger: logger, Logger: logger,
Auth: &auth.JwtAuth{Key: []byte(viper.GetString("chrly.secret"))}, Auth: &auth.JwtAuth{Key: []byte(viper.GetString("chrly.secret"))},
} }
if err := cfg.Run(); err != nil { if err := cfg.Run(); err != nil {
@@ -89,4 +96,5 @@ func init() {
viper.SetDefault("storage.filesystem.basePath", "data") viper.SetDefault("storage.filesystem.basePath", "data")
viper.SetDefault("storage.filesystem.capesDirName", "capes") viper.SetDefault("storage.filesystem.capesDirName", "capes")
viper.SetDefault("queue.loop_delay", 2_500) viper.SetDefault("queue.loop_delay", 2_500)
viper.SetDefault("queue.batch_size", 10)
} }

4
db/factory.go
View File

@@ -3,8 +3,8 @@ package db
import ( import (
"github.com/spf13/viper" "github.com/spf13/viper"
"github.com/elyby/chrly/api/mojang/queue"
"github.com/elyby/chrly/interfaces" "github.com/elyby/chrly/interfaces"
"github.com/elyby/chrly/mojangtextures"
) )
type StorageFactory struct { type StorageFactory struct {
@@ -14,7 +14,7 @@ type StorageFactory struct {
type RepositoriesCreator interface { type RepositoriesCreator interface {
CreateSkinsRepository() (interfaces.SkinsRepository, error) CreateSkinsRepository() (interfaces.SkinsRepository, error)
CreateCapesRepository() (interfaces.CapesRepository, error) CreateCapesRepository() (interfaces.CapesRepository, error)
CreateMojangUuidsRepository() (queue.UuidsStorage, error) CreateMojangUuidsRepository() (mojangtextures.UuidsStorage, error)
} }
func (factory *StorageFactory) CreateFactory(backend string) RepositoriesCreator { func (factory *StorageFactory) CreateFactory(backend string) RepositoriesCreator {

4
db/filesystem.go
View File

@@ -5,9 +5,9 @@ import (
"path" "path"
"strings" "strings"
"github.com/elyby/chrly/api/mojang/queue"
"github.com/elyby/chrly/interfaces" "github.com/elyby/chrly/interfaces"
"github.com/elyby/chrly/model" "github.com/elyby/chrly/model"
"github.com/elyby/chrly/mojangtextures"
) )
type FilesystemFactory struct { type FilesystemFactory struct {
@@ -27,7 +27,7 @@ func (f FilesystemFactory) CreateCapesRepository() (interfaces.CapesRepository,
return &filesStorage{path: path.Join(f.BasePath, f.CapesDirName)}, nil return &filesStorage{path: path.Join(f.BasePath, f.CapesDirName)}, nil
} }
func (f FilesystemFactory) CreateMojangUuidsRepository() (queue.UuidsStorage, error) { func (f FilesystemFactory) CreateMojangUuidsRepository() (mojangtextures.UuidsStorage, error) {
panic("implement me") panic("implement me")
} }

8
db/redis.go
View File

@@ -14,9 +14,9 @@ import (
"github.com/mediocregopher/radix.v2/redis" "github.com/mediocregopher/radix.v2/redis"
"github.com/mediocregopher/radix.v2/util" "github.com/mediocregopher/radix.v2/util"
"github.com/elyby/chrly/api/mojang/queue"
"github.com/elyby/chrly/interfaces" "github.com/elyby/chrly/interfaces"
"github.com/elyby/chrly/model" "github.com/elyby/chrly/model"
"github.com/elyby/chrly/mojangtextures"
) )
type RedisFactory struct { type RedisFactory struct {
@@ -34,7 +34,7 @@ func (f *RedisFactory) CreateCapesRepository() (interfaces.CapesRepository, erro
panic("capes repository not supported for this storage type") panic("capes repository not supported for this storage type")
} }
func (f *RedisFactory) CreateMojangUuidsRepository() (queue.UuidsStorage, error) { func (f *RedisFactory) CreateMojangUuidsRepository() (mojangtextures.UuidsStorage, error) {
return f.createInstance() return f.createInstance()
} }
@@ -255,7 +255,7 @@ func save(skin *model.Skin, conn util.Cmder) error {
func findMojangUuidByUsername(username string, conn util.Cmder) (string, error) { func findMojangUuidByUsername(username string, conn util.Cmder) (string, error) {
response := conn.Cmd("HGET", mojangUsernameToUuidKey, strings.ToLower(username)) response := conn.Cmd("HGET", mojangUsernameToUuidKey, strings.ToLower(username))
if response.IsType(redis.Nil) { if response.IsType(redis.Nil) {
return "", &queue.ValueNotFound{} return "", &mojangtextures.ValueNotFound{}
} }
data, _ := response.Str() data, _ := response.Str()
@@ -263,7 +263,7 @@ func findMojangUuidByUsername(username string, conn util.Cmder) (string, error)
timestamp, _ := strconv.ParseInt(parts[1], 10, 64) timestamp, _ := strconv.ParseInt(parts[1], 10, 64)
storedAt := time.Unix(timestamp, 0) storedAt := time.Unix(timestamp, 0)
if storedAt.Add(time.Hour * 24 * 30).Before(time.Now()) { if storedAt.Add(time.Hour * 24 * 30).Before(time.Now()) {
return "", &queue.ValueNotFound{} return "", &mojangtextures.ValueNotFound{}
} }
return parts[0], nil return parts[0], nil

4
http/cape.go
View File

@@ -20,8 +20,8 @@ func (cfg *Config) Cape(response http.ResponseWriter, request *http.Request) {
return return
} }
mojangTextures := <-cfg.MojangTexturesQueue.GetTexturesForUsername(username) mojangTextures, err := cfg.MojangTexturesProvider.GetForUsername(username)
if mojangTextures == nil { if err != nil || mojangTextures == nil {
response.WriteHeader(http.StatusNotFound) response.WriteHeader(http.StatusNotFound)
return return
} }

22
http/cape_test.go
View File

@@ -28,7 +28,7 @@ type capesTestCase struct {
var capesTestCases = []*capesTestCase{ var capesTestCases = []*capesTestCase{
{ {
Name: "Obtain cape for known username", Name: "Obtain cape for known username",
ExistsInLocalStorage: true, ExistsInLocalStorage: true,
AssertResponse: func(assert *testify.Assertions, resp *http.Response) { AssertResponse: func(assert *testify.Assertions, resp *http.Response) {
assert.Equal(200, resp.StatusCode) assert.Equal(200, resp.StatusCode)
@@ -38,28 +38,28 @@ var capesTestCases = []*capesTestCase{
}, },
}, },
{ {
Name: "Obtain cape for unknown username that exists in Mojang and has a cape", Name: "Obtain cape for unknown username that exists in Mojang and has a cape",
ExistsInLocalStorage: false, ExistsInLocalStorage: false,
ExistsInMojang: true, ExistsInMojang: true,
HasCapeInMojangResp: true, HasCapeInMojangResp: true,
AssertResponse: func(assert *testify.Assertions, resp *http.Response) { AssertResponse: func(assert *testify.Assertions, resp *http.Response) {
assert.Equal(301, resp.StatusCode) assert.Equal(301, resp.StatusCode)
assert.Equal("http://mojang/cape.png", resp.Header.Get("Location")) assert.Equal("http://mojang/cape.png", resp.Header.Get("Location"))
}, },
}, },
{ {
Name: "Obtain cape for unknown username that exists in Mojang, but don't has a cape", Name: "Obtain cape for unknown username that exists in Mojang, but don't has a cape",
ExistsInLocalStorage: false, ExistsInLocalStorage: false,
ExistsInMojang: true, ExistsInMojang: true,
HasCapeInMojangResp: false, HasCapeInMojangResp: false,
AssertResponse: func(assert *testify.Assertions, resp *http.Response) { AssertResponse: func(assert *testify.Assertions, resp *http.Response) {
assert.Equal(404, resp.StatusCode) assert.Equal(404, resp.StatusCode)
}, },
}, },
{ {
Name: "Obtain cape for unknown username that doesn't exists in Mojang", Name: "Obtain cape for unknown username that doesn't exists in Mojang",
ExistsInLocalStorage: false, ExistsInLocalStorage: false,
ExistsInMojang: false, ExistsInMojang: false,
AssertResponse: func(assert *testify.Assertions, resp *http.Response) { AssertResponse: func(assert *testify.Assertions, resp *http.Response) {
assert.Equal(404, resp.StatusCode) assert.Equal(404, resp.StatusCode)
}, },
@@ -86,9 +86,9 @@ func TestConfig_Cape(t *testing.T) {
if testCase.ExistsInMojang { if testCase.ExistsInMojang {
textures := createTexturesResponse(false, testCase.HasCapeInMojangResp) textures := createTexturesResponse(false, testCase.HasCapeInMojangResp)
mocks.Queue.On("GetTexturesForUsername", "mock_username").Return(textures) mocks.MojangProvider.On("GetForUsername", "mock_username").Return(textures, nil)
} else { } else {
mocks.Queue.On("GetTexturesForUsername", "mock_username").Return(nil) mocks.MojangProvider.On("GetForUsername", "mock_username").Return(nil, nil)
} }
req := httptest.NewRequest("GET", testCase.RequestUrl, nil) req := httptest.NewRequest("GET", testCase.RequestUrl, nil)

10
http/http.go
View File

@@ -19,11 +19,11 @@ import (
type Config struct { type Config struct {
ListenSpec string ListenSpec string
SkinsRepo interfaces.SkinsRepository SkinsRepo interfaces.SkinsRepository
CapesRepo interfaces.CapesRepository CapesRepo interfaces.CapesRepository
MojangTexturesQueue interfaces.MojangTexturesQueue MojangTexturesProvider interfaces.MojangTexturesProvider
Logger wd.Watchdog Logger wd.Watchdog
Auth interfaces.AuthChecker Auth interfaces.AuthChecker
} }
func (cfg *Config) Run() error { func (cfg *Config) Run() error {

70
http/http_test.go
View File

@@ -4,9 +4,10 @@ import (
"testing" "testing"
"time" "time"
"github.com/stretchr/testify/mock"
"github.com/elyby/chrly/api/mojang" "github.com/elyby/chrly/api/mojang"
"github.com/elyby/chrly/tests"
"github.com/golang/mock/gomock" "github.com/golang/mock/gomock"
testify "github.com/stretchr/testify/assert" testify "github.com/stretchr/testify/assert"
@@ -20,46 +21,57 @@ func TestParseUsername(t *testing.T) {
assert.Equal("test", parseUsername("test"), "Function should return string itself, if it not contains .png at end") assert.Equal("test", parseUsername("test"), "Function should return string itself, if it not contains .png at end")
} }
type mocks struct { type mojangTexturesProviderMock struct {
Skins *mock_interfaces.MockSkinsRepository mock.Mock
Capes *mock_interfaces.MockCapesRepository
Queue *tests.MojangTexturesQueueMock
Auth *mock_interfaces.MockAuthChecker
Log *mock_wd.MockWatchdog
} }
func setupMocks(ctrl *gomock.Controller) ( func (m *mojangTexturesProviderMock) GetForUsername(username string) (*mojang.SignedTexturesResponse, error) {
*Config, args := m.Called(username)
*mocks, var result *mojang.SignedTexturesResponse
) { if casted, ok := args.Get(0).(*mojang.SignedTexturesResponse); ok {
result = casted
}
return result, args.Error(1)
}
type mocks struct {
Skins *mock_interfaces.MockSkinsRepository
Capes *mock_interfaces.MockCapesRepository
MojangProvider *mojangTexturesProviderMock
Auth *mock_interfaces.MockAuthChecker
Log *mock_wd.MockWatchdog
}
func setupMocks(ctrl *gomock.Controller) (*Config, *mocks) {
skinsRepo := mock_interfaces.NewMockSkinsRepository(ctrl) skinsRepo := mock_interfaces.NewMockSkinsRepository(ctrl)
capesRepo := mock_interfaces.NewMockCapesRepository(ctrl) capesRepo := mock_interfaces.NewMockCapesRepository(ctrl)
authChecker := mock_interfaces.NewMockAuthChecker(ctrl) authChecker := mock_interfaces.NewMockAuthChecker(ctrl)
wd := mock_wd.NewMockWatchdog(ctrl) wd := mock_wd.NewMockWatchdog(ctrl)
texturesQueue := &tests.MojangTexturesQueueMock{} texturesProvider := &mojangTexturesProviderMock{}
return &Config{ return &Config{
SkinsRepo: skinsRepo, SkinsRepo: skinsRepo,
CapesRepo: capesRepo, CapesRepo: capesRepo,
Auth: authChecker, Auth: authChecker,
MojangTexturesQueue: texturesQueue, MojangTexturesProvider: texturesProvider,
Logger: wd, Logger: wd,
}, &mocks{ }, &mocks{
Skins: skinsRepo, Skins: skinsRepo,
Capes: capesRepo, Capes: capesRepo,
Auth: authChecker, Auth: authChecker,
Queue: texturesQueue, MojangProvider: texturesProvider,
Log: wd, Log: wd,
} }
} }
func createTexturesResponse(includeSkin bool, includeCape bool) *mojang.SignedTexturesResponse { func createTexturesResponse(includeSkin bool, includeCape bool) *mojang.SignedTexturesResponse {
timeZone, _ := time.LoadLocation("Europe/Minsk") timeZone, _ := time.LoadLocation("Europe/Minsk")
textures := &mojang.TexturesProp{ textures := &mojang.TexturesProp{
Timestamp: time.Date(2019, 4, 27, 23, 56, 12, 0, timeZone).Unix(), Timestamp: time.Date(2019, 4, 27, 23, 56, 12, 0, timeZone).Unix(),
ProfileID: "00000000000000000000000000000000", ProfileID: "00000000000000000000000000000000",
ProfileName: "mock_user", ProfileName: "mock_user",
Textures: &mojang.TexturesResponse{}, Textures: &mojang.TexturesResponse{},
} }
if includeSkin { if includeSkin {
@@ -75,11 +87,11 @@ func createTexturesResponse(includeSkin bool, includeCape bool) *mojang.SignedTe
} }
response := &mojang.SignedTexturesResponse{ response := &mojang.SignedTexturesResponse{
Id: "00000000000000000000000000000000", Id: "00000000000000000000000000000000",
Name: "mock_user", Name: "mock_user",
Props: []*mojang.Property{ Props: []*mojang.Property{
{ {
Name: "textures", Name: "textures",
Value: mojang.EncodeTextures(textures), Value: mojang.EncodeTextures(textures),
}, },
}, },

5
http/signed_textures.go
View File

@@ -30,7 +30,10 @@ func (cfg *Config) SignedTextures(response http.ResponseWriter, request *http.Re
}, },
} }
} else if request.URL.Query().Get("proxy") != "" { } else if request.URL.Query().Get("proxy") != "" {
responseData = <-cfg.MojangTexturesQueue.GetTexturesForUsername(username) mojangTextures, err := cfg.MojangTexturesProvider.GetForUsername(username)
if err == nil && mojangTextures != nil {
responseData = mojangTextures
}
} }
if responseData == nil { if responseData == nil {

2
http/signed_textures_test.go
View File

@@ -112,7 +112,7 @@ func TestConfig_SignedTextures(t *testing.T) {
mocks.Log.EXPECT().IncCounter("signed_textures.request", int64(1)) mocks.Log.EXPECT().IncCounter("signed_textures.request", int64(1))
mocks.Skins.EXPECT().FindByUsername("mock_user").Return(skinModel, nil) mocks.Skins.EXPECT().FindByUsername("mock_user").Return(skinModel, nil)
mocks.Queue.On("GetTexturesForUsername", "mock_user").Once().Return(createTexturesResponse(true, false)) mocks.MojangProvider.On("GetForUsername", "mock_user").Once().Return(createTexturesResponse(true, false), nil)
req := httptest.NewRequest("GET", "http://chrly/textures/signed/mock_user?proxy=true", nil) req := httptest.NewRequest("GET", "http://chrly/textures/signed/mock_user?proxy=true", nil)
w := httptest.NewRecorder() w := httptest.NewRecorder()

4
http/skin.go
View File

@@ -18,8 +18,8 @@ func (cfg *Config) Skin(response http.ResponseWriter, request *http.Request) {
return return
} }
mojangTextures := <-cfg.MojangTexturesQueue.GetTexturesForUsername(username) mojangTextures, err := cfg.MojangTexturesProvider.GetForUsername(username)
if mojangTextures == nil { if err != nil || mojangTextures == nil {
response.WriteHeader(http.StatusNotFound) response.WriteHeader(http.StatusNotFound)
return return
} }

4
http/skin_test.go
View File

@@ -78,9 +78,9 @@ func TestConfig_Skin(t *testing.T) {
if testCase.ExistsInMojang { if testCase.ExistsInMojang {
textures := createTexturesResponse(testCase.HasSkinInMojangResp, true) textures := createTexturesResponse(testCase.HasSkinInMojangResp, true)
mocks.Queue.On("GetTexturesForUsername", "mock_username").Return(textures) mocks.MojangProvider.On("GetForUsername", "mock_username").Return(textures, nil)
} else { } else {
mocks.Queue.On("GetTexturesForUsername", "mock_username").Return(nil) mocks.MojangProvider.On("GetForUsername", "mock_username").Return(nil, nil)
} }
req := httptest.NewRequest("GET", testCase.RequestUrl, nil) req := httptest.NewRequest("GET", testCase.RequestUrl, nil)

4
http/textures.go
View File

@@ -39,8 +39,8 @@ func (cfg *Config) Textures(response http.ResponseWriter, request *http.Request)
} }
} }
} else { } else {
mojangTextures := <-cfg.MojangTexturesQueue.GetTexturesForUsername(username) mojangTextures, err := cfg.MojangTexturesProvider.GetForUsername(username)
if mojangTextures == nil { if err != nil || mojangTextures == nil {
response.WriteHeader(http.StatusNoContent) response.WriteHeader(http.StatusNoContent)
return return
} }

4
http/textures_test.go
View File

@@ -148,7 +148,7 @@ func TestConfig_Textures(t *testing.T) {
mocks.Skins.EXPECT().FindByUsername("mock_username").Return(nil, &db.SkinNotFoundError{}) mocks.Skins.EXPECT().FindByUsername("mock_username").Return(nil, &db.SkinNotFoundError{})
mocks.Capes.EXPECT().FindByUsername("mock_username").Return(nil, &db.CapeNotFoundError{}) mocks.Capes.EXPECT().FindByUsername("mock_username").Return(nil, &db.CapeNotFoundError{})
mocks.Queue.On("GetTexturesForUsername", "mock_username").Once().Return(createTexturesResponse(true, true)) mocks.MojangProvider.On("GetForUsername", "mock_username").Once().Return(createTexturesResponse(true, true), nil)
req := httptest.NewRequest("GET", "http://chrly/textures/mock_username", nil) req := httptest.NewRequest("GET", "http://chrly/textures/mock_username", nil)
w := httptest.NewRecorder() w := httptest.NewRecorder()
@@ -181,7 +181,7 @@ func TestConfig_Textures(t *testing.T) {
mocks.Skins.EXPECT().FindByUsername("mock_username").Return(nil, &db.SkinNotFoundError{}) mocks.Skins.EXPECT().FindByUsername("mock_username").Return(nil, &db.SkinNotFoundError{})
mocks.Capes.EXPECT().FindByUsername("mock_username").Return(nil, &db.CapeNotFoundError{}) mocks.Capes.EXPECT().FindByUsername("mock_username").Return(nil, &db.CapeNotFoundError{})
mocks.Queue.On("GetTexturesForUsername", "mock_username").Once().Return(nil) mocks.MojangProvider.On("GetForUsername", "mock_username").Once().Return(nil, nil)
req := httptest.NewRequest("GET", "http://chrly/textures/mock_username", nil) req := httptest.NewRequest("GET", "http://chrly/textures/mock_username", nil)
w := httptest.NewRecorder() w := httptest.NewRecorder()

4
interfaces/repositories.go
View File

@@ -17,6 +17,6 @@ type CapesRepository interface {
FindByUsername(username string) (*model.Cape, error) FindByUsername(username string) (*model.Cape, error)
} }
type MojangTexturesQueue interface { type MojangTexturesProvider interface {
GetTexturesForUsername(username string) chan *mojang.SignedTexturesResponse GetForUsername(username string) (*mojang.SignedTexturesResponse, error)
} }

133
mojangtextures/batch_uuids_provider.go Normal file
View File

@@ -0,0 +1,133 @@
package mojangtextures
import (
"strings"
"sync"
"time"
"github.com/mono83/slf/wd"
"github.com/elyby/chrly/api/mojang"
)
type jobResult struct {
profile *mojang.ProfileInfo
error error
}
type jobItem struct {
username string
respondChan chan *jobResult
}
type jobsQueue struct {
lock sync.Mutex
items []*jobItem
}
func (s *jobsQueue) New() *jobsQueue {
s.items = []*jobItem{}
return s
}
func (s *jobsQueue) Enqueue(t *jobItem) {
s.lock.Lock()
defer s.lock.Unlock()
s.items = append(s.items, t)
}
func (s *jobsQueue) Dequeue(n int) []*jobItem {
s.lock.Lock()
defer s.lock.Unlock()
if n > s.Size() {
n = s.Size()
}
items := s.items[0:n]
s.items = s.items[n:len(s.items)]
return items
}
func (s *jobsQueue) Size() int {
return len(s.items)
}
var usernamesToUuids = mojang.UsernamesToUuids
var forever = func() bool {
return true
}
type BatchUuidsProvider struct {
IterationDelay time.Duration
IterationSize int
Logger wd.Watchdog
onFirstCall sync.Once
queue jobsQueue
}
func (ctx *BatchUuidsProvider) GetUuid(username string) (*mojang.ProfileInfo, error) {
ctx.onFirstCall.Do(func() {
ctx.queue.New()
ctx.startQueue()
})
resultChan := make(chan *jobResult)
ctx.queue.Enqueue(&jobItem{username, resultChan})
ctx.Logger.IncCounter("mojang_textures.usernames.queued", 1)
result := <-resultChan
return result.profile, result.error
}
func (ctx *BatchUuidsProvider) startQueue() {
go func() {
time.Sleep(ctx.IterationDelay)
for forever() {
start := time.Now()
ctx.queueRound()
elapsed := time.Since(start)
ctx.Logger.RecordTimer("mojang_textures.usernames.round_time", elapsed)
time.Sleep(ctx.IterationDelay)
}
}()
}
func (ctx *BatchUuidsProvider) queueRound() {
queueSize := ctx.queue.Size()
jobs := ctx.queue.Dequeue(ctx.IterationSize)
ctx.Logger.UpdateGauge("mojang_textures.usernames.queue_size", int64(queueSize-len(jobs)))
ctx.Logger.UpdateGauge("mojang_textures.usernames.iteration_size", int64(len(jobs)))
if len(jobs) == 0 {
return
}
var usernames []string
for _, job := range jobs {
usernames = append(usernames, job.username)
}
profiles, err := usernamesToUuids(usernames)
for _, job := range jobs {
go func(job *jobItem) {
response := &jobResult{}
if err != nil {
response.error = err
} else {
// The profiles in the response aren't ordered, so we must search each username over full array
for _, profile := range profiles {
if strings.EqualFold(job.username, profile.Name) {
response.profile = profile
break
}
}
}
job.respondChan <- response
}(job)
}
}

285
mojangtextures/batch_uuids_provider_test.go Normal file
View File

@@ -0,0 +1,285 @@
package mojangtextures
import (
"crypto/rand"
"encoding/base64"
"strings"
"testing"
"time"
testify "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/suite"
"github.com/elyby/chrly/api/mojang"
mocks "github.com/elyby/chrly/tests"
)
func TestJobsQueue(t *testing.T) {
createQueue := func() *jobsQueue {
queue := &jobsQueue{}
queue.New()
return queue
}
t.Run("Enqueue", func(t *testing.T) {
assert := testify.New(t)
s := createQueue()
s.Enqueue(&jobItem{username: "username1"})
s.Enqueue(&jobItem{username: "username2"})
s.Enqueue(&jobItem{username: "username3"})
assert.Equal(3, s.Size())
})
t.Run("Dequeue", func(t *testing.T) {
assert := testify.New(t)
s := createQueue()
s.Enqueue(&jobItem{username: "username1"})
s.Enqueue(&jobItem{username: "username2"})
s.Enqueue(&jobItem{username: "username3"})
s.Enqueue(&jobItem{username: "username4"})
items := s.Dequeue(2)
assert.Len(items, 2)
assert.Equal("username1", items[0].username)
assert.Equal("username2", items[1].username)
assert.Equal(2, s.Size())
items = s.Dequeue(40)
assert.Len(items, 2)
assert.Equal("username3", items[0].username)
assert.Equal("username4", items[1].username)
})
}
// This is really stupid test just to get 100% coverage on this package :)
func TestBatchUuidsProvider_forever(t *testing.T) {
testify.True(t, forever())
}
type mojangUsernamesToUuidsRequestMock struct {
mock.Mock
}
func (o *mojangUsernamesToUuidsRequestMock) UsernamesToUuids(usernames []string) ([]*mojang.ProfileInfo, error) {
args := o.Called(usernames)
var result []*mojang.ProfileInfo
if casted, ok := args.Get(0).([]*mojang.ProfileInfo); ok {
result = casted
}
return result, args.Error(1)
}
type batchUuidsProviderGetUuidResult struct {
Result *mojang.ProfileInfo
Error error
}
type batchUuidsProviderTestSuite struct {
suite.Suite
Provider *BatchUuidsProvider
GetUuidAsync func(username string) chan *batchUuidsProviderGetUuidResult
Logger *mocks.WdMock
MojangApi *mojangUsernamesToUuidsRequestMock
Iterate func()
done func()
iterateChan chan bool
}
func (suite *batchUuidsProviderTestSuite) SetupTest() {
suite.Logger = &mocks.WdMock{}
suite.Provider = &BatchUuidsProvider{
Logger: suite.Logger,
IterationDelay: 0,
IterationSize: 10,
}
suite.iterateChan = make(chan bool)
forever = func() bool {
return <-suite.iterateChan
}
suite.Iterate = func() {
suite.iterateChan <- true
}
suite.done = func() {
suite.iterateChan <- false
}
suite.GetUuidAsync = func(username string) chan *batchUuidsProviderGetUuidResult {
c := make(chan *batchUuidsProviderGetUuidResult)
go func() {
profile, err := suite.Provider.GetUuid(username)
c <- &batchUuidsProviderGetUuidResult{
Result: profile,
Error: err,
}
}()
return c
}
suite.MojangApi = &mojangUsernamesToUuidsRequestMock{}
usernamesToUuids = suite.MojangApi.UsernamesToUuids
}
func (suite *batchUuidsProviderTestSuite) TearDownTest() {
suite.done()
time.Sleep(10 * time.Millisecond) // Add delay to let finish all goroutines before assert mocks calls
suite.MojangApi.AssertExpectations(suite.T())
suite.Logger.AssertExpectations(suite.T())
}
func TestBatchUuidsProvider(t *testing.T) {
suite.Run(t, new(batchUuidsProviderTestSuite))
}
func (suite *batchUuidsProviderTestSuite) TestGetUuidForOneUsername() {
expectedResult := &mojang.ProfileInfo{Id: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Name: "username"}
suite.Logger.On("IncCounter", "mojang_textures.usernames.queued", int64(1)).Once()
suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(1)).Once()
suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(0)).Once()
suite.Logger.On("RecordTimer", "mojang_textures.usernames.round_time", mock.Anything).Once()
suite.MojangApi.On("UsernamesToUuids", []string{"username"}).Once().Return([]*mojang.ProfileInfo{expectedResult}, nil)
resultChan := suite.GetUuidAsync("username")
suite.Iterate()
result := <-resultChan
suite.Assert().Equal(expectedResult, result.Result)
suite.Assert().Nil(result.Error)
}
func (suite *batchUuidsProviderTestSuite) TestGetUuidForTwoUsernames() {
expectedResult1 := &mojang.ProfileInfo{Id: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Name: "username1"}
expectedResult2 := &mojang.ProfileInfo{Id: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", Name: "username2"}
suite.Logger.On("IncCounter", "mojang_textures.usernames.queued", int64(1)).Twice()
suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(2)).Once()
suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(0)).Once()
suite.Logger.On("RecordTimer", "mojang_textures.usernames.round_time", mock.Anything).Once()
suite.MojangApi.On("UsernamesToUuids", []string{"username1", "username2"}).Once().Return([]*mojang.ProfileInfo{
expectedResult1,
expectedResult2,
}, nil)
resultChan1 := suite.GetUuidAsync("username1")
time.Sleep(time.Millisecond) // Just to keep order for the usernames
resultChan2 := suite.GetUuidAsync("username2")
time.Sleep(time.Millisecond) // Allow to all goroutines begin
suite.Iterate()
result1 := <-resultChan1
suite.Assert().Equal(expectedResult1, result1.Result)
suite.Assert().Nil(result1.Error)
result2 := <-resultChan2
suite.Assert().Equal(expectedResult2, result2.Result)
suite.Assert().Nil(result2.Error)
}
func (suite *batchUuidsProviderTestSuite) TestGetUuidForMoreThan10Usernames() {
usernames := make([]string, 12)
for i := 0; i < cap(usernames); i++ {
usernames[i] = randStr(8)
}
suite.Logger.On("IncCounter", "mojang_textures.usernames.queued", int64(1)).Times(12)
suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(10)).Once()
suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(2)).Once()
suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(2)).Once()
suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(0)).Once()
suite.Logger.On("RecordTimer", "mojang_textures.usernames.round_time", mock.Anything).Twice()
suite.MojangApi.On("UsernamesToUuids", usernames[0:10]).Once().Return([]*mojang.ProfileInfo{}, nil)
suite.MojangApi.On("UsernamesToUuids", usernames[10:12]).Once().Return([]*mojang.ProfileInfo{}, nil)
channels := make([]chan *batchUuidsProviderGetUuidResult, 12)
for i, username := range usernames {
channels[i] = suite.GetUuidAsync(username)
time.Sleep(time.Millisecond) // Just to keep order for the usernames
}
suite.Iterate()
suite.Iterate()
for _, channel := range channels {
<-channel
}
}
func (suite *batchUuidsProviderTestSuite) TestDoNothingWhenNoTasks() {
suite.Logger.On("IncCounter", "mojang_textures.usernames.queued", int64(1)).Once()
suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(1)).Once()
suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(0)).Twice()
suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(0)).Times(3)
suite.Logger.On("RecordTimer", "mojang_textures.usernames.round_time", mock.Anything)
suite.MojangApi.On("UsernamesToUuids", []string{"username"}).Once().Return([]*mojang.ProfileInfo{}, nil)
// Perform first iteration and await it finish
resultChan := suite.GetUuidAsync("username")
suite.Iterate()
result := <-resultChan
suite.Assert().Nil(result.Result)
suite.Assert().Nil(result.Error)
// Let it to perform a few more iterations to ensure, that there is no calls to external APIs
suite.Iterate()
suite.Iterate()
}
func (suite *batchUuidsProviderTestSuite) TestGetUuidForTwoUsernamesWithAnError() {
expectedError := &mojang.TooManyRequestsError{}
suite.Logger.On("IncCounter", "mojang_textures.usernames.queued", int64(1)).Twice()
suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(2)).Once()
suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(0)).Once()
suite.Logger.On("RecordTimer", "mojang_textures.usernames.round_time", mock.Anything).Once()
suite.MojangApi.On("UsernamesToUuids", []string{"username1", "username2"}).Once().Return(nil, expectedError)
resultChan1 := suite.GetUuidAsync("username1")
time.Sleep(time.Millisecond) // Just to keep order for the usernames
resultChan2 := suite.GetUuidAsync("username2")
time.Sleep(time.Millisecond) // Allow to all goroutines begin
suite.Iterate()
result1 := <-resultChan1
suite.Assert().Nil(result1.Result)
suite.Assert().Equal(expectedError, result1.Error)
result2 := <-resultChan2
suite.Assert().Nil(result2.Result)
suite.Assert().Equal(expectedError, result2.Error)
}
var replacer = strings.NewReplacer("-", "_", "=", "")
// https://stackoverflow.com/a/50581165
func randStr(len int) string {
buff := make([]byte, len)
_, _ = rand.Read(buff)
str := replacer.Replace(base64.URLEncoding.EncodeToString(buff))
// Base 64 can be longer than len
return str[:len]
}

39
api/mojang/queue/in_memory_textures_storage.go → mojangtextures/in_memory_textures_storage.go
View File

@@ -1,4 +1,4 @@
package queue package mojangtextures
import ( import (
"sync" "sync"
@@ -9,8 +9,6 @@ import (
"github.com/tevino/abool" "github.com/tevino/abool"
) )
var inMemoryStorageGCPeriod = 10 * time.Second
var inMemoryStoragePersistPeriod = time.Minute + 10*time.Second
var now = time.Now var now = time.Now
type inMemoryItem struct { type inMemoryItem struct {
@@ -18,33 +16,38 @@ type inMemoryItem struct {
timestamp int64 timestamp int64
} }
type inMemoryTexturesStorage struct { type InMemoryTexturesStorage struct {
GCPeriod time.Duration
Duration time.Duration
lock sync.Mutex lock sync.Mutex
data map[string]*inMemoryItem data map[string]*inMemoryItem
working *abool.AtomicBool working *abool.AtomicBool
} }
func CreateInMemoryTexturesStorage() *inMemoryTexturesStorage { func NewInMemoryTexturesStorage() *InMemoryTexturesStorage {
storage := &inMemoryTexturesStorage{ storage := &InMemoryTexturesStorage{
data: make(map[string]*inMemoryItem), GCPeriod: 10 * time.Second,
Duration: time.Minute + 10*time.Second,
data: make(map[string]*inMemoryItem),
} }
return storage return storage
} }
func (s *inMemoryTexturesStorage) Start() { func (s *InMemoryTexturesStorage) Start() {
if s.working == nil { if s.working == nil {
s.working = abool.New() s.working = abool.New()
} }
if !s.working.IsSet() { if !s.working.IsSet() {
go func() { go func() {
time.Sleep(inMemoryStorageGCPeriod) time.Sleep(s.GCPeriod)
// TODO: this can be reimplemented in future with channels, but right now I have no idea how to make it right // TODO: this can be reimplemented in future with channels, but right now I have no idea how to make it right
for s.working.IsSet() { for s.working.IsSet() {
start := time.Now() start := time.Now()
s.gc() s.gc()
time.Sleep(inMemoryStorageGCPeriod - time.Since(start)) time.Sleep(s.GCPeriod - time.Since(start))
} }
}() }()
} }
@@ -52,16 +55,16 @@ func (s *inMemoryTexturesStorage) Start() {
s.working.Set() s.working.Set()
} }
func (s *inMemoryTexturesStorage) Stop() { func (s *InMemoryTexturesStorage) Stop() {
s.working.UnSet() s.working.UnSet()
} }
func (s *inMemoryTexturesStorage) GetTextures(uuid string) (*mojang.SignedTexturesResponse, error) { func (s *InMemoryTexturesStorage) GetTextures(uuid string) (*mojang.SignedTexturesResponse, error) {
s.lock.Lock() s.lock.Lock()
defer s.lock.Unlock() defer s.lock.Unlock()
item, exists := s.data[uuid] item, exists := s.data[uuid]
validRange := getMinimalNotExpiredTimestamp() validRange := s.getMinimalNotExpiredTimestamp()
if !exists || validRange > item.timestamp { if !exists || validRange > item.timestamp {
return nil, &ValueNotFound{} return nil, &ValueNotFound{}
} }
@@ -69,7 +72,7 @@ func (s *inMemoryTexturesStorage) GetTextures(uuid string) (*mojang.SignedTextur
return item.textures, nil return item.textures, nil
} }
func (s *inMemoryTexturesStorage) StoreTextures(uuid string, textures *mojang.SignedTexturesResponse) { func (s *InMemoryTexturesStorage) StoreTextures(uuid string, textures *mojang.SignedTexturesResponse) {
var timestamp int64 var timestamp int64
if textures != nil { if textures != nil {
decoded := textures.DecodeTextures() decoded := textures.DecodeTextures()
@@ -91,11 +94,11 @@ func (s *inMemoryTexturesStorage) StoreTextures(uuid string, textures *mojang.Si
} }
} }
func (s *inMemoryTexturesStorage) gc() { func (s *InMemoryTexturesStorage) gc() {
s.lock.Lock() s.lock.Lock()
defer s.lock.Unlock() defer s.lock.Unlock()
maxTime := getMinimalNotExpiredTimestamp() maxTime := s.getMinimalNotExpiredTimestamp()
for uuid, value := range s.data { for uuid, value := range s.data {
if maxTime > value.timestamp { if maxTime > value.timestamp {
delete(s.data, uuid) delete(s.data, uuid)
@@ -103,8 +106,8 @@ func (s *inMemoryTexturesStorage) gc() {
} }
} }
func getMinimalNotExpiredTimestamp() int64 { func (s *InMemoryTexturesStorage) getMinimalNotExpiredTimestamp() int64 {
return unixNanoToUnixMicro(now().Add(inMemoryStoragePersistPeriod * time.Duration(-1)).UnixNano()) return unixNanoToUnixMicro(now().Add(s.Duration * time.Duration(-1)).UnixNano())
} }
func unixNanoToUnixMicro(unixNano int64) int64 { func unixNanoToUnixMicro(unixNano int64) int64 {

30
api/mojang/queue/in_memory_textures_storage_test.go → mojangtextures/in_memory_textures_storage_test.go
View File

@@ -1,4 +1,4 @@
package queue package mojangtextures
import ( import (
"time" "time"
@@ -48,7 +48,7 @@ func TestInMemoryTexturesStorage_GetTextures(t *testing.T) {
t.Run("get error when uuid is not exists", func(t *testing.T) { t.Run("get error when uuid is not exists", func(t *testing.T) {
assert := testify.New(t) assert := testify.New(t)
storage := CreateInMemoryTexturesStorage() storage := NewInMemoryTexturesStorage()
result, err := storage.GetTextures("b5d58475007d4f9e9ddd1403e2497579") result, err := storage.GetTextures("b5d58475007d4f9e9ddd1403e2497579")
assert.Nil(result) assert.Nil(result)
@@ -58,7 +58,7 @@ func TestInMemoryTexturesStorage_GetTextures(t *testing.T) {
t.Run("get textures object, when uuid is stored in the storage", func(t *testing.T) { t.Run("get textures object, when uuid is stored in the storage", func(t *testing.T) {
assert := testify.New(t) assert := testify.New(t)
storage := CreateInMemoryTexturesStorage() storage := NewInMemoryTexturesStorage()
storage.StoreTextures("dead24f9a4fa4877b7b04c8c6c72bb46", texturesWithSkin) storage.StoreTextures("dead24f9a4fa4877b7b04c8c6c72bb46", texturesWithSkin)
result, err := storage.GetTextures("dead24f9a4fa4877b7b04c8c6c72bb46") result, err := storage.GetTextures("dead24f9a4fa4877b7b04c8c6c72bb46")
@@ -69,7 +69,7 @@ func TestInMemoryTexturesStorage_GetTextures(t *testing.T) {
t.Run("get error when uuid is exists, but textures are expired", func(t *testing.T) { t.Run("get error when uuid is exists, but textures are expired", func(t *testing.T) {
assert := testify.New(t) assert := testify.New(t)
storage := CreateInMemoryTexturesStorage() storage := NewInMemoryTexturesStorage()
storage.StoreTextures("dead24f9a4fa4877b7b04c8c6c72bb46", texturesWithSkin) storage.StoreTextures("dead24f9a4fa4877b7b04c8c6c72bb46", texturesWithSkin)
now = func() time.Time { now = func() time.Time {
@@ -89,7 +89,7 @@ func TestInMemoryTexturesStorage_StoreTextures(t *testing.T) {
t.Run("store textures for previously not existed uuid", func(t *testing.T) { t.Run("store textures for previously not existed uuid", func(t *testing.T) {
assert := testify.New(t) assert := testify.New(t)
storage := CreateInMemoryTexturesStorage() storage := NewInMemoryTexturesStorage()
storage.StoreTextures("dead24f9a4fa4877b7b04c8c6c72bb46", texturesWithSkin) storage.StoreTextures("dead24f9a4fa4877b7b04c8c6c72bb46", texturesWithSkin)
result, err := storage.GetTextures("dead24f9a4fa4877b7b04c8c6c72bb46") result, err := storage.GetTextures("dead24f9a4fa4877b7b04c8c6c72bb46")
@@ -100,7 +100,7 @@ func TestInMemoryTexturesStorage_StoreTextures(t *testing.T) {
t.Run("override already existed textures for uuid", func(t *testing.T) { t.Run("override already existed textures for uuid", func(t *testing.T) {
assert := testify.New(t) assert := testify.New(t)
storage := CreateInMemoryTexturesStorage() storage := NewInMemoryTexturesStorage()
storage.StoreTextures("dead24f9a4fa4877b7b04c8c6c72bb46", texturesWithoutSkin) storage.StoreTextures("dead24f9a4fa4877b7b04c8c6c72bb46", texturesWithoutSkin)
storage.StoreTextures("dead24f9a4fa4877b7b04c8c6c72bb46", texturesWithSkin) storage.StoreTextures("dead24f9a4fa4877b7b04c8c6c72bb46", texturesWithSkin)
result, err := storage.GetTextures("dead24f9a4fa4877b7b04c8c6c72bb46") result, err := storage.GetTextures("dead24f9a4fa4877b7b04c8c6c72bb46")
@@ -113,7 +113,7 @@ func TestInMemoryTexturesStorage_StoreTextures(t *testing.T) {
t.Run("store nil textures", func(t *testing.T) { t.Run("store nil textures", func(t *testing.T) {
assert := testify.New(t) assert := testify.New(t)
storage := CreateInMemoryTexturesStorage() storage := NewInMemoryTexturesStorage()
storage.StoreTextures("dead24f9a4fa4877b7b04c8c6c72bb46", nil) storage.StoreTextures("dead24f9a4fa4877b7b04c8c6c72bb46", nil)
result, err := storage.GetTextures("dead24f9a4fa4877b7b04c8c6c72bb46") result, err := storage.GetTextures("dead24f9a4fa4877b7b04c8c6c72bb46")
@@ -131,7 +131,7 @@ func TestInMemoryTexturesStorage_StoreTextures(t *testing.T) {
} }
assert.PanicsWithValue("unable to decode textures", func() { assert.PanicsWithValue("unable to decode textures", func() {
storage := CreateInMemoryTexturesStorage() storage := NewInMemoryTexturesStorage()
storage.StoreTextures("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", toStore) storage.StoreTextures("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", toStore)
}) })
}) })
@@ -140,8 +140,9 @@ func TestInMemoryTexturesStorage_StoreTextures(t *testing.T) {
func TestInMemoryTexturesStorage_GarbageCollection(t *testing.T) { func TestInMemoryTexturesStorage_GarbageCollection(t *testing.T) {
assert := testify.New(t) assert := testify.New(t)
inMemoryStorageGCPeriod = 10 * time.Millisecond storage := NewInMemoryTexturesStorage()
inMemoryStoragePersistPeriod = 10 * time.Millisecond storage.GCPeriod = 10 * time.Millisecond
storage.Duration = 10 * time.Millisecond
textures1 := &mojang.SignedTexturesResponse{ textures1 := &mojang.SignedTexturesResponse{
Id: "dead24f9a4fa4877b7b04c8c6c72bb46", Id: "dead24f9a4fa4877b7b04c8c6c72bb46",
@@ -150,7 +151,7 @@ func TestInMemoryTexturesStorage_GarbageCollection(t *testing.T) {
{ {
Name: "textures", Name: "textures",
Value: mojang.EncodeTextures(&mojang.TexturesProp{ Value: mojang.EncodeTextures(&mojang.TexturesProp{
Timestamp: time.Now().Add(inMemoryStorageGCPeriod-time.Millisecond*time.Duration(5)).UnixNano() / 10e5, Timestamp: time.Now().Add(storage.GCPeriod-time.Millisecond*time.Duration(5)).UnixNano() / 10e5,
ProfileID: "dead24f9a4fa4877b7b04c8c6c72bb46", ProfileID: "dead24f9a4fa4877b7b04c8c6c72bb46",
ProfileName: "mock1", ProfileName: "mock1",
Textures: &mojang.TexturesResponse{}, Textures: &mojang.TexturesResponse{},
@@ -165,7 +166,7 @@ func TestInMemoryTexturesStorage_GarbageCollection(t *testing.T) {
{ {
Name: "textures", Name: "textures",
Value: mojang.EncodeTextures(&mojang.TexturesProp{ Value: mojang.EncodeTextures(&mojang.TexturesProp{
Timestamp: time.Now().Add(inMemoryStorageGCPeriod-time.Millisecond*time.Duration(15)).UnixNano() / 10e5, Timestamp: time.Now().Add(storage.GCPeriod-time.Millisecond*time.Duration(15)).UnixNano() / 10e5,
ProfileID: "b5d58475007d4f9e9ddd1403e2497579", ProfileID: "b5d58475007d4f9e9ddd1403e2497579",
ProfileName: "mock2", ProfileName: "mock2",
Textures: &mojang.TexturesResponse{}, Textures: &mojang.TexturesResponse{},
@@ -174,13 +175,12 @@ func TestInMemoryTexturesStorage_GarbageCollection(t *testing.T) {
}, },
} }
storage := CreateInMemoryTexturesStorage()
storage.StoreTextures("dead24f9a4fa4877b7b04c8c6c72bb46", textures1) storage.StoreTextures("dead24f9a4fa4877b7b04c8c6c72bb46", textures1)
storage.StoreTextures("b5d58475007d4f9e9ddd1403e2497579", textures2) storage.StoreTextures("b5d58475007d4f9e9ddd1403e2497579", textures2)
storage.Start() storage.Start()
time.Sleep(inMemoryStorageGCPeriod + time.Millisecond) // Let it start first iteration time.Sleep(storage.GCPeriod + time.Millisecond) // Let it start first iteration
_, textures1Err := storage.GetTextures("dead24f9a4fa4877b7b04c8c6c72bb46") _, textures1Err := storage.GetTextures("dead24f9a4fa4877b7b04c8c6c72bb46")
_, textures2Err := storage.GetTextures("b5d58475007d4f9e9ddd1403e2497579") _, textures2Err := storage.GetTextures("b5d58475007d4f9e9ddd1403e2497579")
@@ -188,7 +188,7 @@ func TestInMemoryTexturesStorage_GarbageCollection(t *testing.T) {
assert.Nil(textures1Err) assert.Nil(textures1Err)
assert.Error(textures2Err) assert.Error(textures2Err)
time.Sleep(inMemoryStorageGCPeriod + time.Millisecond) // Let another iteration happen time.Sleep(storage.GCPeriod + time.Millisecond) // Let another iteration happen
_, textures1Err = storage.GetTextures("dead24f9a4fa4877b7b04c8c6c72bb46") _, textures1Err = storage.GetTextures("dead24f9a4fa4877b7b04c8c6c72bb46")
_, textures2Err = storage.GetTextures("b5d58475007d4f9e9ddd1403e2497579") _, textures2Err = storage.GetTextures("b5d58475007d4f9e9ddd1403e2497579")

25
mojangtextures/mojang_api_textures_provider.go Normal file
View File

@@ -0,0 +1,25 @@
package mojangtextures
import (
"time"
"github.com/mono83/slf/wd"
"github.com/elyby/chrly/api/mojang"
)
var uuidToTextures = mojang.UuidToTextures
type MojangApiTexturesProvider struct {
Logger wd.Watchdog
}
func (ctx *MojangApiTexturesProvider) GetTextures(uuid string) (*mojang.SignedTexturesResponse, error) {
ctx.Logger.IncCounter("mojang_textures.textures.request", 1)
start := time.Now()
result, err := uuidToTextures(uuid, true)
ctx.Logger.RecordTimer("mojang_textures.textures.request_time", time.Since(start))
return result, err
}

82
mojangtextures/mojang_api_textures_provider_test.go Normal file
View File

@@ -0,0 +1,82 @@
package mojangtextures
import (
"testing"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/suite"
"github.com/elyby/chrly/api/mojang"
mocks "github.com/elyby/chrly/tests"
)
type mojangUuidToTexturesRequestMock struct {
mock.Mock
}
func (o *mojangUuidToTexturesRequestMock) UuidToTextures(uuid string, signed bool) (*mojang.SignedTexturesResponse, error) {
args := o.Called(uuid, signed)
var result *mojang.SignedTexturesResponse
if casted, ok := args.Get(0).(*mojang.SignedTexturesResponse); ok {
result = casted
}
return result, args.Error(1)
}
type mojangApiTexturesProviderTestSuite struct {
suite.Suite
Provider *MojangApiTexturesProvider
Logger *mocks.WdMock
MojangApi *mojangUuidToTexturesRequestMock
}
func (suite *mojangApiTexturesProviderTestSuite) SetupTest() {
suite.Logger = &mocks.WdMock{}
suite.MojangApi = &mojangUuidToTexturesRequestMock{}
suite.Provider = &MojangApiTexturesProvider{
Logger: suite.Logger,
}
uuidToTextures = suite.MojangApi.UuidToTextures
}
func (suite *mojangApiTexturesProviderTestSuite) TearDownTest() {
suite.MojangApi.AssertExpectations(suite.T())
suite.Logger.AssertExpectations(suite.T())
}
func TestMojangApiTexturesProvider(t *testing.T) {
suite.Run(t, new(mojangApiTexturesProviderTestSuite))
}
func (suite *mojangApiTexturesProviderTestSuite) TestGetTextures() {
expectedResult := &mojang.SignedTexturesResponse{
Id: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
Name: "username",
}
suite.MojangApi.On("UuidToTextures", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", true).Once().Return(expectedResult, nil)
suite.Logger.On("IncCounter", "mojang_textures.textures.request", int64(1)).Once()
suite.Logger.On("RecordTimer", "mojang_textures.textures.request_time", mock.Anything).Once()
result, err := suite.Provider.GetTextures("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
suite.Assert().Equal(expectedResult, result)
suite.Assert().Nil(err)
}
func (suite *mojangApiTexturesProviderTestSuite) TestGetTexturesWithError() {
expectedError := &mojang.TooManyRequestsError{}
suite.MojangApi.On("UuidToTextures", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", true).Once().Return(nil, expectedError)
suite.Logger.On("IncCounter", "mojang_textures.textures.request", int64(1)).Once()
suite.Logger.On("RecordTimer", "mojang_textures.textures.request_time", mock.Anything).Once()
result, err := suite.Provider.GetTextures("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
suite.Assert().Nil(result)
suite.Assert().Equal(expectedError, err)
}

225
mojangtextures/mojang_textures.go Normal file
View File

@@ -0,0 +1,225 @@
package mojangtextures
import (
"errors"
"net"
"net/url"
"regexp"
"strings"
"sync"
"syscall"
"time"
"github.com/mono83/slf/wd"
"github.com/elyby/chrly/api/mojang"
)
type broadcastResult struct {
textures *mojang.SignedTexturesResponse
error error
}
type broadcaster struct {
lock sync.Mutex
listeners map[string][]chan *broadcastResult
}
func createBroadcaster() *broadcaster {
return &broadcaster{
listeners: make(map[string][]chan *broadcastResult),
}
}
// Returns a boolean value, which will be true if the passed username didn't exist before
func (c *broadcaster) AddListener(username string, resultChan chan *broadcastResult) bool {
c.lock.Lock()
defer c.lock.Unlock()
val, alreadyHasSource := c.listeners[username]
if alreadyHasSource {
c.listeners[username] = append(val, resultChan)
return false
}
c.listeners[username] = []chan *broadcastResult{resultChan}
return true
}
func (c *broadcaster) BroadcastAndRemove(username string, result *broadcastResult) {
c.lock.Lock()
defer c.lock.Unlock()
val, ok := c.listeners[username]
if !ok {
return
}
for _, channel := range val {
go func(channel chan *broadcastResult) {
channel <- result
close(channel)
}(channel)
}
delete(c.listeners, username)
}
// https://help.mojang.com/customer/portal/articles/928638
var allowedUsernamesRegex = regexp.MustCompile(`^[\w_]{3,16}$`)
type UuidsProvider interface {
GetUuid(username string) (*mojang.ProfileInfo, error)
}
type TexturesProvider interface {
GetTextures(uuid string) (*mojang.SignedTexturesResponse, error)
}
type Provider struct {
UuidsProvider
TexturesProvider
Storage
Logger wd.Watchdog
onFirstCall sync.Once
*broadcaster
}
func (ctx *Provider) GetForUsername(username string) (*mojang.SignedTexturesResponse, error) {
ctx.onFirstCall.Do(func() {
ctx.broadcaster = createBroadcaster()
})
if !allowedUsernamesRegex.MatchString(username) {
ctx.Logger.IncCounter("mojang_textures.invalid_username", 1)
return nil, errors.New("invalid username")
}
username = strings.ToLower(username)
ctx.Logger.IncCounter("mojang_textures.request", 1)
uuid, err := ctx.Storage.GetUuid(username)
if err == nil && uuid == "" {
ctx.Logger.IncCounter("mojang_textures.usernames.cache_hit_nil", 1)
return nil, nil
}
if uuid != "" {
ctx.Logger.IncCounter("mojang_textures.usernames.cache_hit", 1)
textures, err := ctx.Storage.GetTextures(uuid)
if err == nil {
ctx.Logger.IncCounter("mojang_textures.textures.cache_hit", 1)
return textures, nil
}
}
resultChan := make(chan *broadcastResult)
isFirstListener := ctx.broadcaster.AddListener(username, resultChan)
if isFirstListener {
go ctx.getResultAndBroadcast(username, uuid)
} else {
ctx.Logger.IncCounter("mojang_textures.already_scheduled", 1)
}
result := <-resultChan
return result.textures, result.error
}
func (ctx *Provider) getResultAndBroadcast(username string, uuid string) {
start := time.Now()
result := ctx.getResult(username, uuid)
ctx.broadcaster.BroadcastAndRemove(username, result)
ctx.Logger.RecordTimer("mojang_textures.result_time", time.Since(start))
}
func (ctx *Provider) getResult(username string, uuid string) *broadcastResult {
if uuid == "" {
profile, err := ctx.UuidsProvider.GetUuid(username)
if err != nil {
ctx.handleMojangApiResponseError(err, "usernames")
return &broadcastResult{nil, err}
}
uuid = ""
if profile != nil {
uuid = profile.Id
}
_ = ctx.Storage.StoreUuid(username, uuid)
if uuid == "" {
ctx.Logger.IncCounter("mojang_textures.usernames.uuid_miss", 1)
return &broadcastResult{nil, nil}
}
ctx.Logger.IncCounter("mojang_textures.usernames.uuid_hit", 1)
}
textures, err := ctx.TexturesProvider.GetTextures(uuid)
if err != nil {
ctx.handleMojangApiResponseError(err, "textures")
return &broadcastResult{nil, err}
}
// Mojang can respond with an error, but it will still count as a hit,
// therefore store the result even if textures is nil to prevent 429 error
ctx.Storage.StoreTextures(uuid, textures)
if textures != nil {
ctx.Logger.IncCounter("mojang_textures.usernames.textures_hit", 1)
} else {
ctx.Logger.IncCounter("mojang_textures.usernames.textures_miss", 1)
}
return &broadcastResult{textures, nil}
}
func (ctx *Provider) handleMojangApiResponseError(err error, threadName string) {
errParam := wd.ErrParam(err)
threadParam := wd.NameParam(threadName)
ctx.Logger.Debug(":name: Got response error :err", threadParam, errParam)
switch err.(type) {
case mojang.ResponseError:
if _, ok := err.(*mojang.BadRequestError); ok {
ctx.Logger.Warning(":name: Got 400 Bad Request :err", threadParam, errParam)
return
}
if _, ok := err.(*mojang.ForbiddenError); ok {
ctx.Logger.Warning(":name: Got 403 Forbidden :err", threadParam, errParam)
return
}
if _, ok := err.(*mojang.TooManyRequestsError); ok {
ctx.Logger.Warning(":name: Got 429 Too Many Requests :err", threadParam, errParam)
return
}
return
case net.Error:
if err.(net.Error).Timeout() {
return
}
if _, ok := err.(*url.Error); ok {
return
}
if opErr, ok := err.(*net.OpError); ok && (opErr.Op == "dial" || opErr.Op == "read") {
return
}
if err == syscall.ECONNREFUSED {
return
}
}
ctx.Logger.Emergency(":name: Unknown Mojang response error: :err", threadParam, errParam)
}

439
mojangtextures/mojang_textures_test.go Normal file
View File

@@ -0,0 +1,439 @@
package mojangtextures
import (
"errors"
"net"
"net/url"
"sync"
"syscall"
"testing"
"time"
testify "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/suite"
"github.com/elyby/chrly/api/mojang"
mocks "github.com/elyby/chrly/tests"
)
func TestBroadcaster(t *testing.T) {
t.Run("GetOrAppend", func(t *testing.T) {
t.Run("first call when username didn't exist before should return true", func(t *testing.T) {
assert := testify.New(t)
broadcaster := createBroadcaster()
channel := make(chan *broadcastResult)
isFirstListener := broadcaster.AddListener("mock", channel)
assert.True(isFirstListener)
listeners, ok := broadcaster.listeners["mock"]
assert.True(ok)
assert.Len(listeners, 1)
assert.Equal(channel, listeners[0])
})
t.Run("subsequent calls should return false", func(t *testing.T) {
assert := testify.New(t)
broadcaster := createBroadcaster()
channel1 := make(chan *broadcastResult)
isFirstListener := broadcaster.AddListener("mock", channel1)
assert.True(isFirstListener)
channel2 := make(chan *broadcastResult)
isFirstListener = broadcaster.AddListener("mock", channel2)
assert.False(isFirstListener)
channel3 := make(chan *broadcastResult)
isFirstListener = broadcaster.AddListener("mock", channel3)
assert.False(isFirstListener)
})
})
t.Run("BroadcastAndRemove", func(t *testing.T) {
t.Run("should broadcast to all listeners and remove the key", func(t *testing.T) {
assert := testify.New(t)
broadcaster := createBroadcaster()
channel1 := make(chan *broadcastResult)
channel2 := make(chan *broadcastResult)
broadcaster.AddListener("mock", channel1)
broadcaster.AddListener("mock", channel2)
result := &broadcastResult{}
broadcaster.BroadcastAndRemove("mock", result)
assert.Equal(result, <-channel1)
assert.Equal(result, <-channel2)
channel3 := make(chan *broadcastResult)
isFirstListener := broadcaster.AddListener("mock", channel3)
assert.True(isFirstListener)
})
t.Run("call on not exists username", func(t *testing.T) {
assert := testify.New(t)
assert.NotPanics(func() {
broadcaster := createBroadcaster()
broadcaster.BroadcastAndRemove("mock", &broadcastResult{})
})
})
})
}
type mockUuidsProvider struct {
mock.Mock
}
func (m *mockUuidsProvider) GetUuid(username string) (*mojang.ProfileInfo, error) {
args := m.Called(username)
var result *mojang.ProfileInfo
if casted, ok := args.Get(0).(*mojang.ProfileInfo); ok {
result = casted
}
return result, args.Error(1)
}
type mockTexturesProvider struct {
mock.Mock
}
func (m *mockTexturesProvider) GetTextures(uuid string) (*mojang.SignedTexturesResponse, error) {
args := m.Called(uuid)
var result *mojang.SignedTexturesResponse
if casted, ok := args.Get(0).(*mojang.SignedTexturesResponse); ok {
result = casted
}
return result, args.Error(1)
}
type mockStorage struct {
mock.Mock
}
func (m *mockStorage) GetUuid(username string) (string, error) {
args := m.Called(username)
return args.String(0), args.Error(1)
}
func (m *mockStorage) StoreUuid(username string, uuid string) error {
args := m.Called(username, uuid)
return args.Error(0)
}
func (m *mockStorage) GetTextures(uuid string) (*mojang.SignedTexturesResponse, error) {
args := m.Called(uuid)
var result *mojang.SignedTexturesResponse
if casted, ok := args.Get(0).(*mojang.SignedTexturesResponse); ok {
result = casted
}
return result, args.Error(1)
}
func (m *mockStorage) StoreTextures(uuid string, textures *mojang.SignedTexturesResponse) {
m.Called(uuid, textures)
}
type providerTestSuite struct {
suite.Suite
Provider *Provider
UuidsProvider *mockUuidsProvider
TexturesProvider *mockTexturesProvider
Storage *mockStorage
Logger *mocks.WdMock
}
func (suite *providerTestSuite) SetupTest() {
suite.UuidsProvider = &mockUuidsProvider{}
suite.TexturesProvider = &mockTexturesProvider{}
suite.Storage = &mockStorage{}
suite.Logger = &mocks.WdMock{}
suite.Provider = &Provider{
UuidsProvider: suite.UuidsProvider,
TexturesProvider: suite.TexturesProvider,
Storage: suite.Storage,
Logger: suite.Logger,
}
}
func (suite *providerTestSuite) TearDownTest() {
// time.Sleep(10 * time.Millisecond) // Add delay to let finish all goroutines before assert mocks calls
suite.UuidsProvider.AssertExpectations(suite.T())
suite.TexturesProvider.AssertExpectations(suite.T())
suite.Storage.AssertExpectations(suite.T())
suite.Logger.AssertExpectations(suite.T())
}
func TestProvider(t *testing.T) {
suite.Run(t, new(providerTestSuite))
}
func (suite *providerTestSuite) TestGetForUsernameWithoutAnyCache() {
expectedResult := &mojang.SignedTexturesResponse{Id: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Name: "username"}
suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Once()
suite.Logger.On("IncCounter", "mojang_textures.usernames.uuid_hit", int64(1)).Once()
suite.Logger.On("IncCounter", "mojang_textures.usernames.textures_hit", int64(1)).Once()
suite.Logger.On("RecordTimer", "mojang_textures.result_time", mock.Anything).Once()
suite.Storage.On("GetUuid", "username").Once().Return("", &ValueNotFound{})
suite.Storage.On("StoreUuid", "username", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").Once().Return(nil)
suite.Storage.On("StoreTextures", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", expectedResult).Once()
suite.UuidsProvider.On("GetUuid", "username").Once().Return(&mojang.ProfileInfo{
Id: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
Name: "username",
}, nil)
suite.TexturesProvider.On("GetTextures", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").Once().Return(expectedResult, nil)
result, err := suite.Provider.GetForUsername("username")
suite.Assert().Nil(err)
suite.Assert().Equal(expectedResult, result)
}
func (suite *providerTestSuite) TestGetForUsernameWithCachedUuid() {
expectedResult := &mojang.SignedTexturesResponse{Id: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Name: "username"}
suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Once()
suite.Logger.On("IncCounter", "mojang_textures.usernames.cache_hit", int64(1)).Once()
suite.Logger.On("IncCounter", "mojang_textures.usernames.textures_hit", int64(1)).Once()
suite.Logger.On("RecordTimer", "mojang_textures.result_time", mock.Anything).Once()
suite.Storage.On("GetUuid", "username").Once().Return("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", nil)
suite.Storage.On("GetTextures", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").Once().Return(nil, &ValueNotFound{})
suite.Storage.On("StoreTextures", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", expectedResult).Once()
suite.TexturesProvider.On("GetTextures", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").Return(expectedResult, nil)
result, err := suite.Provider.GetForUsername("username")
suite.Assert().Nil(err)
suite.Assert().Equal(expectedResult, result)
}
func (suite *providerTestSuite) TestGetForUsernameWithFullyCachedResult() {
expectedResult := &mojang.SignedTexturesResponse{Id: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Name: "username"}
suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Once()
suite.Logger.On("IncCounter", "mojang_textures.usernames.cache_hit", int64(1)).Once()
suite.Logger.On("IncCounter", "mojang_textures.textures.cache_hit", int64(1)).Once()
suite.Storage.On("GetUuid", "username").Once().Return("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", nil)
suite.Storage.On("GetTextures", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").Once().Return(expectedResult, nil)
result, err := suite.Provider.GetForUsername("username")
suite.Assert().Nil(err)
suite.Assert().Equal(expectedResult, result)
}
func (suite *providerTestSuite) TestGetForUsernameWithCachedUnknownUuid() {
suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Once()
suite.Logger.On("IncCounter", "mojang_textures.usernames.cache_hit_nil", int64(1)).Once()
suite.Storage.On("GetUuid", "username").Once().Return("", nil)
result, err := suite.Provider.GetForUsername("username")
suite.Assert().Nil(result)
suite.Assert().Nil(err)
}
func (suite *providerTestSuite) TestGetForUsernameWhichHasNoMojangAccount() {
suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Once()
suite.Logger.On("IncCounter", "mojang_textures.usernames.uuid_miss", int64(1)).Once()
suite.Logger.On("RecordTimer", "mojang_textures.result_time", mock.Anything).Once()
suite.Storage.On("GetUuid", "username").Once().Return("", &ValueNotFound{})
suite.Storage.On("StoreUuid", "username", "").Once().Return(nil)
suite.UuidsProvider.On("GetUuid", "username").Once().Return(nil, nil)
result, err := suite.Provider.GetForUsername("username")
suite.Assert().Nil(err)
suite.Assert().Nil(result)
}
func (suite *providerTestSuite) TestGetForUsernameWhichHasMojangAccountButHasNoMojangSkin() {
var expectedResult *mojang.SignedTexturesResponse
suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Once()
suite.Logger.On("IncCounter", "mojang_textures.usernames.uuid_hit", int64(1)).Once()
suite.Logger.On("IncCounter", "mojang_textures.usernames.textures_miss", int64(1)).Once()
suite.Logger.On("RecordTimer", "mojang_textures.result_time", mock.Anything).Once()
suite.Storage.On("GetUuid", "username").Once().Return("", &ValueNotFound{})
suite.Storage.On("StoreUuid", "username", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").Once().Return(nil)
suite.Storage.On("StoreTextures", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", expectedResult).Once()
suite.UuidsProvider.On("GetUuid", "username").Once().Return(&mojang.ProfileInfo{
Id: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
Name: "username",
}, nil)
suite.TexturesProvider.On("GetTextures", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").Once().Return(expectedResult, nil)
result, err := suite.Provider.GetForUsername("username")
suite.Assert().Equal(expectedResult, result)
suite.Assert().Nil(err)
}
func (suite *providerTestSuite) TestGetForTheSameUsernames() {
expectedResult := &mojang.SignedTexturesResponse{Id: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Name: "username"}
suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Twice()
suite.Logger.On("IncCounter", "mojang_textures.already_scheduled", int64(1)).Once()
suite.Logger.On("IncCounter", "mojang_textures.usernames.uuid_hit", int64(1)).Once()
suite.Logger.On("IncCounter", "mojang_textures.usernames.textures_hit", int64(1)).Once()
suite.Logger.On("RecordTimer", "mojang_textures.result_time", mock.Anything).Once()
suite.Storage.On("GetUuid", "username").Twice().Return("", &ValueNotFound{})
suite.Storage.On("StoreUuid", "username", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").Once().Return(nil)
suite.Storage.On("StoreTextures", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", expectedResult).Once()
// If possible, than remove this .After call
suite.UuidsProvider.On("GetUuid", "username").Once().After(time.Millisecond).Return(&mojang.ProfileInfo{
Id: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
Name: "username",
}, nil)
suite.TexturesProvider.On("GetTextures", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").Once().Return(expectedResult, nil)
results := make([]*mojang.SignedTexturesResponse, 2)
var wg sync.WaitGroup
for i := 0; i < 2; i++ {
wg.Add(1)
go func(i int) {
textures, _ := suite.Provider.GetForUsername("username")
results[i] = textures
wg.Done()
}(i)
}
wg.Wait()
suite.Assert().Equal(expectedResult, results[0])
suite.Assert().Equal(expectedResult, results[1])
}
func (suite *providerTestSuite) TestGetForNotAllowedMojangUsername() {
suite.Logger.On("IncCounter", "mojang_textures.invalid_username", int64(1)).Once()
result, err := suite.Provider.GetForUsername("Not allowed")
suite.Assert().Error(err, "invalid username")
suite.Assert().Nil(result)
}
type timeoutError struct {
}
func (*timeoutError) Error() string { return "timeout error" }
func (*timeoutError) Timeout() bool { return true }
func (*timeoutError) Temporary() bool { return false }
var expectedErrors = []error{
&mojang.BadRequestError{},
&mojang.ForbiddenError{},
&mojang.TooManyRequestsError{},
&mojang.ServerError{},
&timeoutError{},
&url.Error{Op: "GET", URL: "http://localhost"},
&net.OpError{Op: "read"},
&net.OpError{Op: "dial"},
syscall.ECONNREFUSED,
}
func (suite *providerTestSuite) TestShouldNotLogErrorWhenExpectedErrorReturnedFromUsernameToUuidRequest() {
suite.Logger.On("IncCounter", mock.Anything, mock.Anything)
suite.Logger.On("RecordTimer", mock.Anything, mock.Anything)
suite.Logger.On("Debug", ":name: Got response error :err", mock.Anything, mock.Anything).Times(len(expectedErrors))
suite.Logger.On("Warning", ":name: Got 400 Bad Request :err", mock.Anything, mock.Anything).Once()
suite.Logger.On("Warning", ":name: Got 403 Forbidden :err", mock.Anything, mock.Anything).Once()
suite.Logger.On("Warning", ":name: Got 429 Too Many Requests :err", mock.Anything, mock.Anything).Once()
suite.Storage.On("GetUuid", "username").Return("", &ValueNotFound{})
for _, err := range expectedErrors {
suite.UuidsProvider.On("GetUuid", "username").Once().Return(nil, err)
result, err := suite.Provider.GetForUsername("username")
suite.Assert().Nil(result)
suite.Assert().NotNil(err)
suite.UuidsProvider.AssertExpectations(suite.T())
suite.UuidsProvider.ExpectedCalls = nil // https://github.com/stretchr/testify/issues/558#issuecomment-372112364
}
}
func (suite *providerTestSuite) TestShouldLogEmergencyOnUnexpectedErrorReturnedFromUsernameToUuidRequest() {
suite.Logger.On("IncCounter", mock.Anything, mock.Anything)
suite.Logger.On("RecordTimer", mock.Anything, mock.Anything)
suite.Logger.On("Debug", ":name: Got response error :err", mock.Anything, mock.Anything).Once()
suite.Logger.On("Emergency", ":name: Unknown Mojang response error: :err", mock.Anything, mock.Anything).Once()
suite.Storage.On("GetUuid", "username").Return("", &ValueNotFound{})
suite.UuidsProvider.On("GetUuid", "username").Once().Return(nil, errors.New("unexpected error"))
result, err := suite.Provider.GetForUsername("username")
suite.Assert().Nil(result)
suite.Assert().NotNil(err)
}
func (suite *providerTestSuite) TestShouldNotLogErrorWhenExpectedErrorReturnedFromUuidToTexturesRequest() {
suite.Logger.On("IncCounter", mock.Anything, mock.Anything)
suite.Logger.On("RecordTimer", mock.Anything, mock.Anything)
suite.Logger.On("Debug", ":name: Got response error :err", mock.Anything, mock.Anything).Times(len(expectedErrors))
suite.Logger.On("Warning", ":name: Got 400 Bad Request :err", mock.Anything, mock.Anything).Once()
suite.Logger.On("Warning", ":name: Got 403 Forbidden :err", mock.Anything, mock.Anything).Once()
suite.Logger.On("Warning", ":name: Got 429 Too Many Requests :err", mock.Anything, mock.Anything).Once()
suite.Storage.On("GetUuid", "username").Return("", &ValueNotFound{})
suite.Storage.On("StoreUuid", "username", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").Return(nil)
// suite.Storage.On("GetTextures", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").Return(nil, &ValueNotFound{})
// suite.Storage.On("StoreTextures", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", (*mojang.SignedTexturesResponse)(nil))
for _, err := range expectedErrors {
suite.UuidsProvider.On("GetUuid", "username").Once().Return(&mojang.ProfileInfo{
Id: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
Name: "username",
}, nil)
suite.TexturesProvider.On("GetTextures", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").Once().Return(nil, err)
result, err := suite.Provider.GetForUsername("username")
suite.Assert().Nil(result)
suite.Assert().NotNil(err)
suite.UuidsProvider.AssertExpectations(suite.T())
suite.TexturesProvider.AssertExpectations(suite.T())
suite.UuidsProvider.ExpectedCalls = nil // https://github.com/stretchr/testify/issues/558#issuecomment-372112364
suite.TexturesProvider.ExpectedCalls = nil // https://github.com/stretchr/testify/issues/558#issuecomment-372112364
}
}
func (suite *providerTestSuite) TestShouldLogEmergencyOnUnexpectedErrorReturnedFromUuidToTexturesRequest() {
suite.Logger.On("IncCounter", mock.Anything, mock.Anything)
suite.Logger.On("RecordTimer", mock.Anything, mock.Anything)
suite.Logger.On("Debug", ":name: Got response error :err", mock.Anything, mock.Anything).Once()
suite.Logger.On("Emergency", ":name: Unknown Mojang response error: :err", mock.Anything, mock.Anything).Once()
suite.Storage.On("GetUuid", "username").Return("", &ValueNotFound{})
suite.Storage.On("StoreUuid", "username", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").Return(nil)
suite.UuidsProvider.On("GetUuid", "username").Once().Return(&mojang.ProfileInfo{
Id: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
Name: "username",
}, nil)
suite.TexturesProvider.On("GetTextures", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").Once().Return(nil, errors.New("unexpected error"))
result, err := suite.Provider.GetForUsername("username")
suite.Assert().Nil(result)
suite.Assert().NotNil(err)
}

61
mojangtextures/storage.go Normal file
View File

@@ -0,0 +1,61 @@
package mojangtextures
import (
"github.com/elyby/chrly/api/mojang"
)
// UuidsStorage is a key-value storage of Mojang usernames pairs to its UUIDs,
// used to reduce the load on the account information queue
type UuidsStorage interface {
// Since only primitive types are used in this method, you should return a special error ValueNotFound
// to return the information that no error has occurred and username does not have uuid
GetUuid(username string) (string, error)
// An empty uuid value can be passed if the corresponding account has not been found
StoreUuid(username string, uuid string) error
}
// TexturesStorage is a Mojang's textures storage, used as a values cache to avoid 429 errors
type TexturesStorage interface {
// Error should not have nil value only if the repository failed to determine if there are any textures
// for this uuid or not at all. If there is information about the absence of textures, nil nil should be returned
GetTextures(uuid string) (*mojang.SignedTexturesResponse, error)
// The nil value can be passed when there are no textures for the corresponding uuid and we know about it
StoreTextures(uuid string, textures *mojang.SignedTexturesResponse)
}
type Storage interface {
UuidsStorage
TexturesStorage
}
// SeparatedStorage allows you to use separate storage engines to satisfy
// the Storage interface
type SeparatedStorage struct {
UuidsStorage
TexturesStorage
}
func (s *SeparatedStorage) GetUuid(username string) (string, error) {
return s.UuidsStorage.GetUuid(username)
}
func (s *SeparatedStorage) StoreUuid(username string, uuid string) error {
return s.UuidsStorage.StoreUuid(username, uuid)
}
func (s *SeparatedStorage) GetTextures(uuid string) (*mojang.SignedTexturesResponse, error) {
return s.TexturesStorage.GetTextures(uuid)
}
func (s *SeparatedStorage) StoreTextures(uuid string, textures *mojang.SignedTexturesResponse) {
s.TexturesStorage.StoreTextures(uuid, textures)
}
// This error can be used to indicate, that requested
// value doesn't exists in the storage
type ValueNotFound struct {
}
func (*ValueNotFound) Error() string {
return "value not found in the storage"
}

6
api/mojang/queue/storage_test.go → mojangtextures/storage_test.go
View File

@@ -1,4 +1,4 @@
package queue package mojangtextures
import ( import (
"github.com/elyby/chrly/api/mojang" "github.com/elyby/chrly/api/mojang"
@@ -41,11 +41,11 @@ func (m *texturesStorageMock) StoreTextures(uuid string, textures *mojang.Signed
} }
func TestSplittedStorage(t *testing.T) { func TestSplittedStorage(t *testing.T) {
createMockedStorage := func() (*SplittedStorage, *uuidsStorageMock, *texturesStorageMock) { createMockedStorage := func() (*SeparatedStorage, *uuidsStorageMock, *texturesStorageMock) {
uuidsStorage := &uuidsStorageMock{} uuidsStorage := &uuidsStorageMock{}
texturesStorage := &texturesStorageMock{} texturesStorage := &texturesStorageMock{}
return &SplittedStorage{uuidsStorage, texturesStorage}, uuidsStorage, texturesStorage return &SeparatedStorage{uuidsStorage, texturesStorage}, uuidsStorage, texturesStorage
} }
t.Run("GetUuid", func(t *testing.T) { t.Run("GetUuid", func(t *testing.T) {

33
tests/mojang_textures_queue_mock.go
View File

@@ -1,33 +0,0 @@
package tests
import (
"github.com/elyby/chrly/api/mojang"
"github.com/stretchr/testify/mock"
)
type MojangTexturesQueueMock struct {
mock.Mock
}
func (m *MojangTexturesQueueMock) GetTexturesForUsername(username string) chan *mojang.SignedTexturesResponse {
args := m.Called(username)
result := make(chan *mojang.SignedTexturesResponse)
arg := args.Get(0)
switch arg.(type) {
case *mojang.SignedTexturesResponse:
go func() {
result <- arg.(*mojang.SignedTexturesResponse)
}()
case chan *mojang.SignedTexturesResponse:
return arg.(chan *mojang.SignedTexturesResponse)
case nil:
go func() {
result <- nil
}()
default:
panic("unsupported return value")
}
return result
}