From 0644dfe0218fdb848441f0fc85283011e0e9d3e3 Mon Sep 17 00:00:00 2001 From: ErickSkrauch Date: Thu, 21 Nov 2019 01:33:13 +0300 Subject: [PATCH] Completely rework mojang textures queue implementation, split it across separate data providers --- CHANGELOG.md | 18 +- api/mojang/queue/broadcast.go | 53 -- api/mojang/queue/broadcast_test.go | 75 --- api/mojang/queue/jobs_structure.go | 56 -- api/mojang/queue/jobs_structure_test.go | 47 -- api/mojang/queue/queue.go | 221 -------- api/mojang/queue/queue_test.go | 525 ------------------ api/mojang/queue/storage.go | 53 -- cmd/serve.go | 30 +- db/factory.go | 4 +- db/filesystem.go | 4 +- db/redis.go | 8 +- http/cape.go | 4 +- http/cape_test.go | 22 +- http/http.go | 10 +- http/http_test.go | 70 ++- http/signed_textures.go | 5 +- http/signed_textures_test.go | 2 +- http/skin.go | 4 +- http/skin_test.go | 4 +- http/textures.go | 4 +- http/textures_test.go | 4 +- interfaces/repositories.go | 4 +- mojangtextures/batch_uuids_provider.go | 133 +++++ mojangtextures/batch_uuids_provider_test.go | 285 ++++++++++ .../in_memory_textures_storage.go | 39 +- .../in_memory_textures_storage_test.go | 30 +- .../mojang_api_textures_provider.go | 25 + .../mojang_api_textures_provider_test.go | 82 +++ mojangtextures/mojang_textures.go | 225 ++++++++ mojangtextures/mojang_textures_test.go | 439 +++++++++++++++ mojangtextures/storage.go | 61 ++ .../queue => mojangtextures}/storage_test.go | 6 +- tests/mojang_textures_queue_mock.go | 33 -- 34 files changed, 1405 insertions(+), 1180 deletions(-) delete mode 100644 api/mojang/queue/broadcast.go delete mode 100644 api/mojang/queue/broadcast_test.go delete mode 100644 api/mojang/queue/jobs_structure.go delete mode 100644 api/mojang/queue/jobs_structure_test.go delete mode 100644 api/mojang/queue/queue.go delete mode 100644 api/mojang/queue/queue_test.go delete mode 100644 api/mojang/queue/storage.go create mode 100644 mojangtextures/batch_uuids_provider.go create mode 100644 mojangtextures/batch_uuids_provider_test.go rename {api/mojang/queue => mojangtextures}/in_memory_textures_storage.go (62%) rename {api/mojang/queue => mojangtextures}/in_memory_textures_storage_test.go (85%) create mode 100644 mojangtextures/mojang_api_textures_provider.go create mode 100644 mojangtextures/mojang_api_textures_provider_test.go create mode 100644 mojangtextures/mojang_textures.go create mode 100644 mojangtextures/mojang_textures_test.go create mode 100644 mojangtextures/storage.go rename {api/mojang/queue => mojangtextures}/storage_test.go (91%) delete mode 100644 tests/mojang_textures_queue_mock.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 4753aa5..450d8e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,14 +5,26 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] - xxxx-xx-xx +### Added +- New StatsD metrics: + - Counters: + - `ely.skinsystem.{hostname}.app.mojang_textures.usernames.textures_hit` + - `ely.skinsystem.{hostname}.app.mojang_textures.usernames.textures_miss` + +### Fixed +- `ely.skinsystem.{hostname}.app.mojang_textures.usernames.iteration_size` and + `ely.skinsystem.{hostname}.app.mojang_textures.usernames.queue_size` are now updated even if the queue is empty. + +### Changed +- Event `ely.skinsystem.{hostname}.app.mojang_textures.already_in_queue` has been renamed into `ely.skinsystem.{hostname}.app.mojang_textures.already_scheduled`. ## [4.3.0] - 2019-11-08 ### Added -- 403 Forbidden errors from the Mojang's API are now logged -- `QUEUE_LOOP_DELAY` configuration param to adjust Mojang's textures queue performance +- 403 Forbidden errors from the Mojang's API are now logged. +- `QUEUE_LOOP_DELAY` configuration param to adjust Mojang's textures queue performance. ### Changed -- Mojang's textures queue loop is now has an iteration delay of 2.5 seconds (was 1) +- Mojang's textures queue loop is now has an iteration delay of 2.5 seconds (was 1). - Bumped Go version to 1.13. ## [4.2.3] - 2019-10-03 diff --git a/api/mojang/queue/broadcast.go b/api/mojang/queue/broadcast.go deleted file mode 100644 index 56c3b5e..0000000 --- a/api/mojang/queue/broadcast.go +++ /dev/null @@ -1,53 +0,0 @@ -package queue - -import ( - "sync" - - "github.com/elyby/chrly/api/mojang" -) - -type broadcastMap struct { - lock sync.Mutex - listeners map[string][]chan *mojang.SignedTexturesResponse -} - -func newBroadcaster() *broadcastMap { - return &broadcastMap{ - listeners: make(map[string][]chan *mojang.SignedTexturesResponse), - } -} - -// Returns a boolean value, which will be true if the username passed didn't exist before -func (c *broadcastMap) AddListener(username string, resultChan chan *mojang.SignedTexturesResponse) bool { - c.lock.Lock() - defer c.lock.Unlock() - - val, alreadyHasSource := c.listeners[username] - if alreadyHasSource { - c.listeners[username] = append(val, resultChan) - return false - } - - c.listeners[username] = []chan *mojang.SignedTexturesResponse{resultChan} - - return true -} - -func (c *broadcastMap) BroadcastAndRemove(username string, result *mojang.SignedTexturesResponse) { - c.lock.Lock() - defer c.lock.Unlock() - - val, ok := c.listeners[username] - if !ok { - return - } - - for _, channel := range val { - go func(channel chan *mojang.SignedTexturesResponse) { - channel <- result - close(channel) - }(channel) - } - - delete(c.listeners, username) -} diff --git a/api/mojang/queue/broadcast_test.go b/api/mojang/queue/broadcast_test.go deleted file mode 100644 index 910ea8b..0000000 --- a/api/mojang/queue/broadcast_test.go +++ /dev/null @@ -1,75 +0,0 @@ -package queue - -import ( - "github.com/elyby/chrly/api/mojang" - - testify "github.com/stretchr/testify/assert" - "testing" -) - -func TestBroadcastMap_GetOrAppend(t *testing.T) { - t.Run("first call when username didn't exist before should return true", func(t *testing.T) { - assert := testify.New(t) - - broadcaster := newBroadcaster() - channel := make(chan *mojang.SignedTexturesResponse) - isFirstListener := broadcaster.AddListener("mock", channel) - - assert.True(isFirstListener) - listeners, ok := broadcaster.listeners["mock"] - assert.True(ok) - assert.Len(listeners, 1) - assert.Equal(channel, listeners[0]) - }) - - t.Run("subsequent calls should return false", func(t *testing.T) { - assert := testify.New(t) - - broadcaster := newBroadcaster() - channel1 := make(chan *mojang.SignedTexturesResponse) - isFirstListener := broadcaster.AddListener("mock", channel1) - - assert.True(isFirstListener) - - channel2 := make(chan *mojang.SignedTexturesResponse) - isFirstListener = broadcaster.AddListener("mock", channel2) - - assert.False(isFirstListener) - - channel3 := make(chan *mojang.SignedTexturesResponse) - isFirstListener = broadcaster.AddListener("mock", channel3) - - assert.False(isFirstListener) - }) -} - -func TestBroadcastMap_BroadcastAndRemove(t *testing.T) { - t.Run("should broadcast to all listeners and remove the key", func(t *testing.T) { - assert := testify.New(t) - - broadcaster := newBroadcaster() - channel1 := make(chan *mojang.SignedTexturesResponse) - channel2 := make(chan *mojang.SignedTexturesResponse) - broadcaster.AddListener("mock", channel1) - broadcaster.AddListener("mock", channel2) - - result := &mojang.SignedTexturesResponse{Id: "mockUuid"} - broadcaster.BroadcastAndRemove("mock", result) - - assert.Equal(result, <-channel1) - assert.Equal(result, <-channel2) - - channel3 := make(chan *mojang.SignedTexturesResponse) - isFirstListener := broadcaster.AddListener("mock", channel3) - assert.True(isFirstListener) - }) - - t.Run("call on not exists username", func(t *testing.T) { - assert := testify.New(t) - - assert.NotPanics(func() { - broadcaster := newBroadcaster() - broadcaster.BroadcastAndRemove("mock", &mojang.SignedTexturesResponse{}) - }) - }) -} diff --git a/api/mojang/queue/jobs_structure.go b/api/mojang/queue/jobs_structure.go deleted file mode 100644 index 987be3d..0000000 --- a/api/mojang/queue/jobs_structure.go +++ /dev/null @@ -1,56 +0,0 @@ -// Based on the implementation from https://flaviocopes.com/golang-data-structure-queue/ - -package queue - -import ( - "sync" - - "github.com/elyby/chrly/api/mojang" -) - -type jobItem struct { - Username string - RespondTo chan *mojang.SignedTexturesResponse -} - -type jobsQueue struct { - lock sync.Mutex - items []*jobItem -} - -func (s *jobsQueue) New() *jobsQueue { - s.items = []*jobItem{} - return s -} - -func (s *jobsQueue) Enqueue(t *jobItem) { - s.lock.Lock() - defer s.lock.Unlock() - - s.items = append(s.items, t) -} - -func (s *jobsQueue) Dequeue(n int) []*jobItem { - s.lock.Lock() - defer s.lock.Unlock() - - if n > s.Size() { - n = s.Size() - } - - items := s.items[0:n] - s.items = s.items[n:len(s.items)] - - return items -} - -func (s *jobsQueue) IsEmpty() bool { - s.lock.Lock() - defer s.lock.Unlock() - - return len(s.items) == 0 -} - -func (s *jobsQueue) Size() int { - return len(s.items) -} diff --git a/api/mojang/queue/jobs_structure_test.go b/api/mojang/queue/jobs_structure_test.go deleted file mode 100644 index b179d26..0000000 --- a/api/mojang/queue/jobs_structure_test.go +++ /dev/null @@ -1,47 +0,0 @@ -package queue - -import ( - "testing" - - testify "github.com/stretchr/testify/assert" -) - -func TestEnqueue(t *testing.T) { - assert := testify.New(t) - - s := createQueue() - s.Enqueue(&jobItem{Username: "username1"}) - s.Enqueue(&jobItem{Username: "username2"}) - s.Enqueue(&jobItem{Username: "username3"}) - - assert.Equal(3, s.Size()) -} - -func TestDequeueN(t *testing.T) { - assert := testify.New(t) - - s := createQueue() - s.Enqueue(&jobItem{Username: "username1"}) - s.Enqueue(&jobItem{Username: "username2"}) - s.Enqueue(&jobItem{Username: "username3"}) - s.Enqueue(&jobItem{Username: "username4"}) - - items := s.Dequeue(2) - assert.Len(items, 2) - assert.Equal("username1", items[0].Username) - assert.Equal("username2", items[1].Username) - assert.Equal(2, s.Size()) - - items = s.Dequeue(40) - assert.Len(items, 2) - assert.Equal("username3", items[0].Username) - assert.Equal("username4", items[1].Username) - assert.True(s.IsEmpty()) -} - -func createQueue() *jobsQueue { - queue := &jobsQueue{} - queue.New() - - return queue -} diff --git a/api/mojang/queue/queue.go b/api/mojang/queue/queue.go deleted file mode 100644 index 7034d0b..0000000 --- a/api/mojang/queue/queue.go +++ /dev/null @@ -1,221 +0,0 @@ -package queue - -import ( - "net" - "net/url" - "regexp" - "strings" - "sync" - "syscall" - "time" - - "github.com/mono83/slf/wd" - - "github.com/elyby/chrly/api/mojang" -) - -var UuidsQueueIterationDelay = 2*time.Second + 500*time.Millisecond - -var usernamesToUuids = mojang.UsernamesToUuids -var uuidToTextures = mojang.UuidToTextures -var forever = func() bool { - return true -} - -// https://help.mojang.com/customer/portal/articles/928638 -var allowedUsernamesRegex = regexp.MustCompile(`^[\w_]{3,16}$`) - -type JobsQueue struct { - Storage Storage - Logger wd.Watchdog - - onFirstCall sync.Once - queue jobsQueue - broadcast *broadcastMap -} - -func (ctx *JobsQueue) GetTexturesForUsername(username string) chan *mojang.SignedTexturesResponse { - // TODO: convert username to lower case - ctx.onFirstCall.Do(func() { - ctx.queue.New() - ctx.broadcast = newBroadcaster() - ctx.startQueue() - }) - - responseChan := make(chan *mojang.SignedTexturesResponse) - if !allowedUsernamesRegex.MatchString(username) { - ctx.Logger.IncCounter("mojang_textures.invalid_username", 1) - go func() { - responseChan <- nil - close(responseChan) - }() - - return responseChan - } - - ctx.Logger.IncCounter("mojang_textures.request", 1) - - uuid, err := ctx.Storage.GetUuid(username) - if err == nil && uuid == "" { - ctx.Logger.IncCounter("mojang_textures.usernames.cache_hit_nil", 1) - - go func() { - responseChan <- nil - close(responseChan) - }() - - return responseChan - } - - isFirstListener := ctx.broadcast.AddListener(username, responseChan) - if isFirstListener { - start := time.Now() - // TODO: respond nil if processing takes more than 5 seconds - - resultChan := make(chan *mojang.SignedTexturesResponse) - if uuid == "" { - ctx.Logger.IncCounter("mojang_textures.usernames.queued", 1) - ctx.queue.Enqueue(&jobItem{username, resultChan}) - } else { - ctx.Logger.IncCounter("mojang_textures.usernames.cache_hit", 1) - go func() { - resultChan <- ctx.getTextures(uuid) - }() - } - - go func() { - result := <-resultChan - close(resultChan) - ctx.broadcast.BroadcastAndRemove(username, result) - ctx.Logger.RecordTimer("mojang_textures.result_time", time.Since(start)) - }() - } else { - ctx.Logger.IncCounter("mojang_textures.already_in_queue", 1) - } - - return responseChan -} - -func (ctx *JobsQueue) startQueue() { - go func() { - time.Sleep(UuidsQueueIterationDelay) - for forever() { - start := time.Now() - ctx.queueRound() - elapsed := time.Since(start) - ctx.Logger.RecordTimer("mojang_textures.usernames.round_time", elapsed) - time.Sleep(UuidsQueueIterationDelay) - } - }() -} - -func (ctx *JobsQueue) queueRound() { - if ctx.queue.IsEmpty() { - return - } - - queueSize := ctx.queue.Size() - jobs := ctx.queue.Dequeue(10) - ctx.Logger.UpdateGauge("mojang_textures.usernames.iteration_size", int64(len(jobs))) - ctx.Logger.UpdateGauge("mojang_textures.usernames.queue_size", int64(queueSize-len(jobs))) - var usernames []string - for _, job := range jobs { - usernames = append(usernames, job.Username) - } - - profiles, err := usernamesToUuids(usernames) - if err != nil { - ctx.handleResponseError(err, "usernames") - for _, job := range jobs { - job.RespondTo <- nil - } - - return - } - - for _, job := range jobs { - go func(job *jobItem) { - var uuid string - // The profiles in the response are not ordered, so we must search each username over full array - for _, profile := range profiles { - if strings.EqualFold(job.Username, profile.Name) { - uuid = profile.Id - break - } - } - - _ = ctx.Storage.StoreUuid(job.Username, uuid) - if uuid == "" { - job.RespondTo <- nil - ctx.Logger.IncCounter("mojang_textures.usernames.uuid_miss", 1) - } else { - job.RespondTo <- ctx.getTextures(uuid) - ctx.Logger.IncCounter("mojang_textures.usernames.uuid_hit", 1) - } - }(job) - } -} - -func (ctx *JobsQueue) getTextures(uuid string) *mojang.SignedTexturesResponse { - existsTextures, err := ctx.Storage.GetTextures(uuid) - if err == nil { - ctx.Logger.IncCounter("mojang_textures.textures.cache_hit", 1) - return existsTextures - } - - ctx.Logger.IncCounter("mojang_textures.textures.request", 1) - - start := time.Now() - result, err := uuidToTextures(uuid, true) - ctx.Logger.RecordTimer("mojang_textures.textures.request_time", time.Since(start)) - if err != nil { - ctx.handleResponseError(err, "textures") - } - - // Mojang can respond with an error, but count it as a hit, so store result even if the textures is nil - ctx.Storage.StoreTextures(uuid, result) - - return result -} - -func (ctx *JobsQueue) handleResponseError(err error, threadName string) { - ctx.Logger.Debug(":name: Got response error :err", wd.NameParam(threadName), wd.ErrParam(err)) - - switch err.(type) { - case mojang.ResponseError: - if _, ok := err.(*mojang.BadRequestError); ok { - ctx.Logger.Warning(":name: Got 400 Bad Request :err", wd.NameParam(threadName), wd.ErrParam(err)) - return - } - - if _, ok := err.(*mojang.ForbiddenError); ok { - ctx.Logger.Warning(":name: Got 403 Forbidden :err", wd.NameParam(threadName), wd.ErrParam(err)) - return - } - - if _, ok := err.(*mojang.TooManyRequestsError); ok { - ctx.Logger.Warning(":name: Got 429 Too Many Requests :err", wd.NameParam(threadName), wd.ErrParam(err)) - return - } - - return - case net.Error: - if err.(net.Error).Timeout() { - return - } - - if _, ok := err.(*url.Error); ok { - return - } - - if opErr, ok := err.(*net.OpError); ok && (opErr.Op == "dial" || opErr.Op == "read") { - return - } - - if err == syscall.ECONNREFUSED { - return - } - } - - ctx.Logger.Emergency(":name: Unknown Mojang response error: :err", wd.NameParam(threadName), wd.ErrParam(err)) -} diff --git a/api/mojang/queue/queue_test.go b/api/mojang/queue/queue_test.go deleted file mode 100644 index 30bc3ee..0000000 --- a/api/mojang/queue/queue_test.go +++ /dev/null @@ -1,525 +0,0 @@ -package queue - -import ( - "crypto/rand" - "encoding/base64" - "errors" - "net" - "net/url" - "strings" - "syscall" - "time" - - "github.com/elyby/chrly/api/mojang" - - "testing" - - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/suite" - - mocks "github.com/elyby/chrly/tests" -) - -type mojangApiMocks struct { - mock.Mock -} - -func (o *mojangApiMocks) UsernamesToUuids(usernames []string) ([]*mojang.ProfileInfo, error) { - args := o.Called(usernames) - var result []*mojang.ProfileInfo - if casted, ok := args.Get(0).([]*mojang.ProfileInfo); ok { - result = casted - } - - return result, args.Error(1) -} - -func (o *mojangApiMocks) UuidToTextures(uuid string, signed bool) (*mojang.SignedTexturesResponse, error) { - args := o.Called(uuid, signed) - var result *mojang.SignedTexturesResponse - if casted, ok := args.Get(0).(*mojang.SignedTexturesResponse); ok { - result = casted - } - - return result, args.Error(1) -} - -type mockStorage struct { - mock.Mock -} - -func (m *mockStorage) GetUuid(username string) (string, error) { - args := m.Called(username) - return args.String(0), args.Error(1) -} - -func (m *mockStorage) StoreUuid(username string, uuid string) error { - args := m.Called(username, uuid) - return args.Error(0) -} - -func (m *mockStorage) GetTextures(uuid string) (*mojang.SignedTexturesResponse, error) { - args := m.Called(uuid) - var result *mojang.SignedTexturesResponse - if casted, ok := args.Get(0).(*mojang.SignedTexturesResponse); ok { - result = casted - } - - return result, args.Error(1) -} - -func (m *mockStorage) StoreTextures(uuid string, textures *mojang.SignedTexturesResponse) { - m.Called(uuid, textures) -} - -type queueTestSuite struct { - suite.Suite - Queue *JobsQueue - Storage *mockStorage - MojangApi *mojangApiMocks - Logger *mocks.WdMock - Iterate func() - - iterateChan chan bool - done func() -} - -func (suite *queueTestSuite) SetupSuite() { - UuidsQueueIterationDelay = 0 -} - -func (suite *queueTestSuite) SetupTest() { - suite.Storage = &mockStorage{} - suite.Logger = &mocks.WdMock{} - - suite.Queue = &JobsQueue{Storage: suite.Storage, Logger: suite.Logger} - - suite.iterateChan = make(chan bool) - forever = func() bool { - return <-suite.iterateChan - } - - suite.Iterate = func() { - suite.iterateChan <- true - } - - suite.done = func() { - suite.iterateChan <- false - } - - suite.MojangApi = new(mojangApiMocks) - usernamesToUuids = suite.MojangApi.UsernamesToUuids - uuidToTextures = suite.MojangApi.UuidToTextures -} - -func (suite *queueTestSuite) TearDownTest() { - suite.done() - time.Sleep(10 * time.Millisecond) // Add delay to let finish all goroutines before assert mocks calls - suite.MojangApi.AssertExpectations(suite.T()) - suite.Storage.AssertExpectations(suite.T()) - suite.Logger.AssertExpectations(suite.T()) -} - -func (suite *queueTestSuite) TestReceiveTexturesForOneUsernameWithoutAnyCache() { - expectedResult := &mojang.SignedTexturesResponse{Id: "0d252b7218b648bfb86c2ae476954d32", Name: "maksimkurb"} - - suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Once() - suite.Logger.On("IncCounter", "mojang_textures.usernames.queued", int64(1)).Once() - suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(1)).Once() - suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(0)).Once() - suite.Logger.On("RecordTimer", "mojang_textures.usernames.round_time", mock.Anything) - suite.Logger.On("IncCounter", "mojang_textures.textures.request", int64(1)).Once() - suite.Logger.On("RecordTimer", "mojang_textures.textures.request_time", mock.Anything).Once() - suite.Logger.On("IncCounter", "mojang_textures.usernames.uuid_hit", int64(1)).Once() - suite.Logger.On("RecordTimer", "mojang_textures.result_time", mock.Anything).Once() - - suite.Storage.On("GetUuid", "maksimkurb").Once().Return("", &ValueNotFound{}) - suite.Storage.On("StoreUuid", "maksimkurb", "0d252b7218b648bfb86c2ae476954d32").Once().Return(nil) - suite.Storage.On("GetTextures", "0d252b7218b648bfb86c2ae476954d32").Once().Return(nil, &ValueNotFound{}) - suite.Storage.On("StoreTextures", "0d252b7218b648bfb86c2ae476954d32", expectedResult).Once() - - suite.MojangApi.On("UsernamesToUuids", []string{"maksimkurb"}).Once().Return([]*mojang.ProfileInfo{ - {Id: "0d252b7218b648bfb86c2ae476954d32", Name: "maksimkurb"}, - }, nil) - suite.MojangApi.On("UuidToTextures", "0d252b7218b648bfb86c2ae476954d32", true).Once().Return(expectedResult, nil) - - resultChan := suite.Queue.GetTexturesForUsername("maksimkurb") - - suite.Iterate() - - result := <-resultChan - suite.Assert().Equal(expectedResult, result) -} - -func (suite *queueTestSuite) TestReceiveTexturesForFewUsernamesWithoutAnyCache() { - expectedResult1 := &mojang.SignedTexturesResponse{Id: "0d252b7218b648bfb86c2ae476954d32", Name: "maksimkurb"} - expectedResult2 := &mojang.SignedTexturesResponse{Id: "4566e69fc90748ee8d71d7ba5aa00d20", Name: "Thinkofdeath"} - - suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Twice() - suite.Logger.On("IncCounter", "mojang_textures.usernames.queued", int64(1)).Twice() - suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(2)).Once() - suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(0)).Once() - suite.Logger.On("RecordTimer", "mojang_textures.usernames.round_time", mock.Anything) - suite.Logger.On("IncCounter", "mojang_textures.textures.request", int64(1)).Twice() - suite.Logger.On("RecordTimer", "mojang_textures.textures.request_time", mock.Anything).Twice() - suite.Logger.On("IncCounter", "mojang_textures.usernames.uuid_hit", int64(1)).Twice() - suite.Logger.On("RecordTimer", "mojang_textures.result_time", mock.Anything).Twice() - - suite.Storage.On("GetUuid", "maksimkurb").Once().Return("", &ValueNotFound{}) - suite.Storage.On("GetUuid", "Thinkofdeath").Once().Return("", &ValueNotFound{}) - suite.Storage.On("StoreUuid", "maksimkurb", "0d252b7218b648bfb86c2ae476954d32").Once().Return(nil) - suite.Storage.On("StoreUuid", "Thinkofdeath", "4566e69fc90748ee8d71d7ba5aa00d20").Once().Return(nil) - suite.Storage.On("GetTextures", "0d252b7218b648bfb86c2ae476954d32").Once().Return(nil, &ValueNotFound{}) - suite.Storage.On("GetTextures", "4566e69fc90748ee8d71d7ba5aa00d20").Once().Return(nil, &ValueNotFound{}) - suite.Storage.On("StoreTextures", "0d252b7218b648bfb86c2ae476954d32", expectedResult1).Once() - suite.Storage.On("StoreTextures", "4566e69fc90748ee8d71d7ba5aa00d20", expectedResult2).Once() - - suite.MojangApi.On("UsernamesToUuids", []string{"maksimkurb", "Thinkofdeath"}).Once().Return([]*mojang.ProfileInfo{ - {Id: "0d252b7218b648bfb86c2ae476954d32", Name: "maksimkurb"}, - {Id: "4566e69fc90748ee8d71d7ba5aa00d20", Name: "Thinkofdeath"}, - }, nil) - suite.MojangApi.On("UuidToTextures", "0d252b7218b648bfb86c2ae476954d32", true).Once().Return(expectedResult1, nil) - suite.MojangApi.On("UuidToTextures", "4566e69fc90748ee8d71d7ba5aa00d20", true).Once().Return(expectedResult2, nil) - - resultChan1 := suite.Queue.GetTexturesForUsername("maksimkurb") - resultChan2 := suite.Queue.GetTexturesForUsername("Thinkofdeath") - - suite.Iterate() - - suite.Assert().Equal(expectedResult1, <-resultChan1) - suite.Assert().Equal(expectedResult2, <-resultChan2) -} - -func (suite *queueTestSuite) TestReceiveTexturesForUsernameWithCachedUuid() { - expectedResult := &mojang.SignedTexturesResponse{Id: "0d252b7218b648bfb86c2ae476954d32", Name: "maksimkurb"} - - suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Once() - suite.Logger.On("IncCounter", "mojang_textures.usernames.cache_hit", int64(1)).Once() - suite.Logger.On("IncCounter", "mojang_textures.textures.request", int64(1)).Once() - suite.Logger.On("RecordTimer", "mojang_textures.textures.request_time", mock.Anything).Once() - suite.Logger.On("RecordTimer", "mojang_textures.result_time", mock.Anything).Once() - - suite.Storage.On("GetUuid", "maksimkurb").Once().Return("0d252b7218b648bfb86c2ae476954d32", nil) - // Storage.StoreUuid shouldn't be called - suite.Storage.On("GetTextures", "0d252b7218b648bfb86c2ae476954d32").Once().Return(nil, &ValueNotFound{}) - suite.Storage.On("StoreTextures", "0d252b7218b648bfb86c2ae476954d32", expectedResult).Once() - - // MojangApi.UsernamesToUuids shouldn't be called - suite.MojangApi.On("UuidToTextures", "0d252b7218b648bfb86c2ae476954d32", true).Once().Return(expectedResult, nil) - - resultChan := suite.Queue.GetTexturesForUsername("maksimkurb") - - // Note that there is no iteration - - result := <-resultChan - suite.Assert().Equal(expectedResult, result) -} - -func (suite *queueTestSuite) TestReceiveTexturesForUsernameWithFullyCachedResult() { - expectedResult := &mojang.SignedTexturesResponse{Id: "0d252b7218b648bfb86c2ae476954d32", Name: "maksimkurb"} - - suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Once() - suite.Logger.On("IncCounter", "mojang_textures.usernames.cache_hit", int64(1)).Once() - suite.Logger.On("IncCounter", "mojang_textures.textures.cache_hit", int64(1)).Once() - suite.Logger.On("RecordTimer", "mojang_textures.result_time", mock.Anything).Once() - - suite.Storage.On("GetUuid", "maksimkurb").Once().Return("0d252b7218b648bfb86c2ae476954d32", nil) - // Storage.StoreUuid shouldn't be called - suite.Storage.On("GetTextures", "0d252b7218b648bfb86c2ae476954d32").Once().Return(expectedResult, nil) - // Storage.StoreTextures shouldn't be called - - // MojangApi.UsernamesToUuids shouldn't be called - // MojangApi.UuidToTextures shouldn't be called - - resultChan := suite.Queue.GetTexturesForUsername("maksimkurb") - - // Note that there is no iteration - - result := <-resultChan - suite.Assert().Equal(expectedResult, result) -} - -func (suite *queueTestSuite) TestReceiveTexturesForUsernameWithCachedUnknownUuid() { - suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Once() - suite.Logger.On("IncCounter", "mojang_textures.usernames.cache_hit_nil", int64(1)).Once() - - suite.Storage.On("GetUuid", "maksimkurb").Once().Return("", nil) - // Storage.StoreUuid shouldn't be called - // Storage.GetTextures shouldn't be called - // Storage.StoreTextures shouldn't be called - - // MojangApi.UsernamesToUuids shouldn't be called - // MojangApi.UuidToTextures shouldn't be called - - resultChan := suite.Queue.GetTexturesForUsername("maksimkurb") - - // Note that there is no iteration - - suite.Assert().Nil(<-resultChan) -} - -func (suite *queueTestSuite) TestReceiveTexturesForMoreThan10Usernames() { - usernames := make([]string, 12) - for i := 0; i < cap(usernames); i++ { - usernames[i] = randStr(8) - } - - suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Times(12) - suite.Logger.On("IncCounter", "mojang_textures.usernames.queued", int64(1)).Times(12) - suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(10)).Once() - suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(2)).Once() - suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(2)).Once() - suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(0)).Once() - suite.Logger.On("RecordTimer", "mojang_textures.usernames.round_time", mock.Anything).Twice() - suite.Logger.On("IncCounter", "mojang_textures.usernames.uuid_miss", int64(1)).Times(12) - suite.Logger.On("RecordTimer", "mojang_textures.result_time", mock.Anything).Times(12) - - suite.Storage.On("GetUuid", mock.Anything).Times(12).Return("", &ValueNotFound{}) - suite.Storage.On("StoreUuid", mock.Anything, "").Times(12).Return(nil) // should be called with "" if username is not compared to uuid - // Storage.GetTextures and Storage.SetTextures shouldn't be called - - suite.MojangApi.On("UsernamesToUuids", usernames[0:10]).Once().Return([]*mojang.ProfileInfo{}, nil) - suite.MojangApi.On("UsernamesToUuids", usernames[10:12]).Once().Return([]*mojang.ProfileInfo{}, nil) - - channels := make([]chan *mojang.SignedTexturesResponse, 12) - for i, username := range usernames { - channels[i] = suite.Queue.GetTexturesForUsername(username) - } - - suite.Iterate() - suite.Iterate() - - for _, channel := range channels { - <-channel - } -} - -func (suite *queueTestSuite) TestReceiveTexturesForTheSameUsernames() { - expectedResult := &mojang.SignedTexturesResponse{Id: "0d252b7218b648bfb86c2ae476954d32", Name: "maksimkurb"} - - suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Twice() - suite.Logger.On("IncCounter", "mojang_textures.usernames.queued", int64(1)).Once() - suite.Logger.On("IncCounter", "mojang_textures.already_in_queue", int64(1)).Once() - suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(1)).Once() - suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(0)).Once() - suite.Logger.On("RecordTimer", "mojang_textures.usernames.round_time", mock.Anything) - suite.Logger.On("IncCounter", "mojang_textures.textures.request", int64(1)).Once() - suite.Logger.On("RecordTimer", "mojang_textures.textures.request_time", mock.Anything).Once() - suite.Logger.On("IncCounter", "mojang_textures.usernames.uuid_hit", int64(1)).Once() - suite.Logger.On("RecordTimer", "mojang_textures.result_time", mock.Anything).Once() - - suite.Storage.On("GetUuid", "maksimkurb").Twice().Return("", &ValueNotFound{}) - suite.Storage.On("StoreUuid", "maksimkurb", "0d252b7218b648bfb86c2ae476954d32").Once().Return(nil) - suite.Storage.On("GetTextures", "0d252b7218b648bfb86c2ae476954d32").Once().Return(nil, &ValueNotFound{}) - suite.Storage.On("StoreTextures", "0d252b7218b648bfb86c2ae476954d32", expectedResult).Once() - suite.MojangApi.On("UsernamesToUuids", []string{"maksimkurb"}).Once().Return([]*mojang.ProfileInfo{ - {Id: "0d252b7218b648bfb86c2ae476954d32", Name: "maksimkurb"}, - }, nil) - suite.MojangApi.On("UuidToTextures", "0d252b7218b648bfb86c2ae476954d32", true).Once().Return(expectedResult, nil) - - resultChan1 := suite.Queue.GetTexturesForUsername("maksimkurb") - resultChan2 := suite.Queue.GetTexturesForUsername("maksimkurb") - - suite.Iterate() - - suite.Assert().Equal(expectedResult, <-resultChan1) - suite.Assert().Equal(expectedResult, <-resultChan2) -} - -func (suite *queueTestSuite) TestReceiveTexturesForUsernameThatAlreadyProcessing() { - expectedResult := &mojang.SignedTexturesResponse{Id: "0d252b7218b648bfb86c2ae476954d32", Name: "maksimkurb"} - - suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Twice() - suite.Logger.On("IncCounter", "mojang_textures.usernames.queued", int64(1)).Once() - suite.Logger.On("IncCounter", "mojang_textures.already_in_queue", int64(1)).Once() - suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(1)).Once() - suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(0)).Once() - suite.Logger.On("RecordTimer", "mojang_textures.usernames.round_time", mock.Anything) - suite.Logger.On("IncCounter", "mojang_textures.textures.request", int64(1)).Once() - suite.Logger.On("RecordTimer", "mojang_textures.textures.request_time", mock.Anything).Once() - suite.Logger.On("IncCounter", "mojang_textures.usernames.uuid_hit", int64(1)).Once() - suite.Logger.On("RecordTimer", "mojang_textures.result_time", mock.Anything).Once() - - suite.Storage.On("GetUuid", "maksimkurb").Twice().Return("", &ValueNotFound{}) - suite.Storage.On("StoreUuid", "maksimkurb", "0d252b7218b648bfb86c2ae476954d32").Once().Return(nil) - suite.Storage.On("GetTextures", "0d252b7218b648bfb86c2ae476954d32").Once().Return(nil, &ValueNotFound{}) - suite.Storage.On("StoreTextures", "0d252b7218b648bfb86c2ae476954d32", expectedResult).Once() - - suite.MojangApi.On("UsernamesToUuids", []string{"maksimkurb"}).Once().Return([]*mojang.ProfileInfo{ - {Id: "0d252b7218b648bfb86c2ae476954d32", Name: "maksimkurb"}, - }, nil) - suite.MojangApi.On("UuidToTextures", "0d252b7218b648bfb86c2ae476954d32", true). - Once(). - After(10*time.Millisecond). // Simulate long round trip - Return(expectedResult, nil) - - resultChan1 := suite.Queue.GetTexturesForUsername("maksimkurb") - - // Note that for entire test there is only one iteration - suite.Iterate() - - // Let it meet delayed UuidToTextures request - time.Sleep(5 * time.Millisecond) - - resultChan2 := suite.Queue.GetTexturesForUsername("maksimkurb") - - suite.Assert().Equal(expectedResult, <-resultChan1) - suite.Assert().Equal(expectedResult, <-resultChan2) -} - -func (suite *queueTestSuite) TestDoNothingWhenNoTasks() { - suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Once() - suite.Logger.On("IncCounter", "mojang_textures.usernames.queued", int64(1)).Once() - suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(1)).Once() - suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(0)).Once() - suite.Logger.On("RecordTimer", "mojang_textures.usernames.round_time", mock.Anything) - suite.Logger.On("IncCounter", "mojang_textures.usernames.uuid_miss", int64(1)).Once() - suite.Logger.On("RecordTimer", "mojang_textures.result_time", mock.Anything).Once() - - suite.Storage.On("GetUuid", "maksimkurb").Once().Return("", &ValueNotFound{}) - suite.Storage.On("StoreUuid", "maksimkurb", "").Once().Return(nil) - // Storage.GetTextures and Storage.StoreTextures shouldn't be called - - suite.MojangApi.On("UsernamesToUuids", []string{"maksimkurb"}).Once().Return([]*mojang.ProfileInfo{}, nil) - - // Perform first iteration and await it finish - resultChan := suite.Queue.GetTexturesForUsername("maksimkurb") - - suite.Iterate() - - suite.Assert().Nil(<-resultChan) - - // Let it to perform a few more iterations to ensure, that there is no calls to external APIs - suite.Iterate() - suite.Iterate() -} - -type timeoutError struct { -} - -func (*timeoutError) Error() string { return "timeout error" } -func (*timeoutError) Timeout() bool { return true } -func (*timeoutError) Temporary() bool { return false } - -var expectedErrors = []error{ - &mojang.BadRequestError{}, - &mojang.ForbiddenError{}, - &mojang.TooManyRequestsError{}, - &mojang.ServerError{}, - &timeoutError{}, - &url.Error{Op: "GET", URL: "http://localhost"}, - &net.OpError{Op: "read"}, - &net.OpError{Op: "dial"}, - syscall.ECONNREFUSED, -} - -func (suite *queueTestSuite) TestShouldNotLogErrorWhenExpectedErrorReturnedFromUsernameToUuidRequest() { - suite.Logger.On("IncCounter", mock.Anything, mock.Anything) - suite.Logger.On("UpdateGauge", mock.Anything, mock.Anything) - suite.Logger.On("RecordTimer", mock.Anything, mock.Anything) - suite.Logger.On("Debug", ":name: Got response error :err", mock.Anything, mock.Anything).Times(len(expectedErrors)) - suite.Logger.On("Warning", ":name: Got 400 Bad Request :err", mock.Anything, mock.Anything).Once() - suite.Logger.On("Warning", ":name: Got 403 Forbidden :err", mock.Anything, mock.Anything).Once() - suite.Logger.On("Warning", ":name: Got 429 Too Many Requests :err", mock.Anything, mock.Anything).Once() - - suite.Storage.On("GetUuid", "maksimkurb").Return("", &ValueNotFound{}) - - for _, err := range expectedErrors { - suite.MojangApi.On("UsernamesToUuids", []string{"maksimkurb"}).Once().Return(nil, err) - - resultChan := suite.Queue.GetTexturesForUsername("maksimkurb") - suite.Iterate() - suite.Assert().Nil(<-resultChan) - suite.MojangApi.AssertExpectations(suite.T()) - suite.MojangApi.ExpectedCalls = nil // https://github.com/stretchr/testify/issues/558#issuecomment-372112364 - } -} - -func (suite *queueTestSuite) TestShouldLogEmergencyOnUnexpectedErrorReturnedFromUsernameToUuidRequest() { - suite.Logger.On("IncCounter", mock.Anything, mock.Anything) - suite.Logger.On("UpdateGauge", mock.Anything, mock.Anything) - suite.Logger.On("RecordTimer", mock.Anything, mock.Anything) - suite.Logger.On("Debug", ":name: Got response error :err", mock.Anything, mock.Anything).Once() - suite.Logger.On("Emergency", ":name: Unknown Mojang response error: :err", mock.Anything, mock.Anything).Once() - - suite.Storage.On("GetUuid", "maksimkurb").Return("", &ValueNotFound{}) - - suite.MojangApi.On("UsernamesToUuids", []string{"maksimkurb"}).Once().Return(nil, errors.New("unexpected error")) - - resultChan := suite.Queue.GetTexturesForUsername("maksimkurb") - suite.Iterate() - suite.Assert().Nil(<-resultChan) -} - -func (suite *queueTestSuite) TestShouldNotLogErrorWhenExpectedErrorReturnedFromUuidToTexturesRequest() { - suite.Logger.On("IncCounter", mock.Anything, mock.Anything) - suite.Logger.On("UpdateGauge", mock.Anything, mock.Anything) - suite.Logger.On("RecordTimer", mock.Anything, mock.Anything) - suite.Logger.On("Debug", ":name: Got response error :err", mock.Anything, mock.Anything).Times(len(expectedErrors)) - suite.Logger.On("Warning", ":name: Got 400 Bad Request :err", mock.Anything, mock.Anything).Once() - suite.Logger.On("Warning", ":name: Got 403 Forbidden :err", mock.Anything, mock.Anything).Once() - suite.Logger.On("Warning", ":name: Got 429 Too Many Requests :err", mock.Anything, mock.Anything).Once() - - suite.Storage.On("GetUuid", "maksimkurb").Return("", &ValueNotFound{}) - suite.Storage.On("StoreUuid", "maksimkurb", "0d252b7218b648bfb86c2ae476954d32").Return(nil) - suite.Storage.On("GetTextures", "0d252b7218b648bfb86c2ae476954d32").Return(nil, &ValueNotFound{}) - suite.Storage.On("StoreTextures", "0d252b7218b648bfb86c2ae476954d32", (*mojang.SignedTexturesResponse)(nil)) - - for _, err := range expectedErrors { - suite.MojangApi.On("UsernamesToUuids", []string{"maksimkurb"}).Once().Return([]*mojang.ProfileInfo{ - {Id: "0d252b7218b648bfb86c2ae476954d32", Name: "maksimkurb"}, - }, nil) - suite.MojangApi.On("UuidToTextures", "0d252b7218b648bfb86c2ae476954d32", true).Once().Return(nil, err) - - resultChan := suite.Queue.GetTexturesForUsername("maksimkurb") - suite.Iterate() - suite.Assert().Nil(<-resultChan) - suite.MojangApi.AssertExpectations(suite.T()) - suite.MojangApi.ExpectedCalls = nil // https://github.com/stretchr/testify/issues/558#issuecomment-372112364 - } -} - -func (suite *queueTestSuite) TestShouldLogEmergencyOnUnexpectedErrorReturnedFromUuidToTexturesRequest() { - suite.Logger.On("IncCounter", mock.Anything, mock.Anything) - suite.Logger.On("UpdateGauge", mock.Anything, mock.Anything) - suite.Logger.On("RecordTimer", mock.Anything, mock.Anything) - suite.Logger.On("Debug", ":name: Got response error :err", mock.Anything, mock.Anything).Once() - suite.Logger.On("Emergency", ":name: Unknown Mojang response error: :err", mock.Anything, mock.Anything).Once() - - suite.Storage.On("GetUuid", "maksimkurb").Return("", &ValueNotFound{}) - suite.Storage.On("StoreUuid", "maksimkurb", "0d252b7218b648bfb86c2ae476954d32").Return(nil) - suite.Storage.On("GetTextures", "0d252b7218b648bfb86c2ae476954d32").Return(nil, &ValueNotFound{}) - suite.Storage.On("StoreTextures", "0d252b7218b648bfb86c2ae476954d32", (*mojang.SignedTexturesResponse)(nil)) - - suite.MojangApi.On("UsernamesToUuids", []string{"maksimkurb"}).Once().Return([]*mojang.ProfileInfo{ - {Id: "0d252b7218b648bfb86c2ae476954d32", Name: "maksimkurb"}, - }, nil) - suite.MojangApi.On("UuidToTextures", "0d252b7218b648bfb86c2ae476954d32", true).Once().Return(nil, errors.New("unexpected error")) - - resultChan := suite.Queue.GetTexturesForUsername("maksimkurb") - suite.Iterate() - suite.Assert().Nil(<-resultChan) -} - -func (suite *queueTestSuite) TestReceiveTexturesForNotAllowedMojangUsername() { - suite.Logger.On("IncCounter", "mojang_textures.invalid_username", int64(1)).Once() - - resultChan := suite.Queue.GetTexturesForUsername("Not allowed") - suite.Assert().Nil(<-resultChan) -} - -func TestJobsQueueSuite(t *testing.T) { - suite.Run(t, new(queueTestSuite)) -} - -var replacer = strings.NewReplacer("-", "_", "=", "") - -// https://stackoverflow.com/a/50581165 -func randStr(len int) string { - buff := make([]byte, len) - _, _ = rand.Read(buff) - str := replacer.Replace(base64.URLEncoding.EncodeToString(buff)) - - // Base 64 can be longer than len - return str[:len] -} diff --git a/api/mojang/queue/storage.go b/api/mojang/queue/storage.go deleted file mode 100644 index 2629e58..0000000 --- a/api/mojang/queue/storage.go +++ /dev/null @@ -1,53 +0,0 @@ -package queue - -import "github.com/elyby/chrly/api/mojang" - -type UuidsStorage interface { - GetUuid(username string) (string, error) - StoreUuid(username string, uuid string) error -} - -// nil value can be passed to the storage to indicate that there is no textures -// for uuid and we know about it. Return err only in case, when storage completely -// unable to load any information about textures -type TexturesStorage interface { - GetTextures(uuid string) (*mojang.SignedTexturesResponse, error) - StoreTextures(uuid string, textures *mojang.SignedTexturesResponse) -} - -type Storage interface { - UuidsStorage - TexturesStorage -} - -// SplittedStorage allows you to use separate storage engines to satisfy -// the Storage interface -type SplittedStorage struct { - UuidsStorage - TexturesStorage -} - -func (s *SplittedStorage) GetUuid(username string) (string, error) { - return s.UuidsStorage.GetUuid(username) -} - -func (s *SplittedStorage) StoreUuid(username string, uuid string) error { - return s.UuidsStorage.StoreUuid(username, uuid) -} - -func (s *SplittedStorage) GetTextures(uuid string) (*mojang.SignedTexturesResponse, error) { - return s.TexturesStorage.GetTextures(uuid) -} - -func (s *SplittedStorage) StoreTextures(uuid string, textures *mojang.SignedTexturesResponse) { - s.TexturesStorage.StoreTextures(uuid, textures) -} - -// This error can be used to indicate, that requested -// value doesn't exists in the storage -type ValueNotFound struct { -} - -func (*ValueNotFound) Error() string { - return "value not found in the storage" -} diff --git a/cmd/serve.go b/cmd/serve.go index 31d137e..2ca852d 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -8,11 +8,11 @@ import ( "github.com/spf13/cobra" "github.com/spf13/viper" - "github.com/elyby/chrly/api/mojang/queue" "github.com/elyby/chrly/auth" "github.com/elyby/chrly/bootstrap" "github.com/elyby/chrly/db" "github.com/elyby/chrly/http" + "github.com/elyby/chrly/mojangtextures" ) var serveCmd = &cobra.Command{ @@ -52,12 +52,19 @@ var serveCmd = &cobra.Command{ return } - queue.UuidsQueueIterationDelay = time.Duration(viper.GetInt("queue.loop_delay")) * time.Millisecond - texturesStorage := queue.CreateInMemoryTexturesStorage() + texturesStorage := mojangtextures.NewInMemoryTexturesStorage() texturesStorage.Start() - mojangTexturesQueue := &queue.JobsQueue{ + mojangTexturesProvider := &mojangtextures.Provider{ Logger: logger, - Storage: &queue.SplittedStorage{ + UuidsProvider: &mojangtextures.BatchUuidsProvider{ + IterationDelay: time.Duration(viper.GetInt("queue.loop_delay")) * time.Millisecond, + IterationSize: viper.GetInt("queue.batch_size"), + Logger: logger, + }, + TexturesProvider: &mojangtextures.MojangApiTexturesProvider{ + Logger: logger, + }, + Storage: &mojangtextures.SeparatedStorage{ UuidsStorage: mojangUuidsRepository, TexturesStorage: texturesStorage, }, @@ -65,12 +72,12 @@ var serveCmd = &cobra.Command{ logger.Info("Mojang's textures queue is successfully initialized") cfg := &http.Config{ - ListenSpec: fmt.Sprintf("%s:%d", viper.GetString("server.host"), viper.GetInt("server.port")), - SkinsRepo: skinsRepo, - CapesRepo: capesRepo, - MojangTexturesQueue: mojangTexturesQueue, - Logger: logger, - Auth: &auth.JwtAuth{Key: []byte(viper.GetString("chrly.secret"))}, + ListenSpec: fmt.Sprintf("%s:%d", viper.GetString("server.host"), viper.GetInt("server.port")), + SkinsRepo: skinsRepo, + CapesRepo: capesRepo, + MojangTexturesProvider: mojangTexturesProvider, + Logger: logger, + Auth: &auth.JwtAuth{Key: []byte(viper.GetString("chrly.secret"))}, } if err := cfg.Run(); err != nil { @@ -89,4 +96,5 @@ func init() { viper.SetDefault("storage.filesystem.basePath", "data") viper.SetDefault("storage.filesystem.capesDirName", "capes") viper.SetDefault("queue.loop_delay", 2_500) + viper.SetDefault("queue.batch_size", 10) } diff --git a/db/factory.go b/db/factory.go index 2ba75a7..337d8bf 100644 --- a/db/factory.go +++ b/db/factory.go @@ -3,8 +3,8 @@ package db import ( "github.com/spf13/viper" - "github.com/elyby/chrly/api/mojang/queue" "github.com/elyby/chrly/interfaces" + "github.com/elyby/chrly/mojangtextures" ) type StorageFactory struct { @@ -14,7 +14,7 @@ type StorageFactory struct { type RepositoriesCreator interface { CreateSkinsRepository() (interfaces.SkinsRepository, error) CreateCapesRepository() (interfaces.CapesRepository, error) - CreateMojangUuidsRepository() (queue.UuidsStorage, error) + CreateMojangUuidsRepository() (mojangtextures.UuidsStorage, error) } func (factory *StorageFactory) CreateFactory(backend string) RepositoriesCreator { diff --git a/db/filesystem.go b/db/filesystem.go index 4674652..4a996c3 100644 --- a/db/filesystem.go +++ b/db/filesystem.go @@ -5,9 +5,9 @@ import ( "path" "strings" - "github.com/elyby/chrly/api/mojang/queue" "github.com/elyby/chrly/interfaces" "github.com/elyby/chrly/model" + "github.com/elyby/chrly/mojangtextures" ) type FilesystemFactory struct { @@ -27,7 +27,7 @@ func (f FilesystemFactory) CreateCapesRepository() (interfaces.CapesRepository, return &filesStorage{path: path.Join(f.BasePath, f.CapesDirName)}, nil } -func (f FilesystemFactory) CreateMojangUuidsRepository() (queue.UuidsStorage, error) { +func (f FilesystemFactory) CreateMojangUuidsRepository() (mojangtextures.UuidsStorage, error) { panic("implement me") } diff --git a/db/redis.go b/db/redis.go index df07135..cb657b0 100644 --- a/db/redis.go +++ b/db/redis.go @@ -14,9 +14,9 @@ import ( "github.com/mediocregopher/radix.v2/redis" "github.com/mediocregopher/radix.v2/util" - "github.com/elyby/chrly/api/mojang/queue" "github.com/elyby/chrly/interfaces" "github.com/elyby/chrly/model" + "github.com/elyby/chrly/mojangtextures" ) type RedisFactory struct { @@ -34,7 +34,7 @@ func (f *RedisFactory) CreateCapesRepository() (interfaces.CapesRepository, erro panic("capes repository not supported for this storage type") } -func (f *RedisFactory) CreateMojangUuidsRepository() (queue.UuidsStorage, error) { +func (f *RedisFactory) CreateMojangUuidsRepository() (mojangtextures.UuidsStorage, error) { return f.createInstance() } @@ -255,7 +255,7 @@ func save(skin *model.Skin, conn util.Cmder) error { func findMojangUuidByUsername(username string, conn util.Cmder) (string, error) { response := conn.Cmd("HGET", mojangUsernameToUuidKey, strings.ToLower(username)) if response.IsType(redis.Nil) { - return "", &queue.ValueNotFound{} + return "", &mojangtextures.ValueNotFound{} } data, _ := response.Str() @@ -263,7 +263,7 @@ func findMojangUuidByUsername(username string, conn util.Cmder) (string, error) timestamp, _ := strconv.ParseInt(parts[1], 10, 64) storedAt := time.Unix(timestamp, 0) if storedAt.Add(time.Hour * 24 * 30).Before(time.Now()) { - return "", &queue.ValueNotFound{} + return "", &mojangtextures.ValueNotFound{} } return parts[0], nil diff --git a/http/cape.go b/http/cape.go index 7753fbd..9bba411 100644 --- a/http/cape.go +++ b/http/cape.go @@ -20,8 +20,8 @@ func (cfg *Config) Cape(response http.ResponseWriter, request *http.Request) { return } - mojangTextures := <-cfg.MojangTexturesQueue.GetTexturesForUsername(username) - if mojangTextures == nil { + mojangTextures, err := cfg.MojangTexturesProvider.GetForUsername(username) + if err != nil || mojangTextures == nil { response.WriteHeader(http.StatusNotFound) return } diff --git a/http/cape_test.go b/http/cape_test.go index 5c0de9a..ededa15 100644 --- a/http/cape_test.go +++ b/http/cape_test.go @@ -28,7 +28,7 @@ type capesTestCase struct { var capesTestCases = []*capesTestCase{ { - Name: "Obtain cape for known username", + Name: "Obtain cape for known username", ExistsInLocalStorage: true, AssertResponse: func(assert *testify.Assertions, resp *http.Response) { assert.Equal(200, resp.StatusCode) @@ -38,28 +38,28 @@ var capesTestCases = []*capesTestCase{ }, }, { - Name: "Obtain cape for unknown username that exists in Mojang and has a cape", + Name: "Obtain cape for unknown username that exists in Mojang and has a cape", ExistsInLocalStorage: false, - ExistsInMojang: true, - HasCapeInMojangResp: true, + ExistsInMojang: true, + HasCapeInMojangResp: true, AssertResponse: func(assert *testify.Assertions, resp *http.Response) { assert.Equal(301, resp.StatusCode) assert.Equal("http://mojang/cape.png", resp.Header.Get("Location")) }, }, { - Name: "Obtain cape for unknown username that exists in Mojang, but don't has a cape", + Name: "Obtain cape for unknown username that exists in Mojang, but don't has a cape", ExistsInLocalStorage: false, - ExistsInMojang: true, - HasCapeInMojangResp: false, + ExistsInMojang: true, + HasCapeInMojangResp: false, AssertResponse: func(assert *testify.Assertions, resp *http.Response) { assert.Equal(404, resp.StatusCode) }, }, { - Name: "Obtain cape for unknown username that doesn't exists in Mojang", + Name: "Obtain cape for unknown username that doesn't exists in Mojang", ExistsInLocalStorage: false, - ExistsInMojang: false, + ExistsInMojang: false, AssertResponse: func(assert *testify.Assertions, resp *http.Response) { assert.Equal(404, resp.StatusCode) }, @@ -86,9 +86,9 @@ func TestConfig_Cape(t *testing.T) { if testCase.ExistsInMojang { textures := createTexturesResponse(false, testCase.HasCapeInMojangResp) - mocks.Queue.On("GetTexturesForUsername", "mock_username").Return(textures) + mocks.MojangProvider.On("GetForUsername", "mock_username").Return(textures, nil) } else { - mocks.Queue.On("GetTexturesForUsername", "mock_username").Return(nil) + mocks.MojangProvider.On("GetForUsername", "mock_username").Return(nil, nil) } req := httptest.NewRequest("GET", testCase.RequestUrl, nil) diff --git a/http/http.go b/http/http.go index f04a5d5..bc1edc4 100644 --- a/http/http.go +++ b/http/http.go @@ -19,11 +19,11 @@ import ( type Config struct { ListenSpec string - SkinsRepo interfaces.SkinsRepository - CapesRepo interfaces.CapesRepository - MojangTexturesQueue interfaces.MojangTexturesQueue - Logger wd.Watchdog - Auth interfaces.AuthChecker + SkinsRepo interfaces.SkinsRepository + CapesRepo interfaces.CapesRepository + MojangTexturesProvider interfaces.MojangTexturesProvider + Logger wd.Watchdog + Auth interfaces.AuthChecker } func (cfg *Config) Run() error { diff --git a/http/http_test.go b/http/http_test.go index 66b3472..ec74497 100644 --- a/http/http_test.go +++ b/http/http_test.go @@ -4,9 +4,10 @@ import ( "testing" "time" + "github.com/stretchr/testify/mock" + "github.com/elyby/chrly/api/mojang" - "github.com/elyby/chrly/tests" "github.com/golang/mock/gomock" testify "github.com/stretchr/testify/assert" @@ -20,46 +21,57 @@ func TestParseUsername(t *testing.T) { assert.Equal("test", parseUsername("test"), "Function should return string itself, if it not contains .png at end") } -type mocks struct { - Skins *mock_interfaces.MockSkinsRepository - Capes *mock_interfaces.MockCapesRepository - Queue *tests.MojangTexturesQueueMock - Auth *mock_interfaces.MockAuthChecker - Log *mock_wd.MockWatchdog +type mojangTexturesProviderMock struct { + mock.Mock } -func setupMocks(ctrl *gomock.Controller) ( - *Config, - *mocks, -) { +func (m *mojangTexturesProviderMock) GetForUsername(username string) (*mojang.SignedTexturesResponse, error) { + args := m.Called(username) + var result *mojang.SignedTexturesResponse + if casted, ok := args.Get(0).(*mojang.SignedTexturesResponse); ok { + result = casted + } + + return result, args.Error(1) +} + +type mocks struct { + Skins *mock_interfaces.MockSkinsRepository + Capes *mock_interfaces.MockCapesRepository + MojangProvider *mojangTexturesProviderMock + Auth *mock_interfaces.MockAuthChecker + Log *mock_wd.MockWatchdog +} + +func setupMocks(ctrl *gomock.Controller) (*Config, *mocks) { skinsRepo := mock_interfaces.NewMockSkinsRepository(ctrl) capesRepo := mock_interfaces.NewMockCapesRepository(ctrl) authChecker := mock_interfaces.NewMockAuthChecker(ctrl) wd := mock_wd.NewMockWatchdog(ctrl) - texturesQueue := &tests.MojangTexturesQueueMock{} + texturesProvider := &mojangTexturesProviderMock{} return &Config{ - SkinsRepo: skinsRepo, - CapesRepo: capesRepo, - Auth: authChecker, - MojangTexturesQueue: texturesQueue, - Logger: wd, - }, &mocks{ - Skins: skinsRepo, - Capes: capesRepo, - Auth: authChecker, - Queue: texturesQueue, - Log: wd, - } + SkinsRepo: skinsRepo, + CapesRepo: capesRepo, + Auth: authChecker, + MojangTexturesProvider: texturesProvider, + Logger: wd, + }, &mocks{ + Skins: skinsRepo, + Capes: capesRepo, + Auth: authChecker, + MojangProvider: texturesProvider, + Log: wd, + } } func createTexturesResponse(includeSkin bool, includeCape bool) *mojang.SignedTexturesResponse { timeZone, _ := time.LoadLocation("Europe/Minsk") textures := &mojang.TexturesProp{ - Timestamp: time.Date(2019, 4, 27, 23, 56, 12, 0, timeZone).Unix(), - ProfileID: "00000000000000000000000000000000", + Timestamp: time.Date(2019, 4, 27, 23, 56, 12, 0, timeZone).Unix(), + ProfileID: "00000000000000000000000000000000", ProfileName: "mock_user", - Textures: &mojang.TexturesResponse{}, + Textures: &mojang.TexturesResponse{}, } if includeSkin { @@ -75,11 +87,11 @@ func createTexturesResponse(includeSkin bool, includeCape bool) *mojang.SignedTe } response := &mojang.SignedTexturesResponse{ - Id: "00000000000000000000000000000000", + Id: "00000000000000000000000000000000", Name: "mock_user", Props: []*mojang.Property{ { - Name: "textures", + Name: "textures", Value: mojang.EncodeTextures(textures), }, }, diff --git a/http/signed_textures.go b/http/signed_textures.go index 34002a5..a3b1a20 100644 --- a/http/signed_textures.go +++ b/http/signed_textures.go @@ -30,7 +30,10 @@ func (cfg *Config) SignedTextures(response http.ResponseWriter, request *http.Re }, } } else if request.URL.Query().Get("proxy") != "" { - responseData = <-cfg.MojangTexturesQueue.GetTexturesForUsername(username) + mojangTextures, err := cfg.MojangTexturesProvider.GetForUsername(username) + if err == nil && mojangTextures != nil { + responseData = mojangTextures + } } if responseData == nil { diff --git a/http/signed_textures_test.go b/http/signed_textures_test.go index c10fe1d..c07a969 100644 --- a/http/signed_textures_test.go +++ b/http/signed_textures_test.go @@ -112,7 +112,7 @@ func TestConfig_SignedTextures(t *testing.T) { mocks.Log.EXPECT().IncCounter("signed_textures.request", int64(1)) mocks.Skins.EXPECT().FindByUsername("mock_user").Return(skinModel, nil) - mocks.Queue.On("GetTexturesForUsername", "mock_user").Once().Return(createTexturesResponse(true, false)) + mocks.MojangProvider.On("GetForUsername", "mock_user").Once().Return(createTexturesResponse(true, false), nil) req := httptest.NewRequest("GET", "http://chrly/textures/signed/mock_user?proxy=true", nil) w := httptest.NewRecorder() diff --git a/http/skin.go b/http/skin.go index faa8fae..a7f9397 100644 --- a/http/skin.go +++ b/http/skin.go @@ -18,8 +18,8 @@ func (cfg *Config) Skin(response http.ResponseWriter, request *http.Request) { return } - mojangTextures := <-cfg.MojangTexturesQueue.GetTexturesForUsername(username) - if mojangTextures == nil { + mojangTextures, err := cfg.MojangTexturesProvider.GetForUsername(username) + if err != nil || mojangTextures == nil { response.WriteHeader(http.StatusNotFound) return } diff --git a/http/skin_test.go b/http/skin_test.go index 4a9fd97..23475ab 100644 --- a/http/skin_test.go +++ b/http/skin_test.go @@ -78,9 +78,9 @@ func TestConfig_Skin(t *testing.T) { if testCase.ExistsInMojang { textures := createTexturesResponse(testCase.HasSkinInMojangResp, true) - mocks.Queue.On("GetTexturesForUsername", "mock_username").Return(textures) + mocks.MojangProvider.On("GetForUsername", "mock_username").Return(textures, nil) } else { - mocks.Queue.On("GetTexturesForUsername", "mock_username").Return(nil) + mocks.MojangProvider.On("GetForUsername", "mock_username").Return(nil, nil) } req := httptest.NewRequest("GET", testCase.RequestUrl, nil) diff --git a/http/textures.go b/http/textures.go index 92cda48..244cd25 100644 --- a/http/textures.go +++ b/http/textures.go @@ -39,8 +39,8 @@ func (cfg *Config) Textures(response http.ResponseWriter, request *http.Request) } } } else { - mojangTextures := <-cfg.MojangTexturesQueue.GetTexturesForUsername(username) - if mojangTextures == nil { + mojangTextures, err := cfg.MojangTexturesProvider.GetForUsername(username) + if err != nil || mojangTextures == nil { response.WriteHeader(http.StatusNoContent) return } diff --git a/http/textures_test.go b/http/textures_test.go index d7c57e5..9ffa91c 100644 --- a/http/textures_test.go +++ b/http/textures_test.go @@ -148,7 +148,7 @@ func TestConfig_Textures(t *testing.T) { mocks.Skins.EXPECT().FindByUsername("mock_username").Return(nil, &db.SkinNotFoundError{}) mocks.Capes.EXPECT().FindByUsername("mock_username").Return(nil, &db.CapeNotFoundError{}) - mocks.Queue.On("GetTexturesForUsername", "mock_username").Once().Return(createTexturesResponse(true, true)) + mocks.MojangProvider.On("GetForUsername", "mock_username").Once().Return(createTexturesResponse(true, true), nil) req := httptest.NewRequest("GET", "http://chrly/textures/mock_username", nil) w := httptest.NewRecorder() @@ -181,7 +181,7 @@ func TestConfig_Textures(t *testing.T) { mocks.Skins.EXPECT().FindByUsername("mock_username").Return(nil, &db.SkinNotFoundError{}) mocks.Capes.EXPECT().FindByUsername("mock_username").Return(nil, &db.CapeNotFoundError{}) - mocks.Queue.On("GetTexturesForUsername", "mock_username").Once().Return(nil) + mocks.MojangProvider.On("GetForUsername", "mock_username").Once().Return(nil, nil) req := httptest.NewRequest("GET", "http://chrly/textures/mock_username", nil) w := httptest.NewRecorder() diff --git a/interfaces/repositories.go b/interfaces/repositories.go index 134f141..848045a 100644 --- a/interfaces/repositories.go +++ b/interfaces/repositories.go @@ -17,6 +17,6 @@ type CapesRepository interface { FindByUsername(username string) (*model.Cape, error) } -type MojangTexturesQueue interface { - GetTexturesForUsername(username string) chan *mojang.SignedTexturesResponse +type MojangTexturesProvider interface { + GetForUsername(username string) (*mojang.SignedTexturesResponse, error) } diff --git a/mojangtextures/batch_uuids_provider.go b/mojangtextures/batch_uuids_provider.go new file mode 100644 index 0000000..d3259c1 --- /dev/null +++ b/mojangtextures/batch_uuids_provider.go @@ -0,0 +1,133 @@ +package mojangtextures + +import ( + "strings" + "sync" + "time" + + "github.com/mono83/slf/wd" + + "github.com/elyby/chrly/api/mojang" +) + +type jobResult struct { + profile *mojang.ProfileInfo + error error +} + +type jobItem struct { + username string + respondChan chan *jobResult +} + +type jobsQueue struct { + lock sync.Mutex + items []*jobItem +} + +func (s *jobsQueue) New() *jobsQueue { + s.items = []*jobItem{} + return s +} + +func (s *jobsQueue) Enqueue(t *jobItem) { + s.lock.Lock() + defer s.lock.Unlock() + + s.items = append(s.items, t) +} + +func (s *jobsQueue) Dequeue(n int) []*jobItem { + s.lock.Lock() + defer s.lock.Unlock() + + if n > s.Size() { + n = s.Size() + } + + items := s.items[0:n] + s.items = s.items[n:len(s.items)] + + return items +} + +func (s *jobsQueue) Size() int { + return len(s.items) +} + +var usernamesToUuids = mojang.UsernamesToUuids +var forever = func() bool { + return true +} + +type BatchUuidsProvider struct { + IterationDelay time.Duration + IterationSize int + Logger wd.Watchdog + + onFirstCall sync.Once + queue jobsQueue +} + +func (ctx *BatchUuidsProvider) GetUuid(username string) (*mojang.ProfileInfo, error) { + ctx.onFirstCall.Do(func() { + ctx.queue.New() + ctx.startQueue() + }) + + resultChan := make(chan *jobResult) + ctx.queue.Enqueue(&jobItem{username, resultChan}) + ctx.Logger.IncCounter("mojang_textures.usernames.queued", 1) + + result := <-resultChan + + return result.profile, result.error +} + +func (ctx *BatchUuidsProvider) startQueue() { + go func() { + time.Sleep(ctx.IterationDelay) + for forever() { + start := time.Now() + ctx.queueRound() + elapsed := time.Since(start) + ctx.Logger.RecordTimer("mojang_textures.usernames.round_time", elapsed) + time.Sleep(ctx.IterationDelay) + } + }() +} + +func (ctx *BatchUuidsProvider) queueRound() { + queueSize := ctx.queue.Size() + jobs := ctx.queue.Dequeue(ctx.IterationSize) + ctx.Logger.UpdateGauge("mojang_textures.usernames.queue_size", int64(queueSize-len(jobs))) + ctx.Logger.UpdateGauge("mojang_textures.usernames.iteration_size", int64(len(jobs))) + if len(jobs) == 0 { + return + } + + var usernames []string + for _, job := range jobs { + usernames = append(usernames, job.username) + } + + profiles, err := usernamesToUuids(usernames) + for _, job := range jobs { + go func(job *jobItem) { + response := &jobResult{} + if err != nil { + response.error = err + } else { + // The profiles in the response aren't ordered, so we must search each username over full array + for _, profile := range profiles { + if strings.EqualFold(job.username, profile.Name) { + response.profile = profile + break + } + } + } + + job.respondChan <- response + }(job) + } +} diff --git a/mojangtextures/batch_uuids_provider_test.go b/mojangtextures/batch_uuids_provider_test.go new file mode 100644 index 0000000..f217dd0 --- /dev/null +++ b/mojangtextures/batch_uuids_provider_test.go @@ -0,0 +1,285 @@ +package mojangtextures + +import ( + "crypto/rand" + "encoding/base64" + "strings" + "testing" + "time" + + testify "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" + + "github.com/elyby/chrly/api/mojang" + mocks "github.com/elyby/chrly/tests" +) + +func TestJobsQueue(t *testing.T) { + createQueue := func() *jobsQueue { + queue := &jobsQueue{} + queue.New() + + return queue + } + + t.Run("Enqueue", func(t *testing.T) { + assert := testify.New(t) + + s := createQueue() + s.Enqueue(&jobItem{username: "username1"}) + s.Enqueue(&jobItem{username: "username2"}) + s.Enqueue(&jobItem{username: "username3"}) + + assert.Equal(3, s.Size()) + }) + + t.Run("Dequeue", func(t *testing.T) { + assert := testify.New(t) + + s := createQueue() + s.Enqueue(&jobItem{username: "username1"}) + s.Enqueue(&jobItem{username: "username2"}) + s.Enqueue(&jobItem{username: "username3"}) + s.Enqueue(&jobItem{username: "username4"}) + + items := s.Dequeue(2) + assert.Len(items, 2) + assert.Equal("username1", items[0].username) + assert.Equal("username2", items[1].username) + assert.Equal(2, s.Size()) + + items = s.Dequeue(40) + assert.Len(items, 2) + assert.Equal("username3", items[0].username) + assert.Equal("username4", items[1].username) + }) +} + +// This is really stupid test just to get 100% coverage on this package :) +func TestBatchUuidsProvider_forever(t *testing.T) { + testify.True(t, forever()) +} + +type mojangUsernamesToUuidsRequestMock struct { + mock.Mock +} + +func (o *mojangUsernamesToUuidsRequestMock) UsernamesToUuids(usernames []string) ([]*mojang.ProfileInfo, error) { + args := o.Called(usernames) + var result []*mojang.ProfileInfo + if casted, ok := args.Get(0).([]*mojang.ProfileInfo); ok { + result = casted + } + + return result, args.Error(1) +} + +type batchUuidsProviderGetUuidResult struct { + Result *mojang.ProfileInfo + Error error +} + +type batchUuidsProviderTestSuite struct { + suite.Suite + + Provider *BatchUuidsProvider + GetUuidAsync func(username string) chan *batchUuidsProviderGetUuidResult + + Logger *mocks.WdMock + MojangApi *mojangUsernamesToUuidsRequestMock + + Iterate func() + done func() + iterateChan chan bool +} + +func (suite *batchUuidsProviderTestSuite) SetupTest() { + suite.Logger = &mocks.WdMock{} + + suite.Provider = &BatchUuidsProvider{ + Logger: suite.Logger, + IterationDelay: 0, + IterationSize: 10, + } + + suite.iterateChan = make(chan bool) + forever = func() bool { + return <-suite.iterateChan + } + + suite.Iterate = func() { + suite.iterateChan <- true + } + + suite.done = func() { + suite.iterateChan <- false + } + + suite.GetUuidAsync = func(username string) chan *batchUuidsProviderGetUuidResult { + c := make(chan *batchUuidsProviderGetUuidResult) + go func() { + profile, err := suite.Provider.GetUuid(username) + c <- &batchUuidsProviderGetUuidResult{ + Result: profile, + Error: err, + } + }() + + return c + } + + suite.MojangApi = &mojangUsernamesToUuidsRequestMock{} + usernamesToUuids = suite.MojangApi.UsernamesToUuids +} + +func (suite *batchUuidsProviderTestSuite) TearDownTest() { + suite.done() + time.Sleep(10 * time.Millisecond) // Add delay to let finish all goroutines before assert mocks calls + suite.MojangApi.AssertExpectations(suite.T()) + suite.Logger.AssertExpectations(suite.T()) +} + +func TestBatchUuidsProvider(t *testing.T) { + suite.Run(t, new(batchUuidsProviderTestSuite)) +} + +func (suite *batchUuidsProviderTestSuite) TestGetUuidForOneUsername() { + expectedResult := &mojang.ProfileInfo{Id: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Name: "username"} + + suite.Logger.On("IncCounter", "mojang_textures.usernames.queued", int64(1)).Once() + suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(1)).Once() + suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(0)).Once() + suite.Logger.On("RecordTimer", "mojang_textures.usernames.round_time", mock.Anything).Once() + + suite.MojangApi.On("UsernamesToUuids", []string{"username"}).Once().Return([]*mojang.ProfileInfo{expectedResult}, nil) + + resultChan := suite.GetUuidAsync("username") + + suite.Iterate() + + result := <-resultChan + suite.Assert().Equal(expectedResult, result.Result) + suite.Assert().Nil(result.Error) +} + +func (suite *batchUuidsProviderTestSuite) TestGetUuidForTwoUsernames() { + expectedResult1 := &mojang.ProfileInfo{Id: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Name: "username1"} + expectedResult2 := &mojang.ProfileInfo{Id: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", Name: "username2"} + + suite.Logger.On("IncCounter", "mojang_textures.usernames.queued", int64(1)).Twice() + suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(2)).Once() + suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(0)).Once() + suite.Logger.On("RecordTimer", "mojang_textures.usernames.round_time", mock.Anything).Once() + + suite.MojangApi.On("UsernamesToUuids", []string{"username1", "username2"}).Once().Return([]*mojang.ProfileInfo{ + expectedResult1, + expectedResult2, + }, nil) + + resultChan1 := suite.GetUuidAsync("username1") + time.Sleep(time.Millisecond) // Just to keep order for the usernames + resultChan2 := suite.GetUuidAsync("username2") + time.Sleep(time.Millisecond) // Allow to all goroutines begin + + suite.Iterate() + + result1 := <-resultChan1 + suite.Assert().Equal(expectedResult1, result1.Result) + suite.Assert().Nil(result1.Error) + + result2 := <-resultChan2 + suite.Assert().Equal(expectedResult2, result2.Result) + suite.Assert().Nil(result2.Error) +} + +func (suite *batchUuidsProviderTestSuite) TestGetUuidForMoreThan10Usernames() { + usernames := make([]string, 12) + for i := 0; i < cap(usernames); i++ { + usernames[i] = randStr(8) + } + + suite.Logger.On("IncCounter", "mojang_textures.usernames.queued", int64(1)).Times(12) + suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(10)).Once() + suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(2)).Once() + suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(2)).Once() + suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(0)).Once() + suite.Logger.On("RecordTimer", "mojang_textures.usernames.round_time", mock.Anything).Twice() + + suite.MojangApi.On("UsernamesToUuids", usernames[0:10]).Once().Return([]*mojang.ProfileInfo{}, nil) + suite.MojangApi.On("UsernamesToUuids", usernames[10:12]).Once().Return([]*mojang.ProfileInfo{}, nil) + + channels := make([]chan *batchUuidsProviderGetUuidResult, 12) + for i, username := range usernames { + channels[i] = suite.GetUuidAsync(username) + time.Sleep(time.Millisecond) // Just to keep order for the usernames + } + + suite.Iterate() + suite.Iterate() + + for _, channel := range channels { + <-channel + } +} + +func (suite *batchUuidsProviderTestSuite) TestDoNothingWhenNoTasks() { + suite.Logger.On("IncCounter", "mojang_textures.usernames.queued", int64(1)).Once() + suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(1)).Once() + suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(0)).Twice() + suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(0)).Times(3) + suite.Logger.On("RecordTimer", "mojang_textures.usernames.round_time", mock.Anything) + + suite.MojangApi.On("UsernamesToUuids", []string{"username"}).Once().Return([]*mojang.ProfileInfo{}, nil) + + // Perform first iteration and await it finish + resultChan := suite.GetUuidAsync("username") + + suite.Iterate() + + result := <-resultChan + suite.Assert().Nil(result.Result) + suite.Assert().Nil(result.Error) + + // Let it to perform a few more iterations to ensure, that there is no calls to external APIs + suite.Iterate() + suite.Iterate() +} + +func (suite *batchUuidsProviderTestSuite) TestGetUuidForTwoUsernamesWithAnError() { + expectedError := &mojang.TooManyRequestsError{} + + suite.Logger.On("IncCounter", "mojang_textures.usernames.queued", int64(1)).Twice() + suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(2)).Once() + suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(0)).Once() + suite.Logger.On("RecordTimer", "mojang_textures.usernames.round_time", mock.Anything).Once() + + suite.MojangApi.On("UsernamesToUuids", []string{"username1", "username2"}).Once().Return(nil, expectedError) + + resultChan1 := suite.GetUuidAsync("username1") + time.Sleep(time.Millisecond) // Just to keep order for the usernames + resultChan2 := suite.GetUuidAsync("username2") + time.Sleep(time.Millisecond) // Allow to all goroutines begin + + suite.Iterate() + + result1 := <-resultChan1 + suite.Assert().Nil(result1.Result) + suite.Assert().Equal(expectedError, result1.Error) + + result2 := <-resultChan2 + suite.Assert().Nil(result2.Result) + suite.Assert().Equal(expectedError, result2.Error) +} + +var replacer = strings.NewReplacer("-", "_", "=", "") + +// https://stackoverflow.com/a/50581165 +func randStr(len int) string { + buff := make([]byte, len) + _, _ = rand.Read(buff) + str := replacer.Replace(base64.URLEncoding.EncodeToString(buff)) + + // Base 64 can be longer than len + return str[:len] +} diff --git a/api/mojang/queue/in_memory_textures_storage.go b/mojangtextures/in_memory_textures_storage.go similarity index 62% rename from api/mojang/queue/in_memory_textures_storage.go rename to mojangtextures/in_memory_textures_storage.go index cb7d8a3..37671df 100644 --- a/api/mojang/queue/in_memory_textures_storage.go +++ b/mojangtextures/in_memory_textures_storage.go @@ -1,4 +1,4 @@ -package queue +package mojangtextures import ( "sync" @@ -9,8 +9,6 @@ import ( "github.com/tevino/abool" ) -var inMemoryStorageGCPeriod = 10 * time.Second -var inMemoryStoragePersistPeriod = time.Minute + 10*time.Second var now = time.Now type inMemoryItem struct { @@ -18,33 +16,38 @@ type inMemoryItem struct { timestamp int64 } -type inMemoryTexturesStorage struct { +type InMemoryTexturesStorage struct { + GCPeriod time.Duration + Duration time.Duration + lock sync.Mutex data map[string]*inMemoryItem working *abool.AtomicBool } -func CreateInMemoryTexturesStorage() *inMemoryTexturesStorage { - storage := &inMemoryTexturesStorage{ - data: make(map[string]*inMemoryItem), +func NewInMemoryTexturesStorage() *InMemoryTexturesStorage { + storage := &InMemoryTexturesStorage{ + GCPeriod: 10 * time.Second, + Duration: time.Minute + 10*time.Second, + data: make(map[string]*inMemoryItem), } return storage } -func (s *inMemoryTexturesStorage) Start() { +func (s *InMemoryTexturesStorage) Start() { if s.working == nil { s.working = abool.New() } if !s.working.IsSet() { go func() { - time.Sleep(inMemoryStorageGCPeriod) + time.Sleep(s.GCPeriod) // TODO: this can be reimplemented in future with channels, but right now I have no idea how to make it right for s.working.IsSet() { start := time.Now() s.gc() - time.Sleep(inMemoryStorageGCPeriod - time.Since(start)) + time.Sleep(s.GCPeriod - time.Since(start)) } }() } @@ -52,16 +55,16 @@ func (s *inMemoryTexturesStorage) Start() { s.working.Set() } -func (s *inMemoryTexturesStorage) Stop() { +func (s *InMemoryTexturesStorage) Stop() { s.working.UnSet() } -func (s *inMemoryTexturesStorage) GetTextures(uuid string) (*mojang.SignedTexturesResponse, error) { +func (s *InMemoryTexturesStorage) GetTextures(uuid string) (*mojang.SignedTexturesResponse, error) { s.lock.Lock() defer s.lock.Unlock() item, exists := s.data[uuid] - validRange := getMinimalNotExpiredTimestamp() + validRange := s.getMinimalNotExpiredTimestamp() if !exists || validRange > item.timestamp { return nil, &ValueNotFound{} } @@ -69,7 +72,7 @@ func (s *inMemoryTexturesStorage) GetTextures(uuid string) (*mojang.SignedTextur return item.textures, nil } -func (s *inMemoryTexturesStorage) StoreTextures(uuid string, textures *mojang.SignedTexturesResponse) { +func (s *InMemoryTexturesStorage) StoreTextures(uuid string, textures *mojang.SignedTexturesResponse) { var timestamp int64 if textures != nil { decoded := textures.DecodeTextures() @@ -91,11 +94,11 @@ func (s *inMemoryTexturesStorage) StoreTextures(uuid string, textures *mojang.Si } } -func (s *inMemoryTexturesStorage) gc() { +func (s *InMemoryTexturesStorage) gc() { s.lock.Lock() defer s.lock.Unlock() - maxTime := getMinimalNotExpiredTimestamp() + maxTime := s.getMinimalNotExpiredTimestamp() for uuid, value := range s.data { if maxTime > value.timestamp { delete(s.data, uuid) @@ -103,8 +106,8 @@ func (s *inMemoryTexturesStorage) gc() { } } -func getMinimalNotExpiredTimestamp() int64 { - return unixNanoToUnixMicro(now().Add(inMemoryStoragePersistPeriod * time.Duration(-1)).UnixNano()) +func (s *InMemoryTexturesStorage) getMinimalNotExpiredTimestamp() int64 { + return unixNanoToUnixMicro(now().Add(s.Duration * time.Duration(-1)).UnixNano()) } func unixNanoToUnixMicro(unixNano int64) int64 { diff --git a/api/mojang/queue/in_memory_textures_storage_test.go b/mojangtextures/in_memory_textures_storage_test.go similarity index 85% rename from api/mojang/queue/in_memory_textures_storage_test.go rename to mojangtextures/in_memory_textures_storage_test.go index 1e90812..3ad6789 100644 --- a/api/mojang/queue/in_memory_textures_storage_test.go +++ b/mojangtextures/in_memory_textures_storage_test.go @@ -1,4 +1,4 @@ -package queue +package mojangtextures import ( "time" @@ -48,7 +48,7 @@ func TestInMemoryTexturesStorage_GetTextures(t *testing.T) { t.Run("get error when uuid is not exists", func(t *testing.T) { assert := testify.New(t) - storage := CreateInMemoryTexturesStorage() + storage := NewInMemoryTexturesStorage() result, err := storage.GetTextures("b5d58475007d4f9e9ddd1403e2497579") assert.Nil(result) @@ -58,7 +58,7 @@ func TestInMemoryTexturesStorage_GetTextures(t *testing.T) { t.Run("get textures object, when uuid is stored in the storage", func(t *testing.T) { assert := testify.New(t) - storage := CreateInMemoryTexturesStorage() + storage := NewInMemoryTexturesStorage() storage.StoreTextures("dead24f9a4fa4877b7b04c8c6c72bb46", texturesWithSkin) result, err := storage.GetTextures("dead24f9a4fa4877b7b04c8c6c72bb46") @@ -69,7 +69,7 @@ func TestInMemoryTexturesStorage_GetTextures(t *testing.T) { t.Run("get error when uuid is exists, but textures are expired", func(t *testing.T) { assert := testify.New(t) - storage := CreateInMemoryTexturesStorage() + storage := NewInMemoryTexturesStorage() storage.StoreTextures("dead24f9a4fa4877b7b04c8c6c72bb46", texturesWithSkin) now = func() time.Time { @@ -89,7 +89,7 @@ func TestInMemoryTexturesStorage_StoreTextures(t *testing.T) { t.Run("store textures for previously not existed uuid", func(t *testing.T) { assert := testify.New(t) - storage := CreateInMemoryTexturesStorage() + storage := NewInMemoryTexturesStorage() storage.StoreTextures("dead24f9a4fa4877b7b04c8c6c72bb46", texturesWithSkin) result, err := storage.GetTextures("dead24f9a4fa4877b7b04c8c6c72bb46") @@ -100,7 +100,7 @@ func TestInMemoryTexturesStorage_StoreTextures(t *testing.T) { t.Run("override already existed textures for uuid", func(t *testing.T) { assert := testify.New(t) - storage := CreateInMemoryTexturesStorage() + storage := NewInMemoryTexturesStorage() storage.StoreTextures("dead24f9a4fa4877b7b04c8c6c72bb46", texturesWithoutSkin) storage.StoreTextures("dead24f9a4fa4877b7b04c8c6c72bb46", texturesWithSkin) result, err := storage.GetTextures("dead24f9a4fa4877b7b04c8c6c72bb46") @@ -113,7 +113,7 @@ func TestInMemoryTexturesStorage_StoreTextures(t *testing.T) { t.Run("store nil textures", func(t *testing.T) { assert := testify.New(t) - storage := CreateInMemoryTexturesStorage() + storage := NewInMemoryTexturesStorage() storage.StoreTextures("dead24f9a4fa4877b7b04c8c6c72bb46", nil) result, err := storage.GetTextures("dead24f9a4fa4877b7b04c8c6c72bb46") @@ -131,7 +131,7 @@ func TestInMemoryTexturesStorage_StoreTextures(t *testing.T) { } assert.PanicsWithValue("unable to decode textures", func() { - storage := CreateInMemoryTexturesStorage() + storage := NewInMemoryTexturesStorage() storage.StoreTextures("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", toStore) }) }) @@ -140,8 +140,9 @@ func TestInMemoryTexturesStorage_StoreTextures(t *testing.T) { func TestInMemoryTexturesStorage_GarbageCollection(t *testing.T) { assert := testify.New(t) - inMemoryStorageGCPeriod = 10 * time.Millisecond - inMemoryStoragePersistPeriod = 10 * time.Millisecond + storage := NewInMemoryTexturesStorage() + storage.GCPeriod = 10 * time.Millisecond + storage.Duration = 10 * time.Millisecond textures1 := &mojang.SignedTexturesResponse{ Id: "dead24f9a4fa4877b7b04c8c6c72bb46", @@ -150,7 +151,7 @@ func TestInMemoryTexturesStorage_GarbageCollection(t *testing.T) { { Name: "textures", Value: mojang.EncodeTextures(&mojang.TexturesProp{ - Timestamp: time.Now().Add(inMemoryStorageGCPeriod-time.Millisecond*time.Duration(5)).UnixNano() / 10e5, + Timestamp: time.Now().Add(storage.GCPeriod-time.Millisecond*time.Duration(5)).UnixNano() / 10e5, ProfileID: "dead24f9a4fa4877b7b04c8c6c72bb46", ProfileName: "mock1", Textures: &mojang.TexturesResponse{}, @@ -165,7 +166,7 @@ func TestInMemoryTexturesStorage_GarbageCollection(t *testing.T) { { Name: "textures", Value: mojang.EncodeTextures(&mojang.TexturesProp{ - Timestamp: time.Now().Add(inMemoryStorageGCPeriod-time.Millisecond*time.Duration(15)).UnixNano() / 10e5, + Timestamp: time.Now().Add(storage.GCPeriod-time.Millisecond*time.Duration(15)).UnixNano() / 10e5, ProfileID: "b5d58475007d4f9e9ddd1403e2497579", ProfileName: "mock2", Textures: &mojang.TexturesResponse{}, @@ -174,13 +175,12 @@ func TestInMemoryTexturesStorage_GarbageCollection(t *testing.T) { }, } - storage := CreateInMemoryTexturesStorage() storage.StoreTextures("dead24f9a4fa4877b7b04c8c6c72bb46", textures1) storage.StoreTextures("b5d58475007d4f9e9ddd1403e2497579", textures2) storage.Start() - time.Sleep(inMemoryStorageGCPeriod + time.Millisecond) // Let it start first iteration + time.Sleep(storage.GCPeriod + time.Millisecond) // Let it start first iteration _, textures1Err := storage.GetTextures("dead24f9a4fa4877b7b04c8c6c72bb46") _, textures2Err := storage.GetTextures("b5d58475007d4f9e9ddd1403e2497579") @@ -188,7 +188,7 @@ func TestInMemoryTexturesStorage_GarbageCollection(t *testing.T) { assert.Nil(textures1Err) assert.Error(textures2Err) - time.Sleep(inMemoryStorageGCPeriod + time.Millisecond) // Let another iteration happen + time.Sleep(storage.GCPeriod + time.Millisecond) // Let another iteration happen _, textures1Err = storage.GetTextures("dead24f9a4fa4877b7b04c8c6c72bb46") _, textures2Err = storage.GetTextures("b5d58475007d4f9e9ddd1403e2497579") diff --git a/mojangtextures/mojang_api_textures_provider.go b/mojangtextures/mojang_api_textures_provider.go new file mode 100644 index 0000000..716f316 --- /dev/null +++ b/mojangtextures/mojang_api_textures_provider.go @@ -0,0 +1,25 @@ +package mojangtextures + +import ( + "time" + + "github.com/mono83/slf/wd" + + "github.com/elyby/chrly/api/mojang" +) + +var uuidToTextures = mojang.UuidToTextures + +type MojangApiTexturesProvider struct { + Logger wd.Watchdog +} + +func (ctx *MojangApiTexturesProvider) GetTextures(uuid string) (*mojang.SignedTexturesResponse, error) { + ctx.Logger.IncCounter("mojang_textures.textures.request", 1) + + start := time.Now() + result, err := uuidToTextures(uuid, true) + ctx.Logger.RecordTimer("mojang_textures.textures.request_time", time.Since(start)) + + return result, err +} diff --git a/mojangtextures/mojang_api_textures_provider_test.go b/mojangtextures/mojang_api_textures_provider_test.go new file mode 100644 index 0000000..ed2a349 --- /dev/null +++ b/mojangtextures/mojang_api_textures_provider_test.go @@ -0,0 +1,82 @@ +package mojangtextures + +import ( + "testing" + + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" + + "github.com/elyby/chrly/api/mojang" + mocks "github.com/elyby/chrly/tests" +) + +type mojangUuidToTexturesRequestMock struct { + mock.Mock +} + +func (o *mojangUuidToTexturesRequestMock) UuidToTextures(uuid string, signed bool) (*mojang.SignedTexturesResponse, error) { + args := o.Called(uuid, signed) + var result *mojang.SignedTexturesResponse + if casted, ok := args.Get(0).(*mojang.SignedTexturesResponse); ok { + result = casted + } + + return result, args.Error(1) +} + +type mojangApiTexturesProviderTestSuite struct { + suite.Suite + + Provider *MojangApiTexturesProvider + Logger *mocks.WdMock + MojangApi *mojangUuidToTexturesRequestMock +} + +func (suite *mojangApiTexturesProviderTestSuite) SetupTest() { + suite.Logger = &mocks.WdMock{} + suite.MojangApi = &mojangUuidToTexturesRequestMock{} + + suite.Provider = &MojangApiTexturesProvider{ + Logger: suite.Logger, + } + + uuidToTextures = suite.MojangApi.UuidToTextures +} + +func (suite *mojangApiTexturesProviderTestSuite) TearDownTest() { + suite.MojangApi.AssertExpectations(suite.T()) + suite.Logger.AssertExpectations(suite.T()) +} + +func TestMojangApiTexturesProvider(t *testing.T) { + suite.Run(t, new(mojangApiTexturesProviderTestSuite)) +} + +func (suite *mojangApiTexturesProviderTestSuite) TestGetTextures() { + expectedResult := &mojang.SignedTexturesResponse{ + Id: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Name: "username", + } + suite.MojangApi.On("UuidToTextures", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", true).Once().Return(expectedResult, nil) + + suite.Logger.On("IncCounter", "mojang_textures.textures.request", int64(1)).Once() + suite.Logger.On("RecordTimer", "mojang_textures.textures.request_time", mock.Anything).Once() + + result, err := suite.Provider.GetTextures("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") + + suite.Assert().Equal(expectedResult, result) + suite.Assert().Nil(err) +} + +func (suite *mojangApiTexturesProviderTestSuite) TestGetTexturesWithError() { + expectedError := &mojang.TooManyRequestsError{} + suite.MojangApi.On("UuidToTextures", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", true).Once().Return(nil, expectedError) + + suite.Logger.On("IncCounter", "mojang_textures.textures.request", int64(1)).Once() + suite.Logger.On("RecordTimer", "mojang_textures.textures.request_time", mock.Anything).Once() + + result, err := suite.Provider.GetTextures("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") + + suite.Assert().Nil(result) + suite.Assert().Equal(expectedError, err) +} diff --git a/mojangtextures/mojang_textures.go b/mojangtextures/mojang_textures.go new file mode 100644 index 0000000..33919c6 --- /dev/null +++ b/mojangtextures/mojang_textures.go @@ -0,0 +1,225 @@ +package mojangtextures + +import ( + "errors" + "net" + "net/url" + "regexp" + "strings" + "sync" + "syscall" + "time" + + "github.com/mono83/slf/wd" + + "github.com/elyby/chrly/api/mojang" +) + +type broadcastResult struct { + textures *mojang.SignedTexturesResponse + error error +} + +type broadcaster struct { + lock sync.Mutex + listeners map[string][]chan *broadcastResult +} + +func createBroadcaster() *broadcaster { + return &broadcaster{ + listeners: make(map[string][]chan *broadcastResult), + } +} + +// Returns a boolean value, which will be true if the passed username didn't exist before +func (c *broadcaster) AddListener(username string, resultChan chan *broadcastResult) bool { + c.lock.Lock() + defer c.lock.Unlock() + + val, alreadyHasSource := c.listeners[username] + if alreadyHasSource { + c.listeners[username] = append(val, resultChan) + return false + } + + c.listeners[username] = []chan *broadcastResult{resultChan} + + return true +} + +func (c *broadcaster) BroadcastAndRemove(username string, result *broadcastResult) { + c.lock.Lock() + defer c.lock.Unlock() + + val, ok := c.listeners[username] + if !ok { + return + } + + for _, channel := range val { + go func(channel chan *broadcastResult) { + channel <- result + close(channel) + }(channel) + } + + delete(c.listeners, username) +} + +// https://help.mojang.com/customer/portal/articles/928638 +var allowedUsernamesRegex = regexp.MustCompile(`^[\w_]{3,16}$`) + +type UuidsProvider interface { + GetUuid(username string) (*mojang.ProfileInfo, error) +} + +type TexturesProvider interface { + GetTextures(uuid string) (*mojang.SignedTexturesResponse, error) +} + +type Provider struct { + UuidsProvider + TexturesProvider + Storage + Logger wd.Watchdog + + onFirstCall sync.Once + *broadcaster +} + +func (ctx *Provider) GetForUsername(username string) (*mojang.SignedTexturesResponse, error) { + ctx.onFirstCall.Do(func() { + ctx.broadcaster = createBroadcaster() + }) + + if !allowedUsernamesRegex.MatchString(username) { + ctx.Logger.IncCounter("mojang_textures.invalid_username", 1) + return nil, errors.New("invalid username") + } + + username = strings.ToLower(username) + ctx.Logger.IncCounter("mojang_textures.request", 1) + + uuid, err := ctx.Storage.GetUuid(username) + if err == nil && uuid == "" { + ctx.Logger.IncCounter("mojang_textures.usernames.cache_hit_nil", 1) + return nil, nil + } + + if uuid != "" { + ctx.Logger.IncCounter("mojang_textures.usernames.cache_hit", 1) + textures, err := ctx.Storage.GetTextures(uuid) + if err == nil { + ctx.Logger.IncCounter("mojang_textures.textures.cache_hit", 1) + return textures, nil + } + } + + resultChan := make(chan *broadcastResult) + isFirstListener := ctx.broadcaster.AddListener(username, resultChan) + if isFirstListener { + go ctx.getResultAndBroadcast(username, uuid) + } else { + ctx.Logger.IncCounter("mojang_textures.already_scheduled", 1) + } + + result := <-resultChan + + return result.textures, result.error +} + +func (ctx *Provider) getResultAndBroadcast(username string, uuid string) { + start := time.Now() + + result := ctx.getResult(username, uuid) + ctx.broadcaster.BroadcastAndRemove(username, result) + + ctx.Logger.RecordTimer("mojang_textures.result_time", time.Since(start)) +} + +func (ctx *Provider) getResult(username string, uuid string) *broadcastResult { + if uuid == "" { + profile, err := ctx.UuidsProvider.GetUuid(username) + if err != nil { + ctx.handleMojangApiResponseError(err, "usernames") + return &broadcastResult{nil, err} + } + + uuid = "" + if profile != nil { + uuid = profile.Id + } + + _ = ctx.Storage.StoreUuid(username, uuid) + + if uuid == "" { + ctx.Logger.IncCounter("mojang_textures.usernames.uuid_miss", 1) + return &broadcastResult{nil, nil} + } + + ctx.Logger.IncCounter("mojang_textures.usernames.uuid_hit", 1) + } + + textures, err := ctx.TexturesProvider.GetTextures(uuid) + if err != nil { + ctx.handleMojangApiResponseError(err, "textures") + return &broadcastResult{nil, err} + } + + // Mojang can respond with an error, but it will still count as a hit, + // therefore store the result even if textures is nil to prevent 429 error + ctx.Storage.StoreTextures(uuid, textures) + + if textures != nil { + ctx.Logger.IncCounter("mojang_textures.usernames.textures_hit", 1) + } else { + ctx.Logger.IncCounter("mojang_textures.usernames.textures_miss", 1) + } + + return &broadcastResult{textures, nil} +} + +func (ctx *Provider) handleMojangApiResponseError(err error, threadName string) { + errParam := wd.ErrParam(err) + threadParam := wd.NameParam(threadName) + + ctx.Logger.Debug(":name: Got response error :err", threadParam, errParam) + + switch err.(type) { + case mojang.ResponseError: + if _, ok := err.(*mojang.BadRequestError); ok { + ctx.Logger.Warning(":name: Got 400 Bad Request :err", threadParam, errParam) + return + } + + if _, ok := err.(*mojang.ForbiddenError); ok { + ctx.Logger.Warning(":name: Got 403 Forbidden :err", threadParam, errParam) + return + } + + if _, ok := err.(*mojang.TooManyRequestsError); ok { + ctx.Logger.Warning(":name: Got 429 Too Many Requests :err", threadParam, errParam) + return + } + + return + case net.Error: + if err.(net.Error).Timeout() { + return + } + + if _, ok := err.(*url.Error); ok { + return + } + + if opErr, ok := err.(*net.OpError); ok && (opErr.Op == "dial" || opErr.Op == "read") { + return + } + + if err == syscall.ECONNREFUSED { + return + } + } + + ctx.Logger.Emergency(":name: Unknown Mojang response error: :err", threadParam, errParam) +} diff --git a/mojangtextures/mojang_textures_test.go b/mojangtextures/mojang_textures_test.go new file mode 100644 index 0000000..fabdee6 --- /dev/null +++ b/mojangtextures/mojang_textures_test.go @@ -0,0 +1,439 @@ +package mojangtextures + +import ( + "errors" + "net" + "net/url" + "sync" + "syscall" + "testing" + "time" + + testify "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" + + "github.com/elyby/chrly/api/mojang" + mocks "github.com/elyby/chrly/tests" +) + +func TestBroadcaster(t *testing.T) { + t.Run("GetOrAppend", func(t *testing.T) { + t.Run("first call when username didn't exist before should return true", func(t *testing.T) { + assert := testify.New(t) + + broadcaster := createBroadcaster() + channel := make(chan *broadcastResult) + isFirstListener := broadcaster.AddListener("mock", channel) + + assert.True(isFirstListener) + listeners, ok := broadcaster.listeners["mock"] + assert.True(ok) + assert.Len(listeners, 1) + assert.Equal(channel, listeners[0]) + }) + + t.Run("subsequent calls should return false", func(t *testing.T) { + assert := testify.New(t) + + broadcaster := createBroadcaster() + channel1 := make(chan *broadcastResult) + isFirstListener := broadcaster.AddListener("mock", channel1) + + assert.True(isFirstListener) + + channel2 := make(chan *broadcastResult) + isFirstListener = broadcaster.AddListener("mock", channel2) + + assert.False(isFirstListener) + + channel3 := make(chan *broadcastResult) + isFirstListener = broadcaster.AddListener("mock", channel3) + + assert.False(isFirstListener) + }) + }) + + t.Run("BroadcastAndRemove", func(t *testing.T) { + t.Run("should broadcast to all listeners and remove the key", func(t *testing.T) { + assert := testify.New(t) + + broadcaster := createBroadcaster() + channel1 := make(chan *broadcastResult) + channel2 := make(chan *broadcastResult) + broadcaster.AddListener("mock", channel1) + broadcaster.AddListener("mock", channel2) + + result := &broadcastResult{} + broadcaster.BroadcastAndRemove("mock", result) + + assert.Equal(result, <-channel1) + assert.Equal(result, <-channel2) + + channel3 := make(chan *broadcastResult) + isFirstListener := broadcaster.AddListener("mock", channel3) + assert.True(isFirstListener) + }) + + t.Run("call on not exists username", func(t *testing.T) { + assert := testify.New(t) + + assert.NotPanics(func() { + broadcaster := createBroadcaster() + broadcaster.BroadcastAndRemove("mock", &broadcastResult{}) + }) + }) + }) +} + +type mockUuidsProvider struct { + mock.Mock +} + +func (m *mockUuidsProvider) GetUuid(username string) (*mojang.ProfileInfo, error) { + args := m.Called(username) + var result *mojang.ProfileInfo + if casted, ok := args.Get(0).(*mojang.ProfileInfo); ok { + result = casted + } + + return result, args.Error(1) +} + +type mockTexturesProvider struct { + mock.Mock +} + +func (m *mockTexturesProvider) GetTextures(uuid string) (*mojang.SignedTexturesResponse, error) { + args := m.Called(uuid) + var result *mojang.SignedTexturesResponse + if casted, ok := args.Get(0).(*mojang.SignedTexturesResponse); ok { + result = casted + } + + return result, args.Error(1) +} + +type mockStorage struct { + mock.Mock +} + +func (m *mockStorage) GetUuid(username string) (string, error) { + args := m.Called(username) + return args.String(0), args.Error(1) +} + +func (m *mockStorage) StoreUuid(username string, uuid string) error { + args := m.Called(username, uuid) + return args.Error(0) +} + +func (m *mockStorage) GetTextures(uuid string) (*mojang.SignedTexturesResponse, error) { + args := m.Called(uuid) + var result *mojang.SignedTexturesResponse + if casted, ok := args.Get(0).(*mojang.SignedTexturesResponse); ok { + result = casted + } + + return result, args.Error(1) +} + +func (m *mockStorage) StoreTextures(uuid string, textures *mojang.SignedTexturesResponse) { + m.Called(uuid, textures) +} + +type providerTestSuite struct { + suite.Suite + Provider *Provider + UuidsProvider *mockUuidsProvider + TexturesProvider *mockTexturesProvider + Storage *mockStorage + Logger *mocks.WdMock +} + +func (suite *providerTestSuite) SetupTest() { + suite.UuidsProvider = &mockUuidsProvider{} + suite.TexturesProvider = &mockTexturesProvider{} + suite.Storage = &mockStorage{} + suite.Logger = &mocks.WdMock{} + + suite.Provider = &Provider{ + UuidsProvider: suite.UuidsProvider, + TexturesProvider: suite.TexturesProvider, + Storage: suite.Storage, + Logger: suite.Logger, + } +} + +func (suite *providerTestSuite) TearDownTest() { + // time.Sleep(10 * time.Millisecond) // Add delay to let finish all goroutines before assert mocks calls + suite.UuidsProvider.AssertExpectations(suite.T()) + suite.TexturesProvider.AssertExpectations(suite.T()) + suite.Storage.AssertExpectations(suite.T()) + suite.Logger.AssertExpectations(suite.T()) +} + +func TestProvider(t *testing.T) { + suite.Run(t, new(providerTestSuite)) +} + +func (suite *providerTestSuite) TestGetForUsernameWithoutAnyCache() { + expectedResult := &mojang.SignedTexturesResponse{Id: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Name: "username"} + + suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Once() + suite.Logger.On("IncCounter", "mojang_textures.usernames.uuid_hit", int64(1)).Once() + suite.Logger.On("IncCounter", "mojang_textures.usernames.textures_hit", int64(1)).Once() + suite.Logger.On("RecordTimer", "mojang_textures.result_time", mock.Anything).Once() + + suite.Storage.On("GetUuid", "username").Once().Return("", &ValueNotFound{}) + suite.Storage.On("StoreUuid", "username", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").Once().Return(nil) + suite.Storage.On("StoreTextures", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", expectedResult).Once() + + suite.UuidsProvider.On("GetUuid", "username").Once().Return(&mojang.ProfileInfo{ + Id: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Name: "username", + }, nil) + suite.TexturesProvider.On("GetTextures", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").Once().Return(expectedResult, nil) + + result, err := suite.Provider.GetForUsername("username") + + suite.Assert().Nil(err) + suite.Assert().Equal(expectedResult, result) +} + +func (suite *providerTestSuite) TestGetForUsernameWithCachedUuid() { + expectedResult := &mojang.SignedTexturesResponse{Id: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Name: "username"} + + suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Once() + suite.Logger.On("IncCounter", "mojang_textures.usernames.cache_hit", int64(1)).Once() + suite.Logger.On("IncCounter", "mojang_textures.usernames.textures_hit", int64(1)).Once() + suite.Logger.On("RecordTimer", "mojang_textures.result_time", mock.Anything).Once() + + suite.Storage.On("GetUuid", "username").Once().Return("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", nil) + suite.Storage.On("GetTextures", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").Once().Return(nil, &ValueNotFound{}) + suite.Storage.On("StoreTextures", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", expectedResult).Once() + + suite.TexturesProvider.On("GetTextures", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").Return(expectedResult, nil) + + result, err := suite.Provider.GetForUsername("username") + + suite.Assert().Nil(err) + suite.Assert().Equal(expectedResult, result) +} + +func (suite *providerTestSuite) TestGetForUsernameWithFullyCachedResult() { + expectedResult := &mojang.SignedTexturesResponse{Id: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Name: "username"} + + suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Once() + suite.Logger.On("IncCounter", "mojang_textures.usernames.cache_hit", int64(1)).Once() + suite.Logger.On("IncCounter", "mojang_textures.textures.cache_hit", int64(1)).Once() + + suite.Storage.On("GetUuid", "username").Once().Return("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", nil) + suite.Storage.On("GetTextures", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").Once().Return(expectedResult, nil) + + result, err := suite.Provider.GetForUsername("username") + + suite.Assert().Nil(err) + suite.Assert().Equal(expectedResult, result) +} + +func (suite *providerTestSuite) TestGetForUsernameWithCachedUnknownUuid() { + suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Once() + suite.Logger.On("IncCounter", "mojang_textures.usernames.cache_hit_nil", int64(1)).Once() + + suite.Storage.On("GetUuid", "username").Once().Return("", nil) + + result, err := suite.Provider.GetForUsername("username") + + suite.Assert().Nil(result) + suite.Assert().Nil(err) +} + +func (suite *providerTestSuite) TestGetForUsernameWhichHasNoMojangAccount() { + suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Once() + suite.Logger.On("IncCounter", "mojang_textures.usernames.uuid_miss", int64(1)).Once() + suite.Logger.On("RecordTimer", "mojang_textures.result_time", mock.Anything).Once() + + suite.Storage.On("GetUuid", "username").Once().Return("", &ValueNotFound{}) + suite.Storage.On("StoreUuid", "username", "").Once().Return(nil) + + suite.UuidsProvider.On("GetUuid", "username").Once().Return(nil, nil) + + result, err := suite.Provider.GetForUsername("username") + + suite.Assert().Nil(err) + suite.Assert().Nil(result) +} + +func (suite *providerTestSuite) TestGetForUsernameWhichHasMojangAccountButHasNoMojangSkin() { + var expectedResult *mojang.SignedTexturesResponse + + suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Once() + suite.Logger.On("IncCounter", "mojang_textures.usernames.uuid_hit", int64(1)).Once() + suite.Logger.On("IncCounter", "mojang_textures.usernames.textures_miss", int64(1)).Once() + suite.Logger.On("RecordTimer", "mojang_textures.result_time", mock.Anything).Once() + + suite.Storage.On("GetUuid", "username").Once().Return("", &ValueNotFound{}) + suite.Storage.On("StoreUuid", "username", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").Once().Return(nil) + suite.Storage.On("StoreTextures", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", expectedResult).Once() + + suite.UuidsProvider.On("GetUuid", "username").Once().Return(&mojang.ProfileInfo{ + Id: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Name: "username", + }, nil) + suite.TexturesProvider.On("GetTextures", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").Once().Return(expectedResult, nil) + + result, err := suite.Provider.GetForUsername("username") + + suite.Assert().Equal(expectedResult, result) + suite.Assert().Nil(err) +} + +func (suite *providerTestSuite) TestGetForTheSameUsernames() { + expectedResult := &mojang.SignedTexturesResponse{Id: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Name: "username"} + + suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Twice() + suite.Logger.On("IncCounter", "mojang_textures.already_scheduled", int64(1)).Once() + suite.Logger.On("IncCounter", "mojang_textures.usernames.uuid_hit", int64(1)).Once() + suite.Logger.On("IncCounter", "mojang_textures.usernames.textures_hit", int64(1)).Once() + suite.Logger.On("RecordTimer", "mojang_textures.result_time", mock.Anything).Once() + + suite.Storage.On("GetUuid", "username").Twice().Return("", &ValueNotFound{}) + suite.Storage.On("StoreUuid", "username", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").Once().Return(nil) + suite.Storage.On("StoreTextures", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", expectedResult).Once() + + // If possible, than remove this .After call + suite.UuidsProvider.On("GetUuid", "username").Once().After(time.Millisecond).Return(&mojang.ProfileInfo{ + Id: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Name: "username", + }, nil) + suite.TexturesProvider.On("GetTextures", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").Once().Return(expectedResult, nil) + + results := make([]*mojang.SignedTexturesResponse, 2) + var wg sync.WaitGroup + for i := 0; i < 2; i++ { + wg.Add(1) + go func(i int) { + textures, _ := suite.Provider.GetForUsername("username") + results[i] = textures + wg.Done() + }(i) + } + wg.Wait() + + suite.Assert().Equal(expectedResult, results[0]) + suite.Assert().Equal(expectedResult, results[1]) +} + +func (suite *providerTestSuite) TestGetForNotAllowedMojangUsername() { + suite.Logger.On("IncCounter", "mojang_textures.invalid_username", int64(1)).Once() + + result, err := suite.Provider.GetForUsername("Not allowed") + suite.Assert().Error(err, "invalid username") + suite.Assert().Nil(result) +} + +type timeoutError struct { +} + +func (*timeoutError) Error() string { return "timeout error" } +func (*timeoutError) Timeout() bool { return true } +func (*timeoutError) Temporary() bool { return false } + +var expectedErrors = []error{ + &mojang.BadRequestError{}, + &mojang.ForbiddenError{}, + &mojang.TooManyRequestsError{}, + &mojang.ServerError{}, + &timeoutError{}, + &url.Error{Op: "GET", URL: "http://localhost"}, + &net.OpError{Op: "read"}, + &net.OpError{Op: "dial"}, + syscall.ECONNREFUSED, +} + +func (suite *providerTestSuite) TestShouldNotLogErrorWhenExpectedErrorReturnedFromUsernameToUuidRequest() { + suite.Logger.On("IncCounter", mock.Anything, mock.Anything) + suite.Logger.On("RecordTimer", mock.Anything, mock.Anything) + suite.Logger.On("Debug", ":name: Got response error :err", mock.Anything, mock.Anything).Times(len(expectedErrors)) + suite.Logger.On("Warning", ":name: Got 400 Bad Request :err", mock.Anything, mock.Anything).Once() + suite.Logger.On("Warning", ":name: Got 403 Forbidden :err", mock.Anything, mock.Anything).Once() + suite.Logger.On("Warning", ":name: Got 429 Too Many Requests :err", mock.Anything, mock.Anything).Once() + + suite.Storage.On("GetUuid", "username").Return("", &ValueNotFound{}) + + for _, err := range expectedErrors { + suite.UuidsProvider.On("GetUuid", "username").Once().Return(nil, err) + + result, err := suite.Provider.GetForUsername("username") + suite.Assert().Nil(result) + suite.Assert().NotNil(err) + suite.UuidsProvider.AssertExpectations(suite.T()) + suite.UuidsProvider.ExpectedCalls = nil // https://github.com/stretchr/testify/issues/558#issuecomment-372112364 + } +} + +func (suite *providerTestSuite) TestShouldLogEmergencyOnUnexpectedErrorReturnedFromUsernameToUuidRequest() { + suite.Logger.On("IncCounter", mock.Anything, mock.Anything) + suite.Logger.On("RecordTimer", mock.Anything, mock.Anything) + suite.Logger.On("Debug", ":name: Got response error :err", mock.Anything, mock.Anything).Once() + suite.Logger.On("Emergency", ":name: Unknown Mojang response error: :err", mock.Anything, mock.Anything).Once() + + suite.Storage.On("GetUuid", "username").Return("", &ValueNotFound{}) + + suite.UuidsProvider.On("GetUuid", "username").Once().Return(nil, errors.New("unexpected error")) + + result, err := suite.Provider.GetForUsername("username") + suite.Assert().Nil(result) + suite.Assert().NotNil(err) +} + +func (suite *providerTestSuite) TestShouldNotLogErrorWhenExpectedErrorReturnedFromUuidToTexturesRequest() { + suite.Logger.On("IncCounter", mock.Anything, mock.Anything) + suite.Logger.On("RecordTimer", mock.Anything, mock.Anything) + suite.Logger.On("Debug", ":name: Got response error :err", mock.Anything, mock.Anything).Times(len(expectedErrors)) + suite.Logger.On("Warning", ":name: Got 400 Bad Request :err", mock.Anything, mock.Anything).Once() + suite.Logger.On("Warning", ":name: Got 403 Forbidden :err", mock.Anything, mock.Anything).Once() + suite.Logger.On("Warning", ":name: Got 429 Too Many Requests :err", mock.Anything, mock.Anything).Once() + + suite.Storage.On("GetUuid", "username").Return("", &ValueNotFound{}) + suite.Storage.On("StoreUuid", "username", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").Return(nil) + // suite.Storage.On("GetTextures", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").Return(nil, &ValueNotFound{}) + // suite.Storage.On("StoreTextures", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", (*mojang.SignedTexturesResponse)(nil)) + + for _, err := range expectedErrors { + suite.UuidsProvider.On("GetUuid", "username").Once().Return(&mojang.ProfileInfo{ + Id: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Name: "username", + }, nil) + suite.TexturesProvider.On("GetTextures", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").Once().Return(nil, err) + + result, err := suite.Provider.GetForUsername("username") + suite.Assert().Nil(result) + suite.Assert().NotNil(err) + suite.UuidsProvider.AssertExpectations(suite.T()) + suite.TexturesProvider.AssertExpectations(suite.T()) + suite.UuidsProvider.ExpectedCalls = nil // https://github.com/stretchr/testify/issues/558#issuecomment-372112364 + suite.TexturesProvider.ExpectedCalls = nil // https://github.com/stretchr/testify/issues/558#issuecomment-372112364 + } +} + +func (suite *providerTestSuite) TestShouldLogEmergencyOnUnexpectedErrorReturnedFromUuidToTexturesRequest() { + suite.Logger.On("IncCounter", mock.Anything, mock.Anything) + suite.Logger.On("RecordTimer", mock.Anything, mock.Anything) + suite.Logger.On("Debug", ":name: Got response error :err", mock.Anything, mock.Anything).Once() + suite.Logger.On("Emergency", ":name: Unknown Mojang response error: :err", mock.Anything, mock.Anything).Once() + + suite.Storage.On("GetUuid", "username").Return("", &ValueNotFound{}) + suite.Storage.On("StoreUuid", "username", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").Return(nil) + + suite.UuidsProvider.On("GetUuid", "username").Once().Return(&mojang.ProfileInfo{ + Id: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Name: "username", + }, nil) + suite.TexturesProvider.On("GetTextures", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").Once().Return(nil, errors.New("unexpected error")) + + result, err := suite.Provider.GetForUsername("username") + suite.Assert().Nil(result) + suite.Assert().NotNil(err) +} diff --git a/mojangtextures/storage.go b/mojangtextures/storage.go new file mode 100644 index 0000000..d73249a --- /dev/null +++ b/mojangtextures/storage.go @@ -0,0 +1,61 @@ +package mojangtextures + +import ( + "github.com/elyby/chrly/api/mojang" +) + +// UuidsStorage is a key-value storage of Mojang usernames pairs to its UUIDs, +// used to reduce the load on the account information queue +type UuidsStorage interface { + // Since only primitive types are used in this method, you should return a special error ValueNotFound + // to return the information that no error has occurred and username does not have uuid + GetUuid(username string) (string, error) + // An empty uuid value can be passed if the corresponding account has not been found + StoreUuid(username string, uuid string) error +} + +// TexturesStorage is a Mojang's textures storage, used as a values cache to avoid 429 errors +type TexturesStorage interface { + // Error should not have nil value only if the repository failed to determine if there are any textures + // for this uuid or not at all. If there is information about the absence of textures, nil nil should be returned + GetTextures(uuid string) (*mojang.SignedTexturesResponse, error) + // The nil value can be passed when there are no textures for the corresponding uuid and we know about it + StoreTextures(uuid string, textures *mojang.SignedTexturesResponse) +} + +type Storage interface { + UuidsStorage + TexturesStorage +} + +// SeparatedStorage allows you to use separate storage engines to satisfy +// the Storage interface +type SeparatedStorage struct { + UuidsStorage + TexturesStorage +} + +func (s *SeparatedStorage) GetUuid(username string) (string, error) { + return s.UuidsStorage.GetUuid(username) +} + +func (s *SeparatedStorage) StoreUuid(username string, uuid string) error { + return s.UuidsStorage.StoreUuid(username, uuid) +} + +func (s *SeparatedStorage) GetTextures(uuid string) (*mojang.SignedTexturesResponse, error) { + return s.TexturesStorage.GetTextures(uuid) +} + +func (s *SeparatedStorage) StoreTextures(uuid string, textures *mojang.SignedTexturesResponse) { + s.TexturesStorage.StoreTextures(uuid, textures) +} + +// This error can be used to indicate, that requested +// value doesn't exists in the storage +type ValueNotFound struct { +} + +func (*ValueNotFound) Error() string { + return "value not found in the storage" +} diff --git a/api/mojang/queue/storage_test.go b/mojangtextures/storage_test.go similarity index 91% rename from api/mojang/queue/storage_test.go rename to mojangtextures/storage_test.go index b9ee64c..1b82349 100644 --- a/api/mojang/queue/storage_test.go +++ b/mojangtextures/storage_test.go @@ -1,4 +1,4 @@ -package queue +package mojangtextures import ( "github.com/elyby/chrly/api/mojang" @@ -41,11 +41,11 @@ func (m *texturesStorageMock) StoreTextures(uuid string, textures *mojang.Signed } func TestSplittedStorage(t *testing.T) { - createMockedStorage := func() (*SplittedStorage, *uuidsStorageMock, *texturesStorageMock) { + createMockedStorage := func() (*SeparatedStorage, *uuidsStorageMock, *texturesStorageMock) { uuidsStorage := &uuidsStorageMock{} texturesStorage := &texturesStorageMock{} - return &SplittedStorage{uuidsStorage, texturesStorage}, uuidsStorage, texturesStorage + return &SeparatedStorage{uuidsStorage, texturesStorage}, uuidsStorage, texturesStorage } t.Run("GetUuid", func(t *testing.T) { diff --git a/tests/mojang_textures_queue_mock.go b/tests/mojang_textures_queue_mock.go deleted file mode 100644 index 0494243..0000000 --- a/tests/mojang_textures_queue_mock.go +++ /dev/null @@ -1,33 +0,0 @@ -package tests - -import ( - "github.com/elyby/chrly/api/mojang" - - "github.com/stretchr/testify/mock" -) - -type MojangTexturesQueueMock struct { - mock.Mock -} - -func (m *MojangTexturesQueueMock) GetTexturesForUsername(username string) chan *mojang.SignedTexturesResponse { - args := m.Called(username) - result := make(chan *mojang.SignedTexturesResponse) - arg := args.Get(0) - switch arg.(type) { - case *mojang.SignedTexturesResponse: - go func() { - result <- arg.(*mojang.SignedTexturesResponse) - }() - case chan *mojang.SignedTexturesResponse: - return arg.(chan *mojang.SignedTexturesResponse) - case nil: - go func() { - result <- nil - }() - default: - panic("unsupported return value") - } - - return result -}