From 0644dfe0218fdb848441f0fc85283011e0e9d3e3 Mon Sep 17 00:00:00 2001 From: ErickSkrauch Date: Thu, 21 Nov 2019 01:33:13 +0300 Subject: [PATCH 01/12] Completely rework mojang textures queue implementation, split it across separate data providers --- CHANGELOG.md | 18 +- api/mojang/queue/broadcast.go | 53 -- api/mojang/queue/broadcast_test.go | 75 --- api/mojang/queue/jobs_structure.go | 56 -- api/mojang/queue/jobs_structure_test.go | 47 -- api/mojang/queue/queue.go | 221 -------- api/mojang/queue/queue_test.go | 525 ------------------ api/mojang/queue/storage.go | 53 -- cmd/serve.go | 30 +- db/factory.go | 4 +- db/filesystem.go | 4 +- db/redis.go | 8 +- http/cape.go | 4 +- http/cape_test.go | 22 +- http/http.go | 10 +- http/http_test.go | 70 ++- http/signed_textures.go | 5 +- http/signed_textures_test.go | 2 +- http/skin.go | 4 +- http/skin_test.go | 4 +- http/textures.go | 4 +- http/textures_test.go | 4 +- interfaces/repositories.go | 4 +- mojangtextures/batch_uuids_provider.go | 133 +++++ mojangtextures/batch_uuids_provider_test.go | 285 ++++++++++ .../in_memory_textures_storage.go | 39 +- .../in_memory_textures_storage_test.go | 30 +- .../mojang_api_textures_provider.go | 25 + .../mojang_api_textures_provider_test.go | 82 +++ mojangtextures/mojang_textures.go | 225 ++++++++ mojangtextures/mojang_textures_test.go | 439 +++++++++++++++ mojangtextures/storage.go | 61 ++ .../queue => mojangtextures}/storage_test.go | 6 +- tests/mojang_textures_queue_mock.go | 33 -- 34 files changed, 1405 insertions(+), 1180 deletions(-) delete mode 100644 api/mojang/queue/broadcast.go delete mode 100644 api/mojang/queue/broadcast_test.go delete mode 100644 api/mojang/queue/jobs_structure.go delete mode 100644 api/mojang/queue/jobs_structure_test.go delete mode 100644 api/mojang/queue/queue.go delete mode 100644 api/mojang/queue/queue_test.go delete mode 100644 api/mojang/queue/storage.go create mode 100644 mojangtextures/batch_uuids_provider.go create mode 100644 mojangtextures/batch_uuids_provider_test.go rename {api/mojang/queue => mojangtextures}/in_memory_textures_storage.go (62%) rename {api/mojang/queue => mojangtextures}/in_memory_textures_storage_test.go (85%) create mode 100644 mojangtextures/mojang_api_textures_provider.go create mode 100644 mojangtextures/mojang_api_textures_provider_test.go create mode 100644 mojangtextures/mojang_textures.go create mode 100644 mojangtextures/mojang_textures_test.go create mode 100644 mojangtextures/storage.go rename {api/mojang/queue => mojangtextures}/storage_test.go (91%) delete mode 100644 tests/mojang_textures_queue_mock.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 4753aa5..450d8e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,14 +5,26 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] - xxxx-xx-xx +### Added +- New StatsD metrics: + - Counters: + - `ely.skinsystem.{hostname}.app.mojang_textures.usernames.textures_hit` + - `ely.skinsystem.{hostname}.app.mojang_textures.usernames.textures_miss` + +### Fixed +- `ely.skinsystem.{hostname}.app.mojang_textures.usernames.iteration_size` and + `ely.skinsystem.{hostname}.app.mojang_textures.usernames.queue_size` are now updated even if the queue is empty. + +### Changed +- Event `ely.skinsystem.{hostname}.app.mojang_textures.already_in_queue` has been renamed into `ely.skinsystem.{hostname}.app.mojang_textures.already_scheduled`. ## [4.3.0] - 2019-11-08 ### Added -- 403 Forbidden errors from the Mojang's API are now logged -- `QUEUE_LOOP_DELAY` configuration param to adjust Mojang's textures queue performance +- 403 Forbidden errors from the Mojang's API are now logged. +- `QUEUE_LOOP_DELAY` configuration param to adjust Mojang's textures queue performance. ### Changed -- Mojang's textures queue loop is now has an iteration delay of 2.5 seconds (was 1) +- Mojang's textures queue loop is now has an iteration delay of 2.5 seconds (was 1). - Bumped Go version to 1.13. ## [4.2.3] - 2019-10-03 diff --git a/api/mojang/queue/broadcast.go b/api/mojang/queue/broadcast.go deleted file mode 100644 index 56c3b5e..0000000 --- a/api/mojang/queue/broadcast.go +++ /dev/null @@ -1,53 +0,0 @@ -package queue - -import ( - "sync" - - "github.com/elyby/chrly/api/mojang" -) - -type broadcastMap struct { - lock sync.Mutex - listeners map[string][]chan *mojang.SignedTexturesResponse -} - -func newBroadcaster() *broadcastMap { - return &broadcastMap{ - listeners: make(map[string][]chan *mojang.SignedTexturesResponse), - } -} - -// Returns a boolean value, which will be true if the username passed didn't exist before -func (c *broadcastMap) AddListener(username string, resultChan chan *mojang.SignedTexturesResponse) bool { - c.lock.Lock() - defer c.lock.Unlock() - - val, alreadyHasSource := c.listeners[username] - if alreadyHasSource { - c.listeners[username] = append(val, resultChan) - return false - } - - c.listeners[username] = []chan *mojang.SignedTexturesResponse{resultChan} - - return true -} - -func (c *broadcastMap) BroadcastAndRemove(username string, result *mojang.SignedTexturesResponse) { - c.lock.Lock() - defer c.lock.Unlock() - - val, ok := c.listeners[username] - if !ok { - return - } - - for _, channel := range val { - go func(channel chan *mojang.SignedTexturesResponse) { - channel <- result - close(channel) - }(channel) - } - - delete(c.listeners, username) -} diff --git a/api/mojang/queue/broadcast_test.go b/api/mojang/queue/broadcast_test.go deleted file mode 100644 index 910ea8b..0000000 --- a/api/mojang/queue/broadcast_test.go +++ /dev/null @@ -1,75 +0,0 @@ -package queue - -import ( - "github.com/elyby/chrly/api/mojang" - - testify "github.com/stretchr/testify/assert" - "testing" -) - -func TestBroadcastMap_GetOrAppend(t *testing.T) { - t.Run("first call when username didn't exist before should return true", func(t *testing.T) { - assert := testify.New(t) - - broadcaster := newBroadcaster() - channel := make(chan *mojang.SignedTexturesResponse) - isFirstListener := broadcaster.AddListener("mock", channel) - - assert.True(isFirstListener) - listeners, ok := broadcaster.listeners["mock"] - assert.True(ok) - assert.Len(listeners, 1) - assert.Equal(channel, listeners[0]) - }) - - t.Run("subsequent calls should return false", func(t *testing.T) { - assert := testify.New(t) - - broadcaster := newBroadcaster() - channel1 := make(chan *mojang.SignedTexturesResponse) - isFirstListener := broadcaster.AddListener("mock", channel1) - - assert.True(isFirstListener) - - channel2 := make(chan *mojang.SignedTexturesResponse) - isFirstListener = broadcaster.AddListener("mock", channel2) - - assert.False(isFirstListener) - - channel3 := make(chan *mojang.SignedTexturesResponse) - isFirstListener = broadcaster.AddListener("mock", channel3) - - assert.False(isFirstListener) - }) -} - -func TestBroadcastMap_BroadcastAndRemove(t *testing.T) { - t.Run("should broadcast to all listeners and remove the key", func(t *testing.T) { - assert := testify.New(t) - - broadcaster := newBroadcaster() - channel1 := make(chan *mojang.SignedTexturesResponse) - channel2 := make(chan *mojang.SignedTexturesResponse) - broadcaster.AddListener("mock", channel1) - broadcaster.AddListener("mock", channel2) - - result := &mojang.SignedTexturesResponse{Id: "mockUuid"} - broadcaster.BroadcastAndRemove("mock", result) - - assert.Equal(result, <-channel1) - assert.Equal(result, <-channel2) - - channel3 := make(chan *mojang.SignedTexturesResponse) - isFirstListener := broadcaster.AddListener("mock", channel3) - assert.True(isFirstListener) - }) - - t.Run("call on not exists username", func(t *testing.T) { - assert := testify.New(t) - - assert.NotPanics(func() { - broadcaster := newBroadcaster() - broadcaster.BroadcastAndRemove("mock", &mojang.SignedTexturesResponse{}) - }) - }) -} diff --git a/api/mojang/queue/jobs_structure.go b/api/mojang/queue/jobs_structure.go deleted file mode 100644 index 987be3d..0000000 --- a/api/mojang/queue/jobs_structure.go +++ /dev/null @@ -1,56 +0,0 @@ -// Based on the implementation from https://flaviocopes.com/golang-data-structure-queue/ - -package queue - -import ( - "sync" - - "github.com/elyby/chrly/api/mojang" -) - -type jobItem struct { - Username string - RespondTo chan *mojang.SignedTexturesResponse -} - -type jobsQueue struct { - lock sync.Mutex - items []*jobItem -} - -func (s *jobsQueue) New() *jobsQueue { - s.items = []*jobItem{} - return s -} - -func (s *jobsQueue) Enqueue(t *jobItem) { - s.lock.Lock() - defer s.lock.Unlock() - - s.items = append(s.items, t) -} - -func (s *jobsQueue) Dequeue(n int) []*jobItem { - s.lock.Lock() - defer s.lock.Unlock() - - if n > s.Size() { - n = s.Size() - } - - items := s.items[0:n] - s.items = s.items[n:len(s.items)] - - return items -} - -func (s *jobsQueue) IsEmpty() bool { - s.lock.Lock() - defer s.lock.Unlock() - - return len(s.items) == 0 -} - -func (s *jobsQueue) Size() int { - return len(s.items) -} diff --git a/api/mojang/queue/jobs_structure_test.go b/api/mojang/queue/jobs_structure_test.go deleted file mode 100644 index b179d26..0000000 --- a/api/mojang/queue/jobs_structure_test.go +++ /dev/null @@ -1,47 +0,0 @@ -package queue - -import ( - "testing" - - testify "github.com/stretchr/testify/assert" -) - -func TestEnqueue(t *testing.T) { - assert := testify.New(t) - - s := createQueue() - s.Enqueue(&jobItem{Username: "username1"}) - s.Enqueue(&jobItem{Username: "username2"}) - s.Enqueue(&jobItem{Username: "username3"}) - - assert.Equal(3, s.Size()) -} - -func TestDequeueN(t *testing.T) { - assert := testify.New(t) - - s := createQueue() - s.Enqueue(&jobItem{Username: "username1"}) - s.Enqueue(&jobItem{Username: "username2"}) - s.Enqueue(&jobItem{Username: "username3"}) - s.Enqueue(&jobItem{Username: "username4"}) - - items := s.Dequeue(2) - assert.Len(items, 2) - assert.Equal("username1", items[0].Username) - assert.Equal("username2", items[1].Username) - assert.Equal(2, s.Size()) - - items = s.Dequeue(40) - assert.Len(items, 2) - assert.Equal("username3", items[0].Username) - assert.Equal("username4", items[1].Username) - assert.True(s.IsEmpty()) -} - -func createQueue() *jobsQueue { - queue := &jobsQueue{} - queue.New() - - return queue -} diff --git a/api/mojang/queue/queue.go b/api/mojang/queue/queue.go deleted file mode 100644 index 7034d0b..0000000 --- a/api/mojang/queue/queue.go +++ /dev/null @@ -1,221 +0,0 @@ -package queue - -import ( - "net" - "net/url" - "regexp" - "strings" - "sync" - "syscall" - "time" - - "github.com/mono83/slf/wd" - - "github.com/elyby/chrly/api/mojang" -) - -var UuidsQueueIterationDelay = 2*time.Second + 500*time.Millisecond - -var usernamesToUuids = mojang.UsernamesToUuids -var uuidToTextures = mojang.UuidToTextures -var forever = func() bool { - return true -} - -// https://help.mojang.com/customer/portal/articles/928638 -var allowedUsernamesRegex = regexp.MustCompile(`^[\w_]{3,16}$`) - -type JobsQueue struct { - Storage Storage - Logger wd.Watchdog - - onFirstCall sync.Once - queue jobsQueue - broadcast *broadcastMap -} - -func (ctx *JobsQueue) GetTexturesForUsername(username string) chan *mojang.SignedTexturesResponse { - // TODO: convert username to lower case - ctx.onFirstCall.Do(func() { - ctx.queue.New() - ctx.broadcast = newBroadcaster() - ctx.startQueue() - }) - - responseChan := make(chan *mojang.SignedTexturesResponse) - if !allowedUsernamesRegex.MatchString(username) { - ctx.Logger.IncCounter("mojang_textures.invalid_username", 1) - go func() { - responseChan <- nil - close(responseChan) - }() - - return responseChan - } - - ctx.Logger.IncCounter("mojang_textures.request", 1) - - uuid, err := ctx.Storage.GetUuid(username) - if err == nil && uuid == "" { - ctx.Logger.IncCounter("mojang_textures.usernames.cache_hit_nil", 1) - - go func() { - responseChan <- nil - close(responseChan) - }() - - return responseChan - } - - isFirstListener := ctx.broadcast.AddListener(username, responseChan) - if isFirstListener { - start := time.Now() - // TODO: respond nil if processing takes more than 5 seconds - - resultChan := make(chan *mojang.SignedTexturesResponse) - if uuid == "" { - ctx.Logger.IncCounter("mojang_textures.usernames.queued", 1) - ctx.queue.Enqueue(&jobItem{username, resultChan}) - } else { - ctx.Logger.IncCounter("mojang_textures.usernames.cache_hit", 1) - go func() { - resultChan <- ctx.getTextures(uuid) - }() - } - - go func() { - result := <-resultChan - close(resultChan) - ctx.broadcast.BroadcastAndRemove(username, result) - ctx.Logger.RecordTimer("mojang_textures.result_time", time.Since(start)) - }() - } else { - ctx.Logger.IncCounter("mojang_textures.already_in_queue", 1) - } - - return responseChan -} - -func (ctx *JobsQueue) startQueue() { - go func() { - time.Sleep(UuidsQueueIterationDelay) - for forever() { - start := time.Now() - ctx.queueRound() - elapsed := time.Since(start) - ctx.Logger.RecordTimer("mojang_textures.usernames.round_time", elapsed) - time.Sleep(UuidsQueueIterationDelay) - } - }() -} - -func (ctx *JobsQueue) queueRound() { - if ctx.queue.IsEmpty() { - return - } - - queueSize := ctx.queue.Size() - jobs := ctx.queue.Dequeue(10) - ctx.Logger.UpdateGauge("mojang_textures.usernames.iteration_size", int64(len(jobs))) - ctx.Logger.UpdateGauge("mojang_textures.usernames.queue_size", int64(queueSize-len(jobs))) - var usernames []string - for _, job := range jobs { - usernames = append(usernames, job.Username) - } - - profiles, err := usernamesToUuids(usernames) - if err != nil { - ctx.handleResponseError(err, "usernames") - for _, job := range jobs { - job.RespondTo <- nil - } - - return - } - - for _, job := range jobs { - go func(job *jobItem) { - var uuid string - // The profiles in the response are not ordered, so we must search each username over full array - for _, profile := range profiles { - if strings.EqualFold(job.Username, profile.Name) { - uuid = profile.Id - break - } - } - - _ = ctx.Storage.StoreUuid(job.Username, uuid) - if uuid == "" { - job.RespondTo <- nil - ctx.Logger.IncCounter("mojang_textures.usernames.uuid_miss", 1) - } else { - job.RespondTo <- ctx.getTextures(uuid) - ctx.Logger.IncCounter("mojang_textures.usernames.uuid_hit", 1) - } - }(job) - } -} - -func (ctx *JobsQueue) getTextures(uuid string) *mojang.SignedTexturesResponse { - existsTextures, err := ctx.Storage.GetTextures(uuid) - if err == nil { - ctx.Logger.IncCounter("mojang_textures.textures.cache_hit", 1) - return existsTextures - } - - ctx.Logger.IncCounter("mojang_textures.textures.request", 1) - - start := time.Now() - result, err := uuidToTextures(uuid, true) - ctx.Logger.RecordTimer("mojang_textures.textures.request_time", time.Since(start)) - if err != nil { - ctx.handleResponseError(err, "textures") - } - - // Mojang can respond with an error, but count it as a hit, so store result even if the textures is nil - ctx.Storage.StoreTextures(uuid, result) - - return result -} - -func (ctx *JobsQueue) handleResponseError(err error, threadName string) { - ctx.Logger.Debug(":name: Got response error :err", wd.NameParam(threadName), wd.ErrParam(err)) - - switch err.(type) { - case mojang.ResponseError: - if _, ok := err.(*mojang.BadRequestError); ok { - ctx.Logger.Warning(":name: Got 400 Bad Request :err", wd.NameParam(threadName), wd.ErrParam(err)) - return - } - - if _, ok := err.(*mojang.ForbiddenError); ok { - ctx.Logger.Warning(":name: Got 403 Forbidden :err", wd.NameParam(threadName), wd.ErrParam(err)) - return - } - - if _, ok := err.(*mojang.TooManyRequestsError); ok { - ctx.Logger.Warning(":name: Got 429 Too Many Requests :err", wd.NameParam(threadName), wd.ErrParam(err)) - return - } - - return - case net.Error: - if err.(net.Error).Timeout() { - return - } - - if _, ok := err.(*url.Error); ok { - return - } - - if opErr, ok := err.(*net.OpError); ok && (opErr.Op == "dial" || opErr.Op == "read") { - return - } - - if err == syscall.ECONNREFUSED { - return - } - } - - ctx.Logger.Emergency(":name: Unknown Mojang response error: :err", wd.NameParam(threadName), wd.ErrParam(err)) -} diff --git a/api/mojang/queue/queue_test.go b/api/mojang/queue/queue_test.go deleted file mode 100644 index 30bc3ee..0000000 --- a/api/mojang/queue/queue_test.go +++ /dev/null @@ -1,525 +0,0 @@ -package queue - -import ( - "crypto/rand" - "encoding/base64" - "errors" - "net" - "net/url" - "strings" - "syscall" - "time" - - "github.com/elyby/chrly/api/mojang" - - "testing" - - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/suite" - - mocks "github.com/elyby/chrly/tests" -) - -type mojangApiMocks struct { - mock.Mock -} - -func (o *mojangApiMocks) UsernamesToUuids(usernames []string) ([]*mojang.ProfileInfo, error) { - args := o.Called(usernames) - var result []*mojang.ProfileInfo - if casted, ok := args.Get(0).([]*mojang.ProfileInfo); ok { - result = casted - } - - return result, args.Error(1) -} - -func (o *mojangApiMocks) UuidToTextures(uuid string, signed bool) (*mojang.SignedTexturesResponse, error) { - args := o.Called(uuid, signed) - var result *mojang.SignedTexturesResponse - if casted, ok := args.Get(0).(*mojang.SignedTexturesResponse); ok { - result = casted - } - - return result, args.Error(1) -} - -type mockStorage struct { - mock.Mock -} - -func (m *mockStorage) GetUuid(username string) (string, error) { - args := m.Called(username) - return args.String(0), args.Error(1) -} - -func (m *mockStorage) StoreUuid(username string, uuid string) error { - args := m.Called(username, uuid) - return args.Error(0) -} - -func (m *mockStorage) GetTextures(uuid string) (*mojang.SignedTexturesResponse, error) { - args := m.Called(uuid) - var result *mojang.SignedTexturesResponse - if casted, ok := args.Get(0).(*mojang.SignedTexturesResponse); ok { - result = casted - } - - return result, args.Error(1) -} - -func (m *mockStorage) StoreTextures(uuid string, textures *mojang.SignedTexturesResponse) { - m.Called(uuid, textures) -} - -type queueTestSuite struct { - suite.Suite - Queue *JobsQueue - Storage *mockStorage - MojangApi *mojangApiMocks - Logger *mocks.WdMock - Iterate func() - - iterateChan chan bool - done func() -} - -func (suite *queueTestSuite) SetupSuite() { - UuidsQueueIterationDelay = 0 -} - -func (suite *queueTestSuite) SetupTest() { - suite.Storage = &mockStorage{} - suite.Logger = &mocks.WdMock{} - - suite.Queue = &JobsQueue{Storage: suite.Storage, Logger: suite.Logger} - - suite.iterateChan = make(chan bool) - forever = func() bool { - return <-suite.iterateChan - } - - suite.Iterate = func() { - suite.iterateChan <- true - } - - suite.done = func() { - suite.iterateChan <- false - } - - suite.MojangApi = new(mojangApiMocks) - usernamesToUuids = suite.MojangApi.UsernamesToUuids - uuidToTextures = suite.MojangApi.UuidToTextures -} - -func (suite *queueTestSuite) TearDownTest() { - suite.done() - time.Sleep(10 * time.Millisecond) // Add delay to let finish all goroutines before assert mocks calls - suite.MojangApi.AssertExpectations(suite.T()) - suite.Storage.AssertExpectations(suite.T()) - suite.Logger.AssertExpectations(suite.T()) -} - -func (suite *queueTestSuite) TestReceiveTexturesForOneUsernameWithoutAnyCache() { - expectedResult := &mojang.SignedTexturesResponse{Id: "0d252b7218b648bfb86c2ae476954d32", Name: "maksimkurb"} - - suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Once() - suite.Logger.On("IncCounter", "mojang_textures.usernames.queued", int64(1)).Once() - suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(1)).Once() - suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(0)).Once() - suite.Logger.On("RecordTimer", "mojang_textures.usernames.round_time", mock.Anything) - suite.Logger.On("IncCounter", "mojang_textures.textures.request", int64(1)).Once() - suite.Logger.On("RecordTimer", "mojang_textures.textures.request_time", mock.Anything).Once() - suite.Logger.On("IncCounter", "mojang_textures.usernames.uuid_hit", int64(1)).Once() - suite.Logger.On("RecordTimer", "mojang_textures.result_time", mock.Anything).Once() - - suite.Storage.On("GetUuid", "maksimkurb").Once().Return("", &ValueNotFound{}) - suite.Storage.On("StoreUuid", "maksimkurb", "0d252b7218b648bfb86c2ae476954d32").Once().Return(nil) - suite.Storage.On("GetTextures", "0d252b7218b648bfb86c2ae476954d32").Once().Return(nil, &ValueNotFound{}) - suite.Storage.On("StoreTextures", "0d252b7218b648bfb86c2ae476954d32", expectedResult).Once() - - suite.MojangApi.On("UsernamesToUuids", []string{"maksimkurb"}).Once().Return([]*mojang.ProfileInfo{ - {Id: "0d252b7218b648bfb86c2ae476954d32", Name: "maksimkurb"}, - }, nil) - suite.MojangApi.On("UuidToTextures", "0d252b7218b648bfb86c2ae476954d32", true).Once().Return(expectedResult, nil) - - resultChan := suite.Queue.GetTexturesForUsername("maksimkurb") - - suite.Iterate() - - result := <-resultChan - suite.Assert().Equal(expectedResult, result) -} - -func (suite *queueTestSuite) TestReceiveTexturesForFewUsernamesWithoutAnyCache() { - expectedResult1 := &mojang.SignedTexturesResponse{Id: "0d252b7218b648bfb86c2ae476954d32", Name: "maksimkurb"} - expectedResult2 := &mojang.SignedTexturesResponse{Id: "4566e69fc90748ee8d71d7ba5aa00d20", Name: "Thinkofdeath"} - - suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Twice() - suite.Logger.On("IncCounter", "mojang_textures.usernames.queued", int64(1)).Twice() - suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(2)).Once() - suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(0)).Once() - suite.Logger.On("RecordTimer", "mojang_textures.usernames.round_time", mock.Anything) - suite.Logger.On("IncCounter", "mojang_textures.textures.request", int64(1)).Twice() - suite.Logger.On("RecordTimer", "mojang_textures.textures.request_time", mock.Anything).Twice() - suite.Logger.On("IncCounter", "mojang_textures.usernames.uuid_hit", int64(1)).Twice() - suite.Logger.On("RecordTimer", "mojang_textures.result_time", mock.Anything).Twice() - - suite.Storage.On("GetUuid", "maksimkurb").Once().Return("", &ValueNotFound{}) - suite.Storage.On("GetUuid", "Thinkofdeath").Once().Return("", &ValueNotFound{}) - suite.Storage.On("StoreUuid", "maksimkurb", "0d252b7218b648bfb86c2ae476954d32").Once().Return(nil) - suite.Storage.On("StoreUuid", "Thinkofdeath", "4566e69fc90748ee8d71d7ba5aa00d20").Once().Return(nil) - suite.Storage.On("GetTextures", "0d252b7218b648bfb86c2ae476954d32").Once().Return(nil, &ValueNotFound{}) - suite.Storage.On("GetTextures", "4566e69fc90748ee8d71d7ba5aa00d20").Once().Return(nil, &ValueNotFound{}) - suite.Storage.On("StoreTextures", "0d252b7218b648bfb86c2ae476954d32", expectedResult1).Once() - suite.Storage.On("StoreTextures", "4566e69fc90748ee8d71d7ba5aa00d20", expectedResult2).Once() - - suite.MojangApi.On("UsernamesToUuids", []string{"maksimkurb", "Thinkofdeath"}).Once().Return([]*mojang.ProfileInfo{ - {Id: "0d252b7218b648bfb86c2ae476954d32", Name: "maksimkurb"}, - {Id: "4566e69fc90748ee8d71d7ba5aa00d20", Name: "Thinkofdeath"}, - }, nil) - suite.MojangApi.On("UuidToTextures", "0d252b7218b648bfb86c2ae476954d32", true).Once().Return(expectedResult1, nil) - suite.MojangApi.On("UuidToTextures", "4566e69fc90748ee8d71d7ba5aa00d20", true).Once().Return(expectedResult2, nil) - - resultChan1 := suite.Queue.GetTexturesForUsername("maksimkurb") - resultChan2 := suite.Queue.GetTexturesForUsername("Thinkofdeath") - - suite.Iterate() - - suite.Assert().Equal(expectedResult1, <-resultChan1) - suite.Assert().Equal(expectedResult2, <-resultChan2) -} - -func (suite *queueTestSuite) TestReceiveTexturesForUsernameWithCachedUuid() { - expectedResult := &mojang.SignedTexturesResponse{Id: "0d252b7218b648bfb86c2ae476954d32", Name: "maksimkurb"} - - suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Once() - suite.Logger.On("IncCounter", "mojang_textures.usernames.cache_hit", int64(1)).Once() - suite.Logger.On("IncCounter", "mojang_textures.textures.request", int64(1)).Once() - suite.Logger.On("RecordTimer", "mojang_textures.textures.request_time", mock.Anything).Once() - suite.Logger.On("RecordTimer", "mojang_textures.result_time", mock.Anything).Once() - - suite.Storage.On("GetUuid", "maksimkurb").Once().Return("0d252b7218b648bfb86c2ae476954d32", nil) - // Storage.StoreUuid shouldn't be called - suite.Storage.On("GetTextures", "0d252b7218b648bfb86c2ae476954d32").Once().Return(nil, &ValueNotFound{}) - suite.Storage.On("StoreTextures", "0d252b7218b648bfb86c2ae476954d32", expectedResult).Once() - - // MojangApi.UsernamesToUuids shouldn't be called - suite.MojangApi.On("UuidToTextures", "0d252b7218b648bfb86c2ae476954d32", true).Once().Return(expectedResult, nil) - - resultChan := suite.Queue.GetTexturesForUsername("maksimkurb") - - // Note that there is no iteration - - result := <-resultChan - suite.Assert().Equal(expectedResult, result) -} - -func (suite *queueTestSuite) TestReceiveTexturesForUsernameWithFullyCachedResult() { - expectedResult := &mojang.SignedTexturesResponse{Id: "0d252b7218b648bfb86c2ae476954d32", Name: "maksimkurb"} - - suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Once() - suite.Logger.On("IncCounter", "mojang_textures.usernames.cache_hit", int64(1)).Once() - suite.Logger.On("IncCounter", "mojang_textures.textures.cache_hit", int64(1)).Once() - suite.Logger.On("RecordTimer", "mojang_textures.result_time", mock.Anything).Once() - - suite.Storage.On("GetUuid", "maksimkurb").Once().Return("0d252b7218b648bfb86c2ae476954d32", nil) - // Storage.StoreUuid shouldn't be called - suite.Storage.On("GetTextures", "0d252b7218b648bfb86c2ae476954d32").Once().Return(expectedResult, nil) - // Storage.StoreTextures shouldn't be called - - // MojangApi.UsernamesToUuids shouldn't be called - // MojangApi.UuidToTextures shouldn't be called - - resultChan := suite.Queue.GetTexturesForUsername("maksimkurb") - - // Note that there is no iteration - - result := <-resultChan - suite.Assert().Equal(expectedResult, result) -} - -func (suite *queueTestSuite) TestReceiveTexturesForUsernameWithCachedUnknownUuid() { - suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Once() - suite.Logger.On("IncCounter", "mojang_textures.usernames.cache_hit_nil", int64(1)).Once() - - suite.Storage.On("GetUuid", "maksimkurb").Once().Return("", nil) - // Storage.StoreUuid shouldn't be called - // Storage.GetTextures shouldn't be called - // Storage.StoreTextures shouldn't be called - - // MojangApi.UsernamesToUuids shouldn't be called - // MojangApi.UuidToTextures shouldn't be called - - resultChan := suite.Queue.GetTexturesForUsername("maksimkurb") - - // Note that there is no iteration - - suite.Assert().Nil(<-resultChan) -} - -func (suite *queueTestSuite) TestReceiveTexturesForMoreThan10Usernames() { - usernames := make([]string, 12) - for i := 0; i < cap(usernames); i++ { - usernames[i] = randStr(8) - } - - suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Times(12) - suite.Logger.On("IncCounter", "mojang_textures.usernames.queued", int64(1)).Times(12) - suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(10)).Once() - suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(2)).Once() - suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(2)).Once() - suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(0)).Once() - suite.Logger.On("RecordTimer", "mojang_textures.usernames.round_time", mock.Anything).Twice() - suite.Logger.On("IncCounter", "mojang_textures.usernames.uuid_miss", int64(1)).Times(12) - suite.Logger.On("RecordTimer", "mojang_textures.result_time", mock.Anything).Times(12) - - suite.Storage.On("GetUuid", mock.Anything).Times(12).Return("", &ValueNotFound{}) - suite.Storage.On("StoreUuid", mock.Anything, "").Times(12).Return(nil) // should be called with "" if username is not compared to uuid - // Storage.GetTextures and Storage.SetTextures shouldn't be called - - suite.MojangApi.On("UsernamesToUuids", usernames[0:10]).Once().Return([]*mojang.ProfileInfo{}, nil) - suite.MojangApi.On("UsernamesToUuids", usernames[10:12]).Once().Return([]*mojang.ProfileInfo{}, nil) - - channels := make([]chan *mojang.SignedTexturesResponse, 12) - for i, username := range usernames { - channels[i] = suite.Queue.GetTexturesForUsername(username) - } - - suite.Iterate() - suite.Iterate() - - for _, channel := range channels { - <-channel - } -} - -func (suite *queueTestSuite) TestReceiveTexturesForTheSameUsernames() { - expectedResult := &mojang.SignedTexturesResponse{Id: "0d252b7218b648bfb86c2ae476954d32", Name: "maksimkurb"} - - suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Twice() - suite.Logger.On("IncCounter", "mojang_textures.usernames.queued", int64(1)).Once() - suite.Logger.On("IncCounter", "mojang_textures.already_in_queue", int64(1)).Once() - suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(1)).Once() - suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(0)).Once() - suite.Logger.On("RecordTimer", "mojang_textures.usernames.round_time", mock.Anything) - suite.Logger.On("IncCounter", "mojang_textures.textures.request", int64(1)).Once() - suite.Logger.On("RecordTimer", "mojang_textures.textures.request_time", mock.Anything).Once() - suite.Logger.On("IncCounter", "mojang_textures.usernames.uuid_hit", int64(1)).Once() - suite.Logger.On("RecordTimer", "mojang_textures.result_time", mock.Anything).Once() - - suite.Storage.On("GetUuid", "maksimkurb").Twice().Return("", &ValueNotFound{}) - suite.Storage.On("StoreUuid", "maksimkurb", "0d252b7218b648bfb86c2ae476954d32").Once().Return(nil) - suite.Storage.On("GetTextures", "0d252b7218b648bfb86c2ae476954d32").Once().Return(nil, &ValueNotFound{}) - suite.Storage.On("StoreTextures", "0d252b7218b648bfb86c2ae476954d32", expectedResult).Once() - suite.MojangApi.On("UsernamesToUuids", []string{"maksimkurb"}).Once().Return([]*mojang.ProfileInfo{ - {Id: "0d252b7218b648bfb86c2ae476954d32", Name: "maksimkurb"}, - }, nil) - suite.MojangApi.On("UuidToTextures", "0d252b7218b648bfb86c2ae476954d32", true).Once().Return(expectedResult, nil) - - resultChan1 := suite.Queue.GetTexturesForUsername("maksimkurb") - resultChan2 := suite.Queue.GetTexturesForUsername("maksimkurb") - - suite.Iterate() - - suite.Assert().Equal(expectedResult, <-resultChan1) - suite.Assert().Equal(expectedResult, <-resultChan2) -} - -func (suite *queueTestSuite) TestReceiveTexturesForUsernameThatAlreadyProcessing() { - expectedResult := &mojang.SignedTexturesResponse{Id: "0d252b7218b648bfb86c2ae476954d32", Name: "maksimkurb"} - - suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Twice() - suite.Logger.On("IncCounter", "mojang_textures.usernames.queued", int64(1)).Once() - suite.Logger.On("IncCounter", "mojang_textures.already_in_queue", int64(1)).Once() - suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(1)).Once() - suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(0)).Once() - suite.Logger.On("RecordTimer", "mojang_textures.usernames.round_time", mock.Anything) - suite.Logger.On("IncCounter", "mojang_textures.textures.request", int64(1)).Once() - suite.Logger.On("RecordTimer", "mojang_textures.textures.request_time", mock.Anything).Once() - suite.Logger.On("IncCounter", "mojang_textures.usernames.uuid_hit", int64(1)).Once() - suite.Logger.On("RecordTimer", "mojang_textures.result_time", mock.Anything).Once() - - suite.Storage.On("GetUuid", "maksimkurb").Twice().Return("", &ValueNotFound{}) - suite.Storage.On("StoreUuid", "maksimkurb", "0d252b7218b648bfb86c2ae476954d32").Once().Return(nil) - suite.Storage.On("GetTextures", "0d252b7218b648bfb86c2ae476954d32").Once().Return(nil, &ValueNotFound{}) - suite.Storage.On("StoreTextures", "0d252b7218b648bfb86c2ae476954d32", expectedResult).Once() - - suite.MojangApi.On("UsernamesToUuids", []string{"maksimkurb"}).Once().Return([]*mojang.ProfileInfo{ - {Id: "0d252b7218b648bfb86c2ae476954d32", Name: "maksimkurb"}, - }, nil) - suite.MojangApi.On("UuidToTextures", "0d252b7218b648bfb86c2ae476954d32", true). - Once(). - After(10*time.Millisecond). // Simulate long round trip - Return(expectedResult, nil) - - resultChan1 := suite.Queue.GetTexturesForUsername("maksimkurb") - - // Note that for entire test there is only one iteration - suite.Iterate() - - // Let it meet delayed UuidToTextures request - time.Sleep(5 * time.Millisecond) - - resultChan2 := suite.Queue.GetTexturesForUsername("maksimkurb") - - suite.Assert().Equal(expectedResult, <-resultChan1) - suite.Assert().Equal(expectedResult, <-resultChan2) -} - -func (suite *queueTestSuite) TestDoNothingWhenNoTasks() { - suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Once() - suite.Logger.On("IncCounter", "mojang_textures.usernames.queued", int64(1)).Once() - suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(1)).Once() - suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(0)).Once() - suite.Logger.On("RecordTimer", "mojang_textures.usernames.round_time", mock.Anything) - suite.Logger.On("IncCounter", "mojang_textures.usernames.uuid_miss", int64(1)).Once() - suite.Logger.On("RecordTimer", "mojang_textures.result_time", mock.Anything).Once() - - suite.Storage.On("GetUuid", "maksimkurb").Once().Return("", &ValueNotFound{}) - suite.Storage.On("StoreUuid", "maksimkurb", "").Once().Return(nil) - // Storage.GetTextures and Storage.StoreTextures shouldn't be called - - suite.MojangApi.On("UsernamesToUuids", []string{"maksimkurb"}).Once().Return([]*mojang.ProfileInfo{}, nil) - - // Perform first iteration and await it finish - resultChan := suite.Queue.GetTexturesForUsername("maksimkurb") - - suite.Iterate() - - suite.Assert().Nil(<-resultChan) - - // Let it to perform a few more iterations to ensure, that there is no calls to external APIs - suite.Iterate() - suite.Iterate() -} - -type timeoutError struct { -} - -func (*timeoutError) Error() string { return "timeout error" } -func (*timeoutError) Timeout() bool { return true } -func (*timeoutError) Temporary() bool { return false } - -var expectedErrors = []error{ - &mojang.BadRequestError{}, - &mojang.ForbiddenError{}, - &mojang.TooManyRequestsError{}, - &mojang.ServerError{}, - &timeoutError{}, - &url.Error{Op: "GET", URL: "http://localhost"}, - &net.OpError{Op: "read"}, - &net.OpError{Op: "dial"}, - syscall.ECONNREFUSED, -} - -func (suite *queueTestSuite) TestShouldNotLogErrorWhenExpectedErrorReturnedFromUsernameToUuidRequest() { - suite.Logger.On("IncCounter", mock.Anything, mock.Anything) - suite.Logger.On("UpdateGauge", mock.Anything, mock.Anything) - suite.Logger.On("RecordTimer", mock.Anything, mock.Anything) - suite.Logger.On("Debug", ":name: Got response error :err", mock.Anything, mock.Anything).Times(len(expectedErrors)) - suite.Logger.On("Warning", ":name: Got 400 Bad Request :err", mock.Anything, mock.Anything).Once() - suite.Logger.On("Warning", ":name: Got 403 Forbidden :err", mock.Anything, mock.Anything).Once() - suite.Logger.On("Warning", ":name: Got 429 Too Many Requests :err", mock.Anything, mock.Anything).Once() - - suite.Storage.On("GetUuid", "maksimkurb").Return("", &ValueNotFound{}) - - for _, err := range expectedErrors { - suite.MojangApi.On("UsernamesToUuids", []string{"maksimkurb"}).Once().Return(nil, err) - - resultChan := suite.Queue.GetTexturesForUsername("maksimkurb") - suite.Iterate() - suite.Assert().Nil(<-resultChan) - suite.MojangApi.AssertExpectations(suite.T()) - suite.MojangApi.ExpectedCalls = nil // https://github.com/stretchr/testify/issues/558#issuecomment-372112364 - } -} - -func (suite *queueTestSuite) TestShouldLogEmergencyOnUnexpectedErrorReturnedFromUsernameToUuidRequest() { - suite.Logger.On("IncCounter", mock.Anything, mock.Anything) - suite.Logger.On("UpdateGauge", mock.Anything, mock.Anything) - suite.Logger.On("RecordTimer", mock.Anything, mock.Anything) - suite.Logger.On("Debug", ":name: Got response error :err", mock.Anything, mock.Anything).Once() - suite.Logger.On("Emergency", ":name: Unknown Mojang response error: :err", mock.Anything, mock.Anything).Once() - - suite.Storage.On("GetUuid", "maksimkurb").Return("", &ValueNotFound{}) - - suite.MojangApi.On("UsernamesToUuids", []string{"maksimkurb"}).Once().Return(nil, errors.New("unexpected error")) - - resultChan := suite.Queue.GetTexturesForUsername("maksimkurb") - suite.Iterate() - suite.Assert().Nil(<-resultChan) -} - -func (suite *queueTestSuite) TestShouldNotLogErrorWhenExpectedErrorReturnedFromUuidToTexturesRequest() { - suite.Logger.On("IncCounter", mock.Anything, mock.Anything) - suite.Logger.On("UpdateGauge", mock.Anything, mock.Anything) - suite.Logger.On("RecordTimer", mock.Anything, mock.Anything) - suite.Logger.On("Debug", ":name: Got response error :err", mock.Anything, mock.Anything).Times(len(expectedErrors)) - suite.Logger.On("Warning", ":name: Got 400 Bad Request :err", mock.Anything, mock.Anything).Once() - suite.Logger.On("Warning", ":name: Got 403 Forbidden :err", mock.Anything, mock.Anything).Once() - suite.Logger.On("Warning", ":name: Got 429 Too Many Requests :err", mock.Anything, mock.Anything).Once() - - suite.Storage.On("GetUuid", "maksimkurb").Return("", &ValueNotFound{}) - suite.Storage.On("StoreUuid", "maksimkurb", "0d252b7218b648bfb86c2ae476954d32").Return(nil) - suite.Storage.On("GetTextures", "0d252b7218b648bfb86c2ae476954d32").Return(nil, &ValueNotFound{}) - suite.Storage.On("StoreTextures", "0d252b7218b648bfb86c2ae476954d32", (*mojang.SignedTexturesResponse)(nil)) - - for _, err := range expectedErrors { - suite.MojangApi.On("UsernamesToUuids", []string{"maksimkurb"}).Once().Return([]*mojang.ProfileInfo{ - {Id: "0d252b7218b648bfb86c2ae476954d32", Name: "maksimkurb"}, - }, nil) - suite.MojangApi.On("UuidToTextures", "0d252b7218b648bfb86c2ae476954d32", true).Once().Return(nil, err) - - resultChan := suite.Queue.GetTexturesForUsername("maksimkurb") - suite.Iterate() - suite.Assert().Nil(<-resultChan) - suite.MojangApi.AssertExpectations(suite.T()) - suite.MojangApi.ExpectedCalls = nil // https://github.com/stretchr/testify/issues/558#issuecomment-372112364 - } -} - -func (suite *queueTestSuite) TestShouldLogEmergencyOnUnexpectedErrorReturnedFromUuidToTexturesRequest() { - suite.Logger.On("IncCounter", mock.Anything, mock.Anything) - suite.Logger.On("UpdateGauge", mock.Anything, mock.Anything) - suite.Logger.On("RecordTimer", mock.Anything, mock.Anything) - suite.Logger.On("Debug", ":name: Got response error :err", mock.Anything, mock.Anything).Once() - suite.Logger.On("Emergency", ":name: Unknown Mojang response error: :err", mock.Anything, mock.Anything).Once() - - suite.Storage.On("GetUuid", "maksimkurb").Return("", &ValueNotFound{}) - suite.Storage.On("StoreUuid", "maksimkurb", "0d252b7218b648bfb86c2ae476954d32").Return(nil) - suite.Storage.On("GetTextures", "0d252b7218b648bfb86c2ae476954d32").Return(nil, &ValueNotFound{}) - suite.Storage.On("StoreTextures", "0d252b7218b648bfb86c2ae476954d32", (*mojang.SignedTexturesResponse)(nil)) - - suite.MojangApi.On("UsernamesToUuids", []string{"maksimkurb"}).Once().Return([]*mojang.ProfileInfo{ - {Id: "0d252b7218b648bfb86c2ae476954d32", Name: "maksimkurb"}, - }, nil) - suite.MojangApi.On("UuidToTextures", "0d252b7218b648bfb86c2ae476954d32", true).Once().Return(nil, errors.New("unexpected error")) - - resultChan := suite.Queue.GetTexturesForUsername("maksimkurb") - suite.Iterate() - suite.Assert().Nil(<-resultChan) -} - -func (suite *queueTestSuite) TestReceiveTexturesForNotAllowedMojangUsername() { - suite.Logger.On("IncCounter", "mojang_textures.invalid_username", int64(1)).Once() - - resultChan := suite.Queue.GetTexturesForUsername("Not allowed") - suite.Assert().Nil(<-resultChan) -} - -func TestJobsQueueSuite(t *testing.T) { - suite.Run(t, new(queueTestSuite)) -} - -var replacer = strings.NewReplacer("-", "_", "=", "") - -// https://stackoverflow.com/a/50581165 -func randStr(len int) string { - buff := make([]byte, len) - _, _ = rand.Read(buff) - str := replacer.Replace(base64.URLEncoding.EncodeToString(buff)) - - // Base 64 can be longer than len - return str[:len] -} diff --git a/api/mojang/queue/storage.go b/api/mojang/queue/storage.go deleted file mode 100644 index 2629e58..0000000 --- a/api/mojang/queue/storage.go +++ /dev/null @@ -1,53 +0,0 @@ -package queue - -import "github.com/elyby/chrly/api/mojang" - -type UuidsStorage interface { - GetUuid(username string) (string, error) - StoreUuid(username string, uuid string) error -} - -// nil value can be passed to the storage to indicate that there is no textures -// for uuid and we know about it. Return err only in case, when storage completely -// unable to load any information about textures -type TexturesStorage interface { - GetTextures(uuid string) (*mojang.SignedTexturesResponse, error) - StoreTextures(uuid string, textures *mojang.SignedTexturesResponse) -} - -type Storage interface { - UuidsStorage - TexturesStorage -} - -// SplittedStorage allows you to use separate storage engines to satisfy -// the Storage interface -type SplittedStorage struct { - UuidsStorage - TexturesStorage -} - -func (s *SplittedStorage) GetUuid(username string) (string, error) { - return s.UuidsStorage.GetUuid(username) -} - -func (s *SplittedStorage) StoreUuid(username string, uuid string) error { - return s.UuidsStorage.StoreUuid(username, uuid) -} - -func (s *SplittedStorage) GetTextures(uuid string) (*mojang.SignedTexturesResponse, error) { - return s.TexturesStorage.GetTextures(uuid) -} - -func (s *SplittedStorage) StoreTextures(uuid string, textures *mojang.SignedTexturesResponse) { - s.TexturesStorage.StoreTextures(uuid, textures) -} - -// This error can be used to indicate, that requested -// value doesn't exists in the storage -type ValueNotFound struct { -} - -func (*ValueNotFound) Error() string { - return "value not found in the storage" -} diff --git a/cmd/serve.go b/cmd/serve.go index 31d137e..2ca852d 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -8,11 +8,11 @@ import ( "github.com/spf13/cobra" "github.com/spf13/viper" - "github.com/elyby/chrly/api/mojang/queue" "github.com/elyby/chrly/auth" "github.com/elyby/chrly/bootstrap" "github.com/elyby/chrly/db" "github.com/elyby/chrly/http" + "github.com/elyby/chrly/mojangtextures" ) var serveCmd = &cobra.Command{ @@ -52,12 +52,19 @@ var serveCmd = &cobra.Command{ return } - queue.UuidsQueueIterationDelay = time.Duration(viper.GetInt("queue.loop_delay")) * time.Millisecond - texturesStorage := queue.CreateInMemoryTexturesStorage() + texturesStorage := mojangtextures.NewInMemoryTexturesStorage() texturesStorage.Start() - mojangTexturesQueue := &queue.JobsQueue{ + mojangTexturesProvider := &mojangtextures.Provider{ Logger: logger, - Storage: &queue.SplittedStorage{ + UuidsProvider: &mojangtextures.BatchUuidsProvider{ + IterationDelay: time.Duration(viper.GetInt("queue.loop_delay")) * time.Millisecond, + IterationSize: viper.GetInt("queue.batch_size"), + Logger: logger, + }, + TexturesProvider: &mojangtextures.MojangApiTexturesProvider{ + Logger: logger, + }, + Storage: &mojangtextures.SeparatedStorage{ UuidsStorage: mojangUuidsRepository, TexturesStorage: texturesStorage, }, @@ -65,12 +72,12 @@ var serveCmd = &cobra.Command{ logger.Info("Mojang's textures queue is successfully initialized") cfg := &http.Config{ - ListenSpec: fmt.Sprintf("%s:%d", viper.GetString("server.host"), viper.GetInt("server.port")), - SkinsRepo: skinsRepo, - CapesRepo: capesRepo, - MojangTexturesQueue: mojangTexturesQueue, - Logger: logger, - Auth: &auth.JwtAuth{Key: []byte(viper.GetString("chrly.secret"))}, + ListenSpec: fmt.Sprintf("%s:%d", viper.GetString("server.host"), viper.GetInt("server.port")), + SkinsRepo: skinsRepo, + CapesRepo: capesRepo, + MojangTexturesProvider: mojangTexturesProvider, + Logger: logger, + Auth: &auth.JwtAuth{Key: []byte(viper.GetString("chrly.secret"))}, } if err := cfg.Run(); err != nil { @@ -89,4 +96,5 @@ func init() { viper.SetDefault("storage.filesystem.basePath", "data") viper.SetDefault("storage.filesystem.capesDirName", "capes") viper.SetDefault("queue.loop_delay", 2_500) + viper.SetDefault("queue.batch_size", 10) } diff --git a/db/factory.go b/db/factory.go index 2ba75a7..337d8bf 100644 --- a/db/factory.go +++ b/db/factory.go @@ -3,8 +3,8 @@ package db import ( "github.com/spf13/viper" - "github.com/elyby/chrly/api/mojang/queue" "github.com/elyby/chrly/interfaces" + "github.com/elyby/chrly/mojangtextures" ) type StorageFactory struct { @@ -14,7 +14,7 @@ type StorageFactory struct { type RepositoriesCreator interface { CreateSkinsRepository() (interfaces.SkinsRepository, error) CreateCapesRepository() (interfaces.CapesRepository, error) - CreateMojangUuidsRepository() (queue.UuidsStorage, error) + CreateMojangUuidsRepository() (mojangtextures.UuidsStorage, error) } func (factory *StorageFactory) CreateFactory(backend string) RepositoriesCreator { diff --git a/db/filesystem.go b/db/filesystem.go index 4674652..4a996c3 100644 --- a/db/filesystem.go +++ b/db/filesystem.go @@ -5,9 +5,9 @@ import ( "path" "strings" - "github.com/elyby/chrly/api/mojang/queue" "github.com/elyby/chrly/interfaces" "github.com/elyby/chrly/model" + "github.com/elyby/chrly/mojangtextures" ) type FilesystemFactory struct { @@ -27,7 +27,7 @@ func (f FilesystemFactory) CreateCapesRepository() (interfaces.CapesRepository, return &filesStorage{path: path.Join(f.BasePath, f.CapesDirName)}, nil } -func (f FilesystemFactory) CreateMojangUuidsRepository() (queue.UuidsStorage, error) { +func (f FilesystemFactory) CreateMojangUuidsRepository() (mojangtextures.UuidsStorage, error) { panic("implement me") } diff --git a/db/redis.go b/db/redis.go index df07135..cb657b0 100644 --- a/db/redis.go +++ b/db/redis.go @@ -14,9 +14,9 @@ import ( "github.com/mediocregopher/radix.v2/redis" "github.com/mediocregopher/radix.v2/util" - "github.com/elyby/chrly/api/mojang/queue" "github.com/elyby/chrly/interfaces" "github.com/elyby/chrly/model" + "github.com/elyby/chrly/mojangtextures" ) type RedisFactory struct { @@ -34,7 +34,7 @@ func (f *RedisFactory) CreateCapesRepository() (interfaces.CapesRepository, erro panic("capes repository not supported for this storage type") } -func (f *RedisFactory) CreateMojangUuidsRepository() (queue.UuidsStorage, error) { +func (f *RedisFactory) CreateMojangUuidsRepository() (mojangtextures.UuidsStorage, error) { return f.createInstance() } @@ -255,7 +255,7 @@ func save(skin *model.Skin, conn util.Cmder) error { func findMojangUuidByUsername(username string, conn util.Cmder) (string, error) { response := conn.Cmd("HGET", mojangUsernameToUuidKey, strings.ToLower(username)) if response.IsType(redis.Nil) { - return "", &queue.ValueNotFound{} + return "", &mojangtextures.ValueNotFound{} } data, _ := response.Str() @@ -263,7 +263,7 @@ func findMojangUuidByUsername(username string, conn util.Cmder) (string, error) timestamp, _ := strconv.ParseInt(parts[1], 10, 64) storedAt := time.Unix(timestamp, 0) if storedAt.Add(time.Hour * 24 * 30).Before(time.Now()) { - return "", &queue.ValueNotFound{} + return "", &mojangtextures.ValueNotFound{} } return parts[0], nil diff --git a/http/cape.go b/http/cape.go index 7753fbd..9bba411 100644 --- a/http/cape.go +++ b/http/cape.go @@ -20,8 +20,8 @@ func (cfg *Config) Cape(response http.ResponseWriter, request *http.Request) { return } - mojangTextures := <-cfg.MojangTexturesQueue.GetTexturesForUsername(username) - if mojangTextures == nil { + mojangTextures, err := cfg.MojangTexturesProvider.GetForUsername(username) + if err != nil || mojangTextures == nil { response.WriteHeader(http.StatusNotFound) return } diff --git a/http/cape_test.go b/http/cape_test.go index 5c0de9a..ededa15 100644 --- a/http/cape_test.go +++ b/http/cape_test.go @@ -28,7 +28,7 @@ type capesTestCase struct { var capesTestCases = []*capesTestCase{ { - Name: "Obtain cape for known username", + Name: "Obtain cape for known username", ExistsInLocalStorage: true, AssertResponse: func(assert *testify.Assertions, resp *http.Response) { assert.Equal(200, resp.StatusCode) @@ -38,28 +38,28 @@ var capesTestCases = []*capesTestCase{ }, }, { - Name: "Obtain cape for unknown username that exists in Mojang and has a cape", + Name: "Obtain cape for unknown username that exists in Mojang and has a cape", ExistsInLocalStorage: false, - ExistsInMojang: true, - HasCapeInMojangResp: true, + ExistsInMojang: true, + HasCapeInMojangResp: true, AssertResponse: func(assert *testify.Assertions, resp *http.Response) { assert.Equal(301, resp.StatusCode) assert.Equal("http://mojang/cape.png", resp.Header.Get("Location")) }, }, { - Name: "Obtain cape for unknown username that exists in Mojang, but don't has a cape", + Name: "Obtain cape for unknown username that exists in Mojang, but don't has a cape", ExistsInLocalStorage: false, - ExistsInMojang: true, - HasCapeInMojangResp: false, + ExistsInMojang: true, + HasCapeInMojangResp: false, AssertResponse: func(assert *testify.Assertions, resp *http.Response) { assert.Equal(404, resp.StatusCode) }, }, { - Name: "Obtain cape for unknown username that doesn't exists in Mojang", + Name: "Obtain cape for unknown username that doesn't exists in Mojang", ExistsInLocalStorage: false, - ExistsInMojang: false, + ExistsInMojang: false, AssertResponse: func(assert *testify.Assertions, resp *http.Response) { assert.Equal(404, resp.StatusCode) }, @@ -86,9 +86,9 @@ func TestConfig_Cape(t *testing.T) { if testCase.ExistsInMojang { textures := createTexturesResponse(false, testCase.HasCapeInMojangResp) - mocks.Queue.On("GetTexturesForUsername", "mock_username").Return(textures) + mocks.MojangProvider.On("GetForUsername", "mock_username").Return(textures, nil) } else { - mocks.Queue.On("GetTexturesForUsername", "mock_username").Return(nil) + mocks.MojangProvider.On("GetForUsername", "mock_username").Return(nil, nil) } req := httptest.NewRequest("GET", testCase.RequestUrl, nil) diff --git a/http/http.go b/http/http.go index f04a5d5..bc1edc4 100644 --- a/http/http.go +++ b/http/http.go @@ -19,11 +19,11 @@ import ( type Config struct { ListenSpec string - SkinsRepo interfaces.SkinsRepository - CapesRepo interfaces.CapesRepository - MojangTexturesQueue interfaces.MojangTexturesQueue - Logger wd.Watchdog - Auth interfaces.AuthChecker + SkinsRepo interfaces.SkinsRepository + CapesRepo interfaces.CapesRepository + MojangTexturesProvider interfaces.MojangTexturesProvider + Logger wd.Watchdog + Auth interfaces.AuthChecker } func (cfg *Config) Run() error { diff --git a/http/http_test.go b/http/http_test.go index 66b3472..ec74497 100644 --- a/http/http_test.go +++ b/http/http_test.go @@ -4,9 +4,10 @@ import ( "testing" "time" + "github.com/stretchr/testify/mock" + "github.com/elyby/chrly/api/mojang" - "github.com/elyby/chrly/tests" "github.com/golang/mock/gomock" testify "github.com/stretchr/testify/assert" @@ -20,46 +21,57 @@ func TestParseUsername(t *testing.T) { assert.Equal("test", parseUsername("test"), "Function should return string itself, if it not contains .png at end") } -type mocks struct { - Skins *mock_interfaces.MockSkinsRepository - Capes *mock_interfaces.MockCapesRepository - Queue *tests.MojangTexturesQueueMock - Auth *mock_interfaces.MockAuthChecker - Log *mock_wd.MockWatchdog +type mojangTexturesProviderMock struct { + mock.Mock } -func setupMocks(ctrl *gomock.Controller) ( - *Config, - *mocks, -) { +func (m *mojangTexturesProviderMock) GetForUsername(username string) (*mojang.SignedTexturesResponse, error) { + args := m.Called(username) + var result *mojang.SignedTexturesResponse + if casted, ok := args.Get(0).(*mojang.SignedTexturesResponse); ok { + result = casted + } + + return result, args.Error(1) +} + +type mocks struct { + Skins *mock_interfaces.MockSkinsRepository + Capes *mock_interfaces.MockCapesRepository + MojangProvider *mojangTexturesProviderMock + Auth *mock_interfaces.MockAuthChecker + Log *mock_wd.MockWatchdog +} + +func setupMocks(ctrl *gomock.Controller) (*Config, *mocks) { skinsRepo := mock_interfaces.NewMockSkinsRepository(ctrl) capesRepo := mock_interfaces.NewMockCapesRepository(ctrl) authChecker := mock_interfaces.NewMockAuthChecker(ctrl) wd := mock_wd.NewMockWatchdog(ctrl) - texturesQueue := &tests.MojangTexturesQueueMock{} + texturesProvider := &mojangTexturesProviderMock{} return &Config{ - SkinsRepo: skinsRepo, - CapesRepo: capesRepo, - Auth: authChecker, - MojangTexturesQueue: texturesQueue, - Logger: wd, - }, &mocks{ - Skins: skinsRepo, - Capes: capesRepo, - Auth: authChecker, - Queue: texturesQueue, - Log: wd, - } + SkinsRepo: skinsRepo, + CapesRepo: capesRepo, + Auth: authChecker, + MojangTexturesProvider: texturesProvider, + Logger: wd, + }, &mocks{ + Skins: skinsRepo, + Capes: capesRepo, + Auth: authChecker, + MojangProvider: texturesProvider, + Log: wd, + } } func createTexturesResponse(includeSkin bool, includeCape bool) *mojang.SignedTexturesResponse { timeZone, _ := time.LoadLocation("Europe/Minsk") textures := &mojang.TexturesProp{ - Timestamp: time.Date(2019, 4, 27, 23, 56, 12, 0, timeZone).Unix(), - ProfileID: "00000000000000000000000000000000", + Timestamp: time.Date(2019, 4, 27, 23, 56, 12, 0, timeZone).Unix(), + ProfileID: "00000000000000000000000000000000", ProfileName: "mock_user", - Textures: &mojang.TexturesResponse{}, + Textures: &mojang.TexturesResponse{}, } if includeSkin { @@ -75,11 +87,11 @@ func createTexturesResponse(includeSkin bool, includeCape bool) *mojang.SignedTe } response := &mojang.SignedTexturesResponse{ - Id: "00000000000000000000000000000000", + Id: "00000000000000000000000000000000", Name: "mock_user", Props: []*mojang.Property{ { - Name: "textures", + Name: "textures", Value: mojang.EncodeTextures(textures), }, }, diff --git a/http/signed_textures.go b/http/signed_textures.go index 34002a5..a3b1a20 100644 --- a/http/signed_textures.go +++ b/http/signed_textures.go @@ -30,7 +30,10 @@ func (cfg *Config) SignedTextures(response http.ResponseWriter, request *http.Re }, } } else if request.URL.Query().Get("proxy") != "" { - responseData = <-cfg.MojangTexturesQueue.GetTexturesForUsername(username) + mojangTextures, err := cfg.MojangTexturesProvider.GetForUsername(username) + if err == nil && mojangTextures != nil { + responseData = mojangTextures + } } if responseData == nil { diff --git a/http/signed_textures_test.go b/http/signed_textures_test.go index c10fe1d..c07a969 100644 --- a/http/signed_textures_test.go +++ b/http/signed_textures_test.go @@ -112,7 +112,7 @@ func TestConfig_SignedTextures(t *testing.T) { mocks.Log.EXPECT().IncCounter("signed_textures.request", int64(1)) mocks.Skins.EXPECT().FindByUsername("mock_user").Return(skinModel, nil) - mocks.Queue.On("GetTexturesForUsername", "mock_user").Once().Return(createTexturesResponse(true, false)) + mocks.MojangProvider.On("GetForUsername", "mock_user").Once().Return(createTexturesResponse(true, false), nil) req := httptest.NewRequest("GET", "http://chrly/textures/signed/mock_user?proxy=true", nil) w := httptest.NewRecorder() diff --git a/http/skin.go b/http/skin.go index faa8fae..a7f9397 100644 --- a/http/skin.go +++ b/http/skin.go @@ -18,8 +18,8 @@ func (cfg *Config) Skin(response http.ResponseWriter, request *http.Request) { return } - mojangTextures := <-cfg.MojangTexturesQueue.GetTexturesForUsername(username) - if mojangTextures == nil { + mojangTextures, err := cfg.MojangTexturesProvider.GetForUsername(username) + if err != nil || mojangTextures == nil { response.WriteHeader(http.StatusNotFound) return } diff --git a/http/skin_test.go b/http/skin_test.go index 4a9fd97..23475ab 100644 --- a/http/skin_test.go +++ b/http/skin_test.go @@ -78,9 +78,9 @@ func TestConfig_Skin(t *testing.T) { if testCase.ExistsInMojang { textures := createTexturesResponse(testCase.HasSkinInMojangResp, true) - mocks.Queue.On("GetTexturesForUsername", "mock_username").Return(textures) + mocks.MojangProvider.On("GetForUsername", "mock_username").Return(textures, nil) } else { - mocks.Queue.On("GetTexturesForUsername", "mock_username").Return(nil) + mocks.MojangProvider.On("GetForUsername", "mock_username").Return(nil, nil) } req := httptest.NewRequest("GET", testCase.RequestUrl, nil) diff --git a/http/textures.go b/http/textures.go index 92cda48..244cd25 100644 --- a/http/textures.go +++ b/http/textures.go @@ -39,8 +39,8 @@ func (cfg *Config) Textures(response http.ResponseWriter, request *http.Request) } } } else { - mojangTextures := <-cfg.MojangTexturesQueue.GetTexturesForUsername(username) - if mojangTextures == nil { + mojangTextures, err := cfg.MojangTexturesProvider.GetForUsername(username) + if err != nil || mojangTextures == nil { response.WriteHeader(http.StatusNoContent) return } diff --git a/http/textures_test.go b/http/textures_test.go index d7c57e5..9ffa91c 100644 --- a/http/textures_test.go +++ b/http/textures_test.go @@ -148,7 +148,7 @@ func TestConfig_Textures(t *testing.T) { mocks.Skins.EXPECT().FindByUsername("mock_username").Return(nil, &db.SkinNotFoundError{}) mocks.Capes.EXPECT().FindByUsername("mock_username").Return(nil, &db.CapeNotFoundError{}) - mocks.Queue.On("GetTexturesForUsername", "mock_username").Once().Return(createTexturesResponse(true, true)) + mocks.MojangProvider.On("GetForUsername", "mock_username").Once().Return(createTexturesResponse(true, true), nil) req := httptest.NewRequest("GET", "http://chrly/textures/mock_username", nil) w := httptest.NewRecorder() @@ -181,7 +181,7 @@ func TestConfig_Textures(t *testing.T) { mocks.Skins.EXPECT().FindByUsername("mock_username").Return(nil, &db.SkinNotFoundError{}) mocks.Capes.EXPECT().FindByUsername("mock_username").Return(nil, &db.CapeNotFoundError{}) - mocks.Queue.On("GetTexturesForUsername", "mock_username").Once().Return(nil) + mocks.MojangProvider.On("GetForUsername", "mock_username").Once().Return(nil, nil) req := httptest.NewRequest("GET", "http://chrly/textures/mock_username", nil) w := httptest.NewRecorder() diff --git a/interfaces/repositories.go b/interfaces/repositories.go index 134f141..848045a 100644 --- a/interfaces/repositories.go +++ b/interfaces/repositories.go @@ -17,6 +17,6 @@ type CapesRepository interface { FindByUsername(username string) (*model.Cape, error) } -type MojangTexturesQueue interface { - GetTexturesForUsername(username string) chan *mojang.SignedTexturesResponse +type MojangTexturesProvider interface { + GetForUsername(username string) (*mojang.SignedTexturesResponse, error) } diff --git a/mojangtextures/batch_uuids_provider.go b/mojangtextures/batch_uuids_provider.go new file mode 100644 index 0000000..d3259c1 --- /dev/null +++ b/mojangtextures/batch_uuids_provider.go @@ -0,0 +1,133 @@ +package mojangtextures + +import ( + "strings" + "sync" + "time" + + "github.com/mono83/slf/wd" + + "github.com/elyby/chrly/api/mojang" +) + +type jobResult struct { + profile *mojang.ProfileInfo + error error +} + +type jobItem struct { + username string + respondChan chan *jobResult +} + +type jobsQueue struct { + lock sync.Mutex + items []*jobItem +} + +func (s *jobsQueue) New() *jobsQueue { + s.items = []*jobItem{} + return s +} + +func (s *jobsQueue) Enqueue(t *jobItem) { + s.lock.Lock() + defer s.lock.Unlock() + + s.items = append(s.items, t) +} + +func (s *jobsQueue) Dequeue(n int) []*jobItem { + s.lock.Lock() + defer s.lock.Unlock() + + if n > s.Size() { + n = s.Size() + } + + items := s.items[0:n] + s.items = s.items[n:len(s.items)] + + return items +} + +func (s *jobsQueue) Size() int { + return len(s.items) +} + +var usernamesToUuids = mojang.UsernamesToUuids +var forever = func() bool { + return true +} + +type BatchUuidsProvider struct { + IterationDelay time.Duration + IterationSize int + Logger wd.Watchdog + + onFirstCall sync.Once + queue jobsQueue +} + +func (ctx *BatchUuidsProvider) GetUuid(username string) (*mojang.ProfileInfo, error) { + ctx.onFirstCall.Do(func() { + ctx.queue.New() + ctx.startQueue() + }) + + resultChan := make(chan *jobResult) + ctx.queue.Enqueue(&jobItem{username, resultChan}) + ctx.Logger.IncCounter("mojang_textures.usernames.queued", 1) + + result := <-resultChan + + return result.profile, result.error +} + +func (ctx *BatchUuidsProvider) startQueue() { + go func() { + time.Sleep(ctx.IterationDelay) + for forever() { + start := time.Now() + ctx.queueRound() + elapsed := time.Since(start) + ctx.Logger.RecordTimer("mojang_textures.usernames.round_time", elapsed) + time.Sleep(ctx.IterationDelay) + } + }() +} + +func (ctx *BatchUuidsProvider) queueRound() { + queueSize := ctx.queue.Size() + jobs := ctx.queue.Dequeue(ctx.IterationSize) + ctx.Logger.UpdateGauge("mojang_textures.usernames.queue_size", int64(queueSize-len(jobs))) + ctx.Logger.UpdateGauge("mojang_textures.usernames.iteration_size", int64(len(jobs))) + if len(jobs) == 0 { + return + } + + var usernames []string + for _, job := range jobs { + usernames = append(usernames, job.username) + } + + profiles, err := usernamesToUuids(usernames) + for _, job := range jobs { + go func(job *jobItem) { + response := &jobResult{} + if err != nil { + response.error = err + } else { + // The profiles in the response aren't ordered, so we must search each username over full array + for _, profile := range profiles { + if strings.EqualFold(job.username, profile.Name) { + response.profile = profile + break + } + } + } + + job.respondChan <- response + }(job) + } +} diff --git a/mojangtextures/batch_uuids_provider_test.go b/mojangtextures/batch_uuids_provider_test.go new file mode 100644 index 0000000..f217dd0 --- /dev/null +++ b/mojangtextures/batch_uuids_provider_test.go @@ -0,0 +1,285 @@ +package mojangtextures + +import ( + "crypto/rand" + "encoding/base64" + "strings" + "testing" + "time" + + testify "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" + + "github.com/elyby/chrly/api/mojang" + mocks "github.com/elyby/chrly/tests" +) + +func TestJobsQueue(t *testing.T) { + createQueue := func() *jobsQueue { + queue := &jobsQueue{} + queue.New() + + return queue + } + + t.Run("Enqueue", func(t *testing.T) { + assert := testify.New(t) + + s := createQueue() + s.Enqueue(&jobItem{username: "username1"}) + s.Enqueue(&jobItem{username: "username2"}) + s.Enqueue(&jobItem{username: "username3"}) + + assert.Equal(3, s.Size()) + }) + + t.Run("Dequeue", func(t *testing.T) { + assert := testify.New(t) + + s := createQueue() + s.Enqueue(&jobItem{username: "username1"}) + s.Enqueue(&jobItem{username: "username2"}) + s.Enqueue(&jobItem{username: "username3"}) + s.Enqueue(&jobItem{username: "username4"}) + + items := s.Dequeue(2) + assert.Len(items, 2) + assert.Equal("username1", items[0].username) + assert.Equal("username2", items[1].username) + assert.Equal(2, s.Size()) + + items = s.Dequeue(40) + assert.Len(items, 2) + assert.Equal("username3", items[0].username) + assert.Equal("username4", items[1].username) + }) +} + +// This is really stupid test just to get 100% coverage on this package :) +func TestBatchUuidsProvider_forever(t *testing.T) { + testify.True(t, forever()) +} + +type mojangUsernamesToUuidsRequestMock struct { + mock.Mock +} + +func (o *mojangUsernamesToUuidsRequestMock) UsernamesToUuids(usernames []string) ([]*mojang.ProfileInfo, error) { + args := o.Called(usernames) + var result []*mojang.ProfileInfo + if casted, ok := args.Get(0).([]*mojang.ProfileInfo); ok { + result = casted + } + + return result, args.Error(1) +} + +type batchUuidsProviderGetUuidResult struct { + Result *mojang.ProfileInfo + Error error +} + +type batchUuidsProviderTestSuite struct { + suite.Suite + + Provider *BatchUuidsProvider + GetUuidAsync func(username string) chan *batchUuidsProviderGetUuidResult + + Logger *mocks.WdMock + MojangApi *mojangUsernamesToUuidsRequestMock + + Iterate func() + done func() + iterateChan chan bool +} + +func (suite *batchUuidsProviderTestSuite) SetupTest() { + suite.Logger = &mocks.WdMock{} + + suite.Provider = &BatchUuidsProvider{ + Logger: suite.Logger, + IterationDelay: 0, + IterationSize: 10, + } + + suite.iterateChan = make(chan bool) + forever = func() bool { + return <-suite.iterateChan + } + + suite.Iterate = func() { + suite.iterateChan <- true + } + + suite.done = func() { + suite.iterateChan <- false + } + + suite.GetUuidAsync = func(username string) chan *batchUuidsProviderGetUuidResult { + c := make(chan *batchUuidsProviderGetUuidResult) + go func() { + profile, err := suite.Provider.GetUuid(username) + c <- &batchUuidsProviderGetUuidResult{ + Result: profile, + Error: err, + } + }() + + return c + } + + suite.MojangApi = &mojangUsernamesToUuidsRequestMock{} + usernamesToUuids = suite.MojangApi.UsernamesToUuids +} + +func (suite *batchUuidsProviderTestSuite) TearDownTest() { + suite.done() + time.Sleep(10 * time.Millisecond) // Add delay to let finish all goroutines before assert mocks calls + suite.MojangApi.AssertExpectations(suite.T()) + suite.Logger.AssertExpectations(suite.T()) +} + +func TestBatchUuidsProvider(t *testing.T) { + suite.Run(t, new(batchUuidsProviderTestSuite)) +} + +func (suite *batchUuidsProviderTestSuite) TestGetUuidForOneUsername() { + expectedResult := &mojang.ProfileInfo{Id: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Name: "username"} + + suite.Logger.On("IncCounter", "mojang_textures.usernames.queued", int64(1)).Once() + suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(1)).Once() + suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(0)).Once() + suite.Logger.On("RecordTimer", "mojang_textures.usernames.round_time", mock.Anything).Once() + + suite.MojangApi.On("UsernamesToUuids", []string{"username"}).Once().Return([]*mojang.ProfileInfo{expectedResult}, nil) + + resultChan := suite.GetUuidAsync("username") + + suite.Iterate() + + result := <-resultChan + suite.Assert().Equal(expectedResult, result.Result) + suite.Assert().Nil(result.Error) +} + +func (suite *batchUuidsProviderTestSuite) TestGetUuidForTwoUsernames() { + expectedResult1 := &mojang.ProfileInfo{Id: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Name: "username1"} + expectedResult2 := &mojang.ProfileInfo{Id: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", Name: "username2"} + + suite.Logger.On("IncCounter", "mojang_textures.usernames.queued", int64(1)).Twice() + suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(2)).Once() + suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(0)).Once() + suite.Logger.On("RecordTimer", "mojang_textures.usernames.round_time", mock.Anything).Once() + + suite.MojangApi.On("UsernamesToUuids", []string{"username1", "username2"}).Once().Return([]*mojang.ProfileInfo{ + expectedResult1, + expectedResult2, + }, nil) + + resultChan1 := suite.GetUuidAsync("username1") + time.Sleep(time.Millisecond) // Just to keep order for the usernames + resultChan2 := suite.GetUuidAsync("username2") + time.Sleep(time.Millisecond) // Allow to all goroutines begin + + suite.Iterate() + + result1 := <-resultChan1 + suite.Assert().Equal(expectedResult1, result1.Result) + suite.Assert().Nil(result1.Error) + + result2 := <-resultChan2 + suite.Assert().Equal(expectedResult2, result2.Result) + suite.Assert().Nil(result2.Error) +} + +func (suite *batchUuidsProviderTestSuite) TestGetUuidForMoreThan10Usernames() { + usernames := make([]string, 12) + for i := 0; i < cap(usernames); i++ { + usernames[i] = randStr(8) + } + + suite.Logger.On("IncCounter", "mojang_textures.usernames.queued", int64(1)).Times(12) + suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(10)).Once() + suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(2)).Once() + suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(2)).Once() + suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(0)).Once() + suite.Logger.On("RecordTimer", "mojang_textures.usernames.round_time", mock.Anything).Twice() + + suite.MojangApi.On("UsernamesToUuids", usernames[0:10]).Once().Return([]*mojang.ProfileInfo{}, nil) + suite.MojangApi.On("UsernamesToUuids", usernames[10:12]).Once().Return([]*mojang.ProfileInfo{}, nil) + + channels := make([]chan *batchUuidsProviderGetUuidResult, 12) + for i, username := range usernames { + channels[i] = suite.GetUuidAsync(username) + time.Sleep(time.Millisecond) // Just to keep order for the usernames + } + + suite.Iterate() + suite.Iterate() + + for _, channel := range channels { + <-channel + } +} + +func (suite *batchUuidsProviderTestSuite) TestDoNothingWhenNoTasks() { + suite.Logger.On("IncCounter", "mojang_textures.usernames.queued", int64(1)).Once() + suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(1)).Once() + suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(0)).Twice() + suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(0)).Times(3) + suite.Logger.On("RecordTimer", "mojang_textures.usernames.round_time", mock.Anything) + + suite.MojangApi.On("UsernamesToUuids", []string{"username"}).Once().Return([]*mojang.ProfileInfo{}, nil) + + // Perform first iteration and await it finish + resultChan := suite.GetUuidAsync("username") + + suite.Iterate() + + result := <-resultChan + suite.Assert().Nil(result.Result) + suite.Assert().Nil(result.Error) + + // Let it to perform a few more iterations to ensure, that there is no calls to external APIs + suite.Iterate() + suite.Iterate() +} + +func (suite *batchUuidsProviderTestSuite) TestGetUuidForTwoUsernamesWithAnError() { + expectedError := &mojang.TooManyRequestsError{} + + suite.Logger.On("IncCounter", "mojang_textures.usernames.queued", int64(1)).Twice() + suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(2)).Once() + suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(0)).Once() + suite.Logger.On("RecordTimer", "mojang_textures.usernames.round_time", mock.Anything).Once() + + suite.MojangApi.On("UsernamesToUuids", []string{"username1", "username2"}).Once().Return(nil, expectedError) + + resultChan1 := suite.GetUuidAsync("username1") + time.Sleep(time.Millisecond) // Just to keep order for the usernames + resultChan2 := suite.GetUuidAsync("username2") + time.Sleep(time.Millisecond) // Allow to all goroutines begin + + suite.Iterate() + + result1 := <-resultChan1 + suite.Assert().Nil(result1.Result) + suite.Assert().Equal(expectedError, result1.Error) + + result2 := <-resultChan2 + suite.Assert().Nil(result2.Result) + suite.Assert().Equal(expectedError, result2.Error) +} + +var replacer = strings.NewReplacer("-", "_", "=", "") + +// https://stackoverflow.com/a/50581165 +func randStr(len int) string { + buff := make([]byte, len) + _, _ = rand.Read(buff) + str := replacer.Replace(base64.URLEncoding.EncodeToString(buff)) + + // Base 64 can be longer than len + return str[:len] +} diff --git a/api/mojang/queue/in_memory_textures_storage.go b/mojangtextures/in_memory_textures_storage.go similarity index 62% rename from api/mojang/queue/in_memory_textures_storage.go rename to mojangtextures/in_memory_textures_storage.go index cb7d8a3..37671df 100644 --- a/api/mojang/queue/in_memory_textures_storage.go +++ b/mojangtextures/in_memory_textures_storage.go @@ -1,4 +1,4 @@ -package queue +package mojangtextures import ( "sync" @@ -9,8 +9,6 @@ import ( "github.com/tevino/abool" ) -var inMemoryStorageGCPeriod = 10 * time.Second -var inMemoryStoragePersistPeriod = time.Minute + 10*time.Second var now = time.Now type inMemoryItem struct { @@ -18,33 +16,38 @@ type inMemoryItem struct { timestamp int64 } -type inMemoryTexturesStorage struct { +type InMemoryTexturesStorage struct { + GCPeriod time.Duration + Duration time.Duration + lock sync.Mutex data map[string]*inMemoryItem working *abool.AtomicBool } -func CreateInMemoryTexturesStorage() *inMemoryTexturesStorage { - storage := &inMemoryTexturesStorage{ - data: make(map[string]*inMemoryItem), +func NewInMemoryTexturesStorage() *InMemoryTexturesStorage { + storage := &InMemoryTexturesStorage{ + GCPeriod: 10 * time.Second, + Duration: time.Minute + 10*time.Second, + data: make(map[string]*inMemoryItem), } return storage } -func (s *inMemoryTexturesStorage) Start() { +func (s *InMemoryTexturesStorage) Start() { if s.working == nil { s.working = abool.New() } if !s.working.IsSet() { go func() { - time.Sleep(inMemoryStorageGCPeriod) + time.Sleep(s.GCPeriod) // TODO: this can be reimplemented in future with channels, but right now I have no idea how to make it right for s.working.IsSet() { start := time.Now() s.gc() - time.Sleep(inMemoryStorageGCPeriod - time.Since(start)) + time.Sleep(s.GCPeriod - time.Since(start)) } }() } @@ -52,16 +55,16 @@ func (s *inMemoryTexturesStorage) Start() { s.working.Set() } -func (s *inMemoryTexturesStorage) Stop() { +func (s *InMemoryTexturesStorage) Stop() { s.working.UnSet() } -func (s *inMemoryTexturesStorage) GetTextures(uuid string) (*mojang.SignedTexturesResponse, error) { +func (s *InMemoryTexturesStorage) GetTextures(uuid string) (*mojang.SignedTexturesResponse, error) { s.lock.Lock() defer s.lock.Unlock() item, exists := s.data[uuid] - validRange := getMinimalNotExpiredTimestamp() + validRange := s.getMinimalNotExpiredTimestamp() if !exists || validRange > item.timestamp { return nil, &ValueNotFound{} } @@ -69,7 +72,7 @@ func (s *inMemoryTexturesStorage) GetTextures(uuid string) (*mojang.SignedTextur return item.textures, nil } -func (s *inMemoryTexturesStorage) StoreTextures(uuid string, textures *mojang.SignedTexturesResponse) { +func (s *InMemoryTexturesStorage) StoreTextures(uuid string, textures *mojang.SignedTexturesResponse) { var timestamp int64 if textures != nil { decoded := textures.DecodeTextures() @@ -91,11 +94,11 @@ func (s *inMemoryTexturesStorage) StoreTextures(uuid string, textures *mojang.Si } } -func (s *inMemoryTexturesStorage) gc() { +func (s *InMemoryTexturesStorage) gc() { s.lock.Lock() defer s.lock.Unlock() - maxTime := getMinimalNotExpiredTimestamp() + maxTime := s.getMinimalNotExpiredTimestamp() for uuid, value := range s.data { if maxTime > value.timestamp { delete(s.data, uuid) @@ -103,8 +106,8 @@ func (s *inMemoryTexturesStorage) gc() { } } -func getMinimalNotExpiredTimestamp() int64 { - return unixNanoToUnixMicro(now().Add(inMemoryStoragePersistPeriod * time.Duration(-1)).UnixNano()) +func (s *InMemoryTexturesStorage) getMinimalNotExpiredTimestamp() int64 { + return unixNanoToUnixMicro(now().Add(s.Duration * time.Duration(-1)).UnixNano()) } func unixNanoToUnixMicro(unixNano int64) int64 { diff --git a/api/mojang/queue/in_memory_textures_storage_test.go b/mojangtextures/in_memory_textures_storage_test.go similarity index 85% rename from api/mojang/queue/in_memory_textures_storage_test.go rename to mojangtextures/in_memory_textures_storage_test.go index 1e90812..3ad6789 100644 --- a/api/mojang/queue/in_memory_textures_storage_test.go +++ b/mojangtextures/in_memory_textures_storage_test.go @@ -1,4 +1,4 @@ -package queue +package mojangtextures import ( "time" @@ -48,7 +48,7 @@ func TestInMemoryTexturesStorage_GetTextures(t *testing.T) { t.Run("get error when uuid is not exists", func(t *testing.T) { assert := testify.New(t) - storage := CreateInMemoryTexturesStorage() + storage := NewInMemoryTexturesStorage() result, err := storage.GetTextures("b5d58475007d4f9e9ddd1403e2497579") assert.Nil(result) @@ -58,7 +58,7 @@ func TestInMemoryTexturesStorage_GetTextures(t *testing.T) { t.Run("get textures object, when uuid is stored in the storage", func(t *testing.T) { assert := testify.New(t) - storage := CreateInMemoryTexturesStorage() + storage := NewInMemoryTexturesStorage() storage.StoreTextures("dead24f9a4fa4877b7b04c8c6c72bb46", texturesWithSkin) result, err := storage.GetTextures("dead24f9a4fa4877b7b04c8c6c72bb46") @@ -69,7 +69,7 @@ func TestInMemoryTexturesStorage_GetTextures(t *testing.T) { t.Run("get error when uuid is exists, but textures are expired", func(t *testing.T) { assert := testify.New(t) - storage := CreateInMemoryTexturesStorage() + storage := NewInMemoryTexturesStorage() storage.StoreTextures("dead24f9a4fa4877b7b04c8c6c72bb46", texturesWithSkin) now = func() time.Time { @@ -89,7 +89,7 @@ func TestInMemoryTexturesStorage_StoreTextures(t *testing.T) { t.Run("store textures for previously not existed uuid", func(t *testing.T) { assert := testify.New(t) - storage := CreateInMemoryTexturesStorage() + storage := NewInMemoryTexturesStorage() storage.StoreTextures("dead24f9a4fa4877b7b04c8c6c72bb46", texturesWithSkin) result, err := storage.GetTextures("dead24f9a4fa4877b7b04c8c6c72bb46") @@ -100,7 +100,7 @@ func TestInMemoryTexturesStorage_StoreTextures(t *testing.T) { t.Run("override already existed textures for uuid", func(t *testing.T) { assert := testify.New(t) - storage := CreateInMemoryTexturesStorage() + storage := NewInMemoryTexturesStorage() storage.StoreTextures("dead24f9a4fa4877b7b04c8c6c72bb46", texturesWithoutSkin) storage.StoreTextures("dead24f9a4fa4877b7b04c8c6c72bb46", texturesWithSkin) result, err := storage.GetTextures("dead24f9a4fa4877b7b04c8c6c72bb46") @@ -113,7 +113,7 @@ func TestInMemoryTexturesStorage_StoreTextures(t *testing.T) { t.Run("store nil textures", func(t *testing.T) { assert := testify.New(t) - storage := CreateInMemoryTexturesStorage() + storage := NewInMemoryTexturesStorage() storage.StoreTextures("dead24f9a4fa4877b7b04c8c6c72bb46", nil) result, err := storage.GetTextures("dead24f9a4fa4877b7b04c8c6c72bb46") @@ -131,7 +131,7 @@ func TestInMemoryTexturesStorage_StoreTextures(t *testing.T) { } assert.PanicsWithValue("unable to decode textures", func() { - storage := CreateInMemoryTexturesStorage() + storage := NewInMemoryTexturesStorage() storage.StoreTextures("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", toStore) }) }) @@ -140,8 +140,9 @@ func TestInMemoryTexturesStorage_StoreTextures(t *testing.T) { func TestInMemoryTexturesStorage_GarbageCollection(t *testing.T) { assert := testify.New(t) - inMemoryStorageGCPeriod = 10 * time.Millisecond - inMemoryStoragePersistPeriod = 10 * time.Millisecond + storage := NewInMemoryTexturesStorage() + storage.GCPeriod = 10 * time.Millisecond + storage.Duration = 10 * time.Millisecond textures1 := &mojang.SignedTexturesResponse{ Id: "dead24f9a4fa4877b7b04c8c6c72bb46", @@ -150,7 +151,7 @@ func TestInMemoryTexturesStorage_GarbageCollection(t *testing.T) { { Name: "textures", Value: mojang.EncodeTextures(&mojang.TexturesProp{ - Timestamp: time.Now().Add(inMemoryStorageGCPeriod-time.Millisecond*time.Duration(5)).UnixNano() / 10e5, + Timestamp: time.Now().Add(storage.GCPeriod-time.Millisecond*time.Duration(5)).UnixNano() / 10e5, ProfileID: "dead24f9a4fa4877b7b04c8c6c72bb46", ProfileName: "mock1", Textures: &mojang.TexturesResponse{}, @@ -165,7 +166,7 @@ func TestInMemoryTexturesStorage_GarbageCollection(t *testing.T) { { Name: "textures", Value: mojang.EncodeTextures(&mojang.TexturesProp{ - Timestamp: time.Now().Add(inMemoryStorageGCPeriod-time.Millisecond*time.Duration(15)).UnixNano() / 10e5, + Timestamp: time.Now().Add(storage.GCPeriod-time.Millisecond*time.Duration(15)).UnixNano() / 10e5, ProfileID: "b5d58475007d4f9e9ddd1403e2497579", ProfileName: "mock2", Textures: &mojang.TexturesResponse{}, @@ -174,13 +175,12 @@ func TestInMemoryTexturesStorage_GarbageCollection(t *testing.T) { }, } - storage := CreateInMemoryTexturesStorage() storage.StoreTextures("dead24f9a4fa4877b7b04c8c6c72bb46", textures1) storage.StoreTextures("b5d58475007d4f9e9ddd1403e2497579", textures2) storage.Start() - time.Sleep(inMemoryStorageGCPeriod + time.Millisecond) // Let it start first iteration + time.Sleep(storage.GCPeriod + time.Millisecond) // Let it start first iteration _, textures1Err := storage.GetTextures("dead24f9a4fa4877b7b04c8c6c72bb46") _, textures2Err := storage.GetTextures("b5d58475007d4f9e9ddd1403e2497579") @@ -188,7 +188,7 @@ func TestInMemoryTexturesStorage_GarbageCollection(t *testing.T) { assert.Nil(textures1Err) assert.Error(textures2Err) - time.Sleep(inMemoryStorageGCPeriod + time.Millisecond) // Let another iteration happen + time.Sleep(storage.GCPeriod + time.Millisecond) // Let another iteration happen _, textures1Err = storage.GetTextures("dead24f9a4fa4877b7b04c8c6c72bb46") _, textures2Err = storage.GetTextures("b5d58475007d4f9e9ddd1403e2497579") diff --git a/mojangtextures/mojang_api_textures_provider.go b/mojangtextures/mojang_api_textures_provider.go new file mode 100644 index 0000000..716f316 --- /dev/null +++ b/mojangtextures/mojang_api_textures_provider.go @@ -0,0 +1,25 @@ +package mojangtextures + +import ( + "time" + + "github.com/mono83/slf/wd" + + "github.com/elyby/chrly/api/mojang" +) + +var uuidToTextures = mojang.UuidToTextures + +type MojangApiTexturesProvider struct { + Logger wd.Watchdog +} + +func (ctx *MojangApiTexturesProvider) GetTextures(uuid string) (*mojang.SignedTexturesResponse, error) { + ctx.Logger.IncCounter("mojang_textures.textures.request", 1) + + start := time.Now() + result, err := uuidToTextures(uuid, true) + ctx.Logger.RecordTimer("mojang_textures.textures.request_time", time.Since(start)) + + return result, err +} diff --git a/mojangtextures/mojang_api_textures_provider_test.go b/mojangtextures/mojang_api_textures_provider_test.go new file mode 100644 index 0000000..ed2a349 --- /dev/null +++ b/mojangtextures/mojang_api_textures_provider_test.go @@ -0,0 +1,82 @@ +package mojangtextures + +import ( + "testing" + + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" + + "github.com/elyby/chrly/api/mojang" + mocks "github.com/elyby/chrly/tests" +) + +type mojangUuidToTexturesRequestMock struct { + mock.Mock +} + +func (o *mojangUuidToTexturesRequestMock) UuidToTextures(uuid string, signed bool) (*mojang.SignedTexturesResponse, error) { + args := o.Called(uuid, signed) + var result *mojang.SignedTexturesResponse + if casted, ok := args.Get(0).(*mojang.SignedTexturesResponse); ok { + result = casted + } + + return result, args.Error(1) +} + +type mojangApiTexturesProviderTestSuite struct { + suite.Suite + + Provider *MojangApiTexturesProvider + Logger *mocks.WdMock + MojangApi *mojangUuidToTexturesRequestMock +} + +func (suite *mojangApiTexturesProviderTestSuite) SetupTest() { + suite.Logger = &mocks.WdMock{} + suite.MojangApi = &mojangUuidToTexturesRequestMock{} + + suite.Provider = &MojangApiTexturesProvider{ + Logger: suite.Logger, + } + + uuidToTextures = suite.MojangApi.UuidToTextures +} + +func (suite *mojangApiTexturesProviderTestSuite) TearDownTest() { + suite.MojangApi.AssertExpectations(suite.T()) + suite.Logger.AssertExpectations(suite.T()) +} + +func TestMojangApiTexturesProvider(t *testing.T) { + suite.Run(t, new(mojangApiTexturesProviderTestSuite)) +} + +func (suite *mojangApiTexturesProviderTestSuite) TestGetTextures() { + expectedResult := &mojang.SignedTexturesResponse{ + Id: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Name: "username", + } + suite.MojangApi.On("UuidToTextures", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", true).Once().Return(expectedResult, nil) + + suite.Logger.On("IncCounter", "mojang_textures.textures.request", int64(1)).Once() + suite.Logger.On("RecordTimer", "mojang_textures.textures.request_time", mock.Anything).Once() + + result, err := suite.Provider.GetTextures("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") + + suite.Assert().Equal(expectedResult, result) + suite.Assert().Nil(err) +} + +func (suite *mojangApiTexturesProviderTestSuite) TestGetTexturesWithError() { + expectedError := &mojang.TooManyRequestsError{} + suite.MojangApi.On("UuidToTextures", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", true).Once().Return(nil, expectedError) + + suite.Logger.On("IncCounter", "mojang_textures.textures.request", int64(1)).Once() + suite.Logger.On("RecordTimer", "mojang_textures.textures.request_time", mock.Anything).Once() + + result, err := suite.Provider.GetTextures("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") + + suite.Assert().Nil(result) + suite.Assert().Equal(expectedError, err) +} diff --git a/mojangtextures/mojang_textures.go b/mojangtextures/mojang_textures.go new file mode 100644 index 0000000..33919c6 --- /dev/null +++ b/mojangtextures/mojang_textures.go @@ -0,0 +1,225 @@ +package mojangtextures + +import ( + "errors" + "net" + "net/url" + "regexp" + "strings" + "sync" + "syscall" + "time" + + "github.com/mono83/slf/wd" + + "github.com/elyby/chrly/api/mojang" +) + +type broadcastResult struct { + textures *mojang.SignedTexturesResponse + error error +} + +type broadcaster struct { + lock sync.Mutex + listeners map[string][]chan *broadcastResult +} + +func createBroadcaster() *broadcaster { + return &broadcaster{ + listeners: make(map[string][]chan *broadcastResult), + } +} + +// Returns a boolean value, which will be true if the passed username didn't exist before +func (c *broadcaster) AddListener(username string, resultChan chan *broadcastResult) bool { + c.lock.Lock() + defer c.lock.Unlock() + + val, alreadyHasSource := c.listeners[username] + if alreadyHasSource { + c.listeners[username] = append(val, resultChan) + return false + } + + c.listeners[username] = []chan *broadcastResult{resultChan} + + return true +} + +func (c *broadcaster) BroadcastAndRemove(username string, result *broadcastResult) { + c.lock.Lock() + defer c.lock.Unlock() + + val, ok := c.listeners[username] + if !ok { + return + } + + for _, channel := range val { + go func(channel chan *broadcastResult) { + channel <- result + close(channel) + }(channel) + } + + delete(c.listeners, username) +} + +// https://help.mojang.com/customer/portal/articles/928638 +var allowedUsernamesRegex = regexp.MustCompile(`^[\w_]{3,16}$`) + +type UuidsProvider interface { + GetUuid(username string) (*mojang.ProfileInfo, error) +} + +type TexturesProvider interface { + GetTextures(uuid string) (*mojang.SignedTexturesResponse, error) +} + +type Provider struct { + UuidsProvider + TexturesProvider + Storage + Logger wd.Watchdog + + onFirstCall sync.Once + *broadcaster +} + +func (ctx *Provider) GetForUsername(username string) (*mojang.SignedTexturesResponse, error) { + ctx.onFirstCall.Do(func() { + ctx.broadcaster = createBroadcaster() + }) + + if !allowedUsernamesRegex.MatchString(username) { + ctx.Logger.IncCounter("mojang_textures.invalid_username", 1) + return nil, errors.New("invalid username") + } + + username = strings.ToLower(username) + ctx.Logger.IncCounter("mojang_textures.request", 1) + + uuid, err := ctx.Storage.GetUuid(username) + if err == nil && uuid == "" { + ctx.Logger.IncCounter("mojang_textures.usernames.cache_hit_nil", 1) + return nil, nil + } + + if uuid != "" { + ctx.Logger.IncCounter("mojang_textures.usernames.cache_hit", 1) + textures, err := ctx.Storage.GetTextures(uuid) + if err == nil { + ctx.Logger.IncCounter("mojang_textures.textures.cache_hit", 1) + return textures, nil + } + } + + resultChan := make(chan *broadcastResult) + isFirstListener := ctx.broadcaster.AddListener(username, resultChan) + if isFirstListener { + go ctx.getResultAndBroadcast(username, uuid) + } else { + ctx.Logger.IncCounter("mojang_textures.already_scheduled", 1) + } + + result := <-resultChan + + return result.textures, result.error +} + +func (ctx *Provider) getResultAndBroadcast(username string, uuid string) { + start := time.Now() + + result := ctx.getResult(username, uuid) + ctx.broadcaster.BroadcastAndRemove(username, result) + + ctx.Logger.RecordTimer("mojang_textures.result_time", time.Since(start)) +} + +func (ctx *Provider) getResult(username string, uuid string) *broadcastResult { + if uuid == "" { + profile, err := ctx.UuidsProvider.GetUuid(username) + if err != nil { + ctx.handleMojangApiResponseError(err, "usernames") + return &broadcastResult{nil, err} + } + + uuid = "" + if profile != nil { + uuid = profile.Id + } + + _ = ctx.Storage.StoreUuid(username, uuid) + + if uuid == "" { + ctx.Logger.IncCounter("mojang_textures.usernames.uuid_miss", 1) + return &broadcastResult{nil, nil} + } + + ctx.Logger.IncCounter("mojang_textures.usernames.uuid_hit", 1) + } + + textures, err := ctx.TexturesProvider.GetTextures(uuid) + if err != nil { + ctx.handleMojangApiResponseError(err, "textures") + return &broadcastResult{nil, err} + } + + // Mojang can respond with an error, but it will still count as a hit, + // therefore store the result even if textures is nil to prevent 429 error + ctx.Storage.StoreTextures(uuid, textures) + + if textures != nil { + ctx.Logger.IncCounter("mojang_textures.usernames.textures_hit", 1) + } else { + ctx.Logger.IncCounter("mojang_textures.usernames.textures_miss", 1) + } + + return &broadcastResult{textures, nil} +} + +func (ctx *Provider) handleMojangApiResponseError(err error, threadName string) { + errParam := wd.ErrParam(err) + threadParam := wd.NameParam(threadName) + + ctx.Logger.Debug(":name: Got response error :err", threadParam, errParam) + + switch err.(type) { + case mojang.ResponseError: + if _, ok := err.(*mojang.BadRequestError); ok { + ctx.Logger.Warning(":name: Got 400 Bad Request :err", threadParam, errParam) + return + } + + if _, ok := err.(*mojang.ForbiddenError); ok { + ctx.Logger.Warning(":name: Got 403 Forbidden :err", threadParam, errParam) + return + } + + if _, ok := err.(*mojang.TooManyRequestsError); ok { + ctx.Logger.Warning(":name: Got 429 Too Many Requests :err", threadParam, errParam) + return + } + + return + case net.Error: + if err.(net.Error).Timeout() { + return + } + + if _, ok := err.(*url.Error); ok { + return + } + + if opErr, ok := err.(*net.OpError); ok && (opErr.Op == "dial" || opErr.Op == "read") { + return + } + + if err == syscall.ECONNREFUSED { + return + } + } + + ctx.Logger.Emergency(":name: Unknown Mojang response error: :err", threadParam, errParam) +} diff --git a/mojangtextures/mojang_textures_test.go b/mojangtextures/mojang_textures_test.go new file mode 100644 index 0000000..fabdee6 --- /dev/null +++ b/mojangtextures/mojang_textures_test.go @@ -0,0 +1,439 @@ +package mojangtextures + +import ( + "errors" + "net" + "net/url" + "sync" + "syscall" + "testing" + "time" + + testify "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" + + "github.com/elyby/chrly/api/mojang" + mocks "github.com/elyby/chrly/tests" +) + +func TestBroadcaster(t *testing.T) { + t.Run("GetOrAppend", func(t *testing.T) { + t.Run("first call when username didn't exist before should return true", func(t *testing.T) { + assert := testify.New(t) + + broadcaster := createBroadcaster() + channel := make(chan *broadcastResult) + isFirstListener := broadcaster.AddListener("mock", channel) + + assert.True(isFirstListener) + listeners, ok := broadcaster.listeners["mock"] + assert.True(ok) + assert.Len(listeners, 1) + assert.Equal(channel, listeners[0]) + }) + + t.Run("subsequent calls should return false", func(t *testing.T) { + assert := testify.New(t) + + broadcaster := createBroadcaster() + channel1 := make(chan *broadcastResult) + isFirstListener := broadcaster.AddListener("mock", channel1) + + assert.True(isFirstListener) + + channel2 := make(chan *broadcastResult) + isFirstListener = broadcaster.AddListener("mock", channel2) + + assert.False(isFirstListener) + + channel3 := make(chan *broadcastResult) + isFirstListener = broadcaster.AddListener("mock", channel3) + + assert.False(isFirstListener) + }) + }) + + t.Run("BroadcastAndRemove", func(t *testing.T) { + t.Run("should broadcast to all listeners and remove the key", func(t *testing.T) { + assert := testify.New(t) + + broadcaster := createBroadcaster() + channel1 := make(chan *broadcastResult) + channel2 := make(chan *broadcastResult) + broadcaster.AddListener("mock", channel1) + broadcaster.AddListener("mock", channel2) + + result := &broadcastResult{} + broadcaster.BroadcastAndRemove("mock", result) + + assert.Equal(result, <-channel1) + assert.Equal(result, <-channel2) + + channel3 := make(chan *broadcastResult) + isFirstListener := broadcaster.AddListener("mock", channel3) + assert.True(isFirstListener) + }) + + t.Run("call on not exists username", func(t *testing.T) { + assert := testify.New(t) + + assert.NotPanics(func() { + broadcaster := createBroadcaster() + broadcaster.BroadcastAndRemove("mock", &broadcastResult{}) + }) + }) + }) +} + +type mockUuidsProvider struct { + mock.Mock +} + +func (m *mockUuidsProvider) GetUuid(username string) (*mojang.ProfileInfo, error) { + args := m.Called(username) + var result *mojang.ProfileInfo + if casted, ok := args.Get(0).(*mojang.ProfileInfo); ok { + result = casted + } + + return result, args.Error(1) +} + +type mockTexturesProvider struct { + mock.Mock +} + +func (m *mockTexturesProvider) GetTextures(uuid string) (*mojang.SignedTexturesResponse, error) { + args := m.Called(uuid) + var result *mojang.SignedTexturesResponse + if casted, ok := args.Get(0).(*mojang.SignedTexturesResponse); ok { + result = casted + } + + return result, args.Error(1) +} + +type mockStorage struct { + mock.Mock +} + +func (m *mockStorage) GetUuid(username string) (string, error) { + args := m.Called(username) + return args.String(0), args.Error(1) +} + +func (m *mockStorage) StoreUuid(username string, uuid string) error { + args := m.Called(username, uuid) + return args.Error(0) +} + +func (m *mockStorage) GetTextures(uuid string) (*mojang.SignedTexturesResponse, error) { + args := m.Called(uuid) + var result *mojang.SignedTexturesResponse + if casted, ok := args.Get(0).(*mojang.SignedTexturesResponse); ok { + result = casted + } + + return result, args.Error(1) +} + +func (m *mockStorage) StoreTextures(uuid string, textures *mojang.SignedTexturesResponse) { + m.Called(uuid, textures) +} + +type providerTestSuite struct { + suite.Suite + Provider *Provider + UuidsProvider *mockUuidsProvider + TexturesProvider *mockTexturesProvider + Storage *mockStorage + Logger *mocks.WdMock +} + +func (suite *providerTestSuite) SetupTest() { + suite.UuidsProvider = &mockUuidsProvider{} + suite.TexturesProvider = &mockTexturesProvider{} + suite.Storage = &mockStorage{} + suite.Logger = &mocks.WdMock{} + + suite.Provider = &Provider{ + UuidsProvider: suite.UuidsProvider, + TexturesProvider: suite.TexturesProvider, + Storage: suite.Storage, + Logger: suite.Logger, + } +} + +func (suite *providerTestSuite) TearDownTest() { + // time.Sleep(10 * time.Millisecond) // Add delay to let finish all goroutines before assert mocks calls + suite.UuidsProvider.AssertExpectations(suite.T()) + suite.TexturesProvider.AssertExpectations(suite.T()) + suite.Storage.AssertExpectations(suite.T()) + suite.Logger.AssertExpectations(suite.T()) +} + +func TestProvider(t *testing.T) { + suite.Run(t, new(providerTestSuite)) +} + +func (suite *providerTestSuite) TestGetForUsernameWithoutAnyCache() { + expectedResult := &mojang.SignedTexturesResponse{Id: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Name: "username"} + + suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Once() + suite.Logger.On("IncCounter", "mojang_textures.usernames.uuid_hit", int64(1)).Once() + suite.Logger.On("IncCounter", "mojang_textures.usernames.textures_hit", int64(1)).Once() + suite.Logger.On("RecordTimer", "mojang_textures.result_time", mock.Anything).Once() + + suite.Storage.On("GetUuid", "username").Once().Return("", &ValueNotFound{}) + suite.Storage.On("StoreUuid", "username", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").Once().Return(nil) + suite.Storage.On("StoreTextures", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", expectedResult).Once() + + suite.UuidsProvider.On("GetUuid", "username").Once().Return(&mojang.ProfileInfo{ + Id: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Name: "username", + }, nil) + suite.TexturesProvider.On("GetTextures", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").Once().Return(expectedResult, nil) + + result, err := suite.Provider.GetForUsername("username") + + suite.Assert().Nil(err) + suite.Assert().Equal(expectedResult, result) +} + +func (suite *providerTestSuite) TestGetForUsernameWithCachedUuid() { + expectedResult := &mojang.SignedTexturesResponse{Id: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Name: "username"} + + suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Once() + suite.Logger.On("IncCounter", "mojang_textures.usernames.cache_hit", int64(1)).Once() + suite.Logger.On("IncCounter", "mojang_textures.usernames.textures_hit", int64(1)).Once() + suite.Logger.On("RecordTimer", "mojang_textures.result_time", mock.Anything).Once() + + suite.Storage.On("GetUuid", "username").Once().Return("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", nil) + suite.Storage.On("GetTextures", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").Once().Return(nil, &ValueNotFound{}) + suite.Storage.On("StoreTextures", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", expectedResult).Once() + + suite.TexturesProvider.On("GetTextures", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").Return(expectedResult, nil) + + result, err := suite.Provider.GetForUsername("username") + + suite.Assert().Nil(err) + suite.Assert().Equal(expectedResult, result) +} + +func (suite *providerTestSuite) TestGetForUsernameWithFullyCachedResult() { + expectedResult := &mojang.SignedTexturesResponse{Id: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Name: "username"} + + suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Once() + suite.Logger.On("IncCounter", "mojang_textures.usernames.cache_hit", int64(1)).Once() + suite.Logger.On("IncCounter", "mojang_textures.textures.cache_hit", int64(1)).Once() + + suite.Storage.On("GetUuid", "username").Once().Return("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", nil) + suite.Storage.On("GetTextures", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").Once().Return(expectedResult, nil) + + result, err := suite.Provider.GetForUsername("username") + + suite.Assert().Nil(err) + suite.Assert().Equal(expectedResult, result) +} + +func (suite *providerTestSuite) TestGetForUsernameWithCachedUnknownUuid() { + suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Once() + suite.Logger.On("IncCounter", "mojang_textures.usernames.cache_hit_nil", int64(1)).Once() + + suite.Storage.On("GetUuid", "username").Once().Return("", nil) + + result, err := suite.Provider.GetForUsername("username") + + suite.Assert().Nil(result) + suite.Assert().Nil(err) +} + +func (suite *providerTestSuite) TestGetForUsernameWhichHasNoMojangAccount() { + suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Once() + suite.Logger.On("IncCounter", "mojang_textures.usernames.uuid_miss", int64(1)).Once() + suite.Logger.On("RecordTimer", "mojang_textures.result_time", mock.Anything).Once() + + suite.Storage.On("GetUuid", "username").Once().Return("", &ValueNotFound{}) + suite.Storage.On("StoreUuid", "username", "").Once().Return(nil) + + suite.UuidsProvider.On("GetUuid", "username").Once().Return(nil, nil) + + result, err := suite.Provider.GetForUsername("username") + + suite.Assert().Nil(err) + suite.Assert().Nil(result) +} + +func (suite *providerTestSuite) TestGetForUsernameWhichHasMojangAccountButHasNoMojangSkin() { + var expectedResult *mojang.SignedTexturesResponse + + suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Once() + suite.Logger.On("IncCounter", "mojang_textures.usernames.uuid_hit", int64(1)).Once() + suite.Logger.On("IncCounter", "mojang_textures.usernames.textures_miss", int64(1)).Once() + suite.Logger.On("RecordTimer", "mojang_textures.result_time", mock.Anything).Once() + + suite.Storage.On("GetUuid", "username").Once().Return("", &ValueNotFound{}) + suite.Storage.On("StoreUuid", "username", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").Once().Return(nil) + suite.Storage.On("StoreTextures", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", expectedResult).Once() + + suite.UuidsProvider.On("GetUuid", "username").Once().Return(&mojang.ProfileInfo{ + Id: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Name: "username", + }, nil) + suite.TexturesProvider.On("GetTextures", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").Once().Return(expectedResult, nil) + + result, err := suite.Provider.GetForUsername("username") + + suite.Assert().Equal(expectedResult, result) + suite.Assert().Nil(err) +} + +func (suite *providerTestSuite) TestGetForTheSameUsernames() { + expectedResult := &mojang.SignedTexturesResponse{Id: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Name: "username"} + + suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Twice() + suite.Logger.On("IncCounter", "mojang_textures.already_scheduled", int64(1)).Once() + suite.Logger.On("IncCounter", "mojang_textures.usernames.uuid_hit", int64(1)).Once() + suite.Logger.On("IncCounter", "mojang_textures.usernames.textures_hit", int64(1)).Once() + suite.Logger.On("RecordTimer", "mojang_textures.result_time", mock.Anything).Once() + + suite.Storage.On("GetUuid", "username").Twice().Return("", &ValueNotFound{}) + suite.Storage.On("StoreUuid", "username", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").Once().Return(nil) + suite.Storage.On("StoreTextures", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", expectedResult).Once() + + // If possible, than remove this .After call + suite.UuidsProvider.On("GetUuid", "username").Once().After(time.Millisecond).Return(&mojang.ProfileInfo{ + Id: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Name: "username", + }, nil) + suite.TexturesProvider.On("GetTextures", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").Once().Return(expectedResult, nil) + + results := make([]*mojang.SignedTexturesResponse, 2) + var wg sync.WaitGroup + for i := 0; i < 2; i++ { + wg.Add(1) + go func(i int) { + textures, _ := suite.Provider.GetForUsername("username") + results[i] = textures + wg.Done() + }(i) + } + wg.Wait() + + suite.Assert().Equal(expectedResult, results[0]) + suite.Assert().Equal(expectedResult, results[1]) +} + +func (suite *providerTestSuite) TestGetForNotAllowedMojangUsername() { + suite.Logger.On("IncCounter", "mojang_textures.invalid_username", int64(1)).Once() + + result, err := suite.Provider.GetForUsername("Not allowed") + suite.Assert().Error(err, "invalid username") + suite.Assert().Nil(result) +} + +type timeoutError struct { +} + +func (*timeoutError) Error() string { return "timeout error" } +func (*timeoutError) Timeout() bool { return true } +func (*timeoutError) Temporary() bool { return false } + +var expectedErrors = []error{ + &mojang.BadRequestError{}, + &mojang.ForbiddenError{}, + &mojang.TooManyRequestsError{}, + &mojang.ServerError{}, + &timeoutError{}, + &url.Error{Op: "GET", URL: "http://localhost"}, + &net.OpError{Op: "read"}, + &net.OpError{Op: "dial"}, + syscall.ECONNREFUSED, +} + +func (suite *providerTestSuite) TestShouldNotLogErrorWhenExpectedErrorReturnedFromUsernameToUuidRequest() { + suite.Logger.On("IncCounter", mock.Anything, mock.Anything) + suite.Logger.On("RecordTimer", mock.Anything, mock.Anything) + suite.Logger.On("Debug", ":name: Got response error :err", mock.Anything, mock.Anything).Times(len(expectedErrors)) + suite.Logger.On("Warning", ":name: Got 400 Bad Request :err", mock.Anything, mock.Anything).Once() + suite.Logger.On("Warning", ":name: Got 403 Forbidden :err", mock.Anything, mock.Anything).Once() + suite.Logger.On("Warning", ":name: Got 429 Too Many Requests :err", mock.Anything, mock.Anything).Once() + + suite.Storage.On("GetUuid", "username").Return("", &ValueNotFound{}) + + for _, err := range expectedErrors { + suite.UuidsProvider.On("GetUuid", "username").Once().Return(nil, err) + + result, err := suite.Provider.GetForUsername("username") + suite.Assert().Nil(result) + suite.Assert().NotNil(err) + suite.UuidsProvider.AssertExpectations(suite.T()) + suite.UuidsProvider.ExpectedCalls = nil // https://github.com/stretchr/testify/issues/558#issuecomment-372112364 + } +} + +func (suite *providerTestSuite) TestShouldLogEmergencyOnUnexpectedErrorReturnedFromUsernameToUuidRequest() { + suite.Logger.On("IncCounter", mock.Anything, mock.Anything) + suite.Logger.On("RecordTimer", mock.Anything, mock.Anything) + suite.Logger.On("Debug", ":name: Got response error :err", mock.Anything, mock.Anything).Once() + suite.Logger.On("Emergency", ":name: Unknown Mojang response error: :err", mock.Anything, mock.Anything).Once() + + suite.Storage.On("GetUuid", "username").Return("", &ValueNotFound{}) + + suite.UuidsProvider.On("GetUuid", "username").Once().Return(nil, errors.New("unexpected error")) + + result, err := suite.Provider.GetForUsername("username") + suite.Assert().Nil(result) + suite.Assert().NotNil(err) +} + +func (suite *providerTestSuite) TestShouldNotLogErrorWhenExpectedErrorReturnedFromUuidToTexturesRequest() { + suite.Logger.On("IncCounter", mock.Anything, mock.Anything) + suite.Logger.On("RecordTimer", mock.Anything, mock.Anything) + suite.Logger.On("Debug", ":name: Got response error :err", mock.Anything, mock.Anything).Times(len(expectedErrors)) + suite.Logger.On("Warning", ":name: Got 400 Bad Request :err", mock.Anything, mock.Anything).Once() + suite.Logger.On("Warning", ":name: Got 403 Forbidden :err", mock.Anything, mock.Anything).Once() + suite.Logger.On("Warning", ":name: Got 429 Too Many Requests :err", mock.Anything, mock.Anything).Once() + + suite.Storage.On("GetUuid", "username").Return("", &ValueNotFound{}) + suite.Storage.On("StoreUuid", "username", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").Return(nil) + // suite.Storage.On("GetTextures", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").Return(nil, &ValueNotFound{}) + // suite.Storage.On("StoreTextures", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", (*mojang.SignedTexturesResponse)(nil)) + + for _, err := range expectedErrors { + suite.UuidsProvider.On("GetUuid", "username").Once().Return(&mojang.ProfileInfo{ + Id: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Name: "username", + }, nil) + suite.TexturesProvider.On("GetTextures", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").Once().Return(nil, err) + + result, err := suite.Provider.GetForUsername("username") + suite.Assert().Nil(result) + suite.Assert().NotNil(err) + suite.UuidsProvider.AssertExpectations(suite.T()) + suite.TexturesProvider.AssertExpectations(suite.T()) + suite.UuidsProvider.ExpectedCalls = nil // https://github.com/stretchr/testify/issues/558#issuecomment-372112364 + suite.TexturesProvider.ExpectedCalls = nil // https://github.com/stretchr/testify/issues/558#issuecomment-372112364 + } +} + +func (suite *providerTestSuite) TestShouldLogEmergencyOnUnexpectedErrorReturnedFromUuidToTexturesRequest() { + suite.Logger.On("IncCounter", mock.Anything, mock.Anything) + suite.Logger.On("RecordTimer", mock.Anything, mock.Anything) + suite.Logger.On("Debug", ":name: Got response error :err", mock.Anything, mock.Anything).Once() + suite.Logger.On("Emergency", ":name: Unknown Mojang response error: :err", mock.Anything, mock.Anything).Once() + + suite.Storage.On("GetUuid", "username").Return("", &ValueNotFound{}) + suite.Storage.On("StoreUuid", "username", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").Return(nil) + + suite.UuidsProvider.On("GetUuid", "username").Once().Return(&mojang.ProfileInfo{ + Id: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Name: "username", + }, nil) + suite.TexturesProvider.On("GetTextures", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").Once().Return(nil, errors.New("unexpected error")) + + result, err := suite.Provider.GetForUsername("username") + suite.Assert().Nil(result) + suite.Assert().NotNil(err) +} diff --git a/mojangtextures/storage.go b/mojangtextures/storage.go new file mode 100644 index 0000000..d73249a --- /dev/null +++ b/mojangtextures/storage.go @@ -0,0 +1,61 @@ +package mojangtextures + +import ( + "github.com/elyby/chrly/api/mojang" +) + +// UuidsStorage is a key-value storage of Mojang usernames pairs to its UUIDs, +// used to reduce the load on the account information queue +type UuidsStorage interface { + // Since only primitive types are used in this method, you should return a special error ValueNotFound + // to return the information that no error has occurred and username does not have uuid + GetUuid(username string) (string, error) + // An empty uuid value can be passed if the corresponding account has not been found + StoreUuid(username string, uuid string) error +} + +// TexturesStorage is a Mojang's textures storage, used as a values cache to avoid 429 errors +type TexturesStorage interface { + // Error should not have nil value only if the repository failed to determine if there are any textures + // for this uuid or not at all. If there is information about the absence of textures, nil nil should be returned + GetTextures(uuid string) (*mojang.SignedTexturesResponse, error) + // The nil value can be passed when there are no textures for the corresponding uuid and we know about it + StoreTextures(uuid string, textures *mojang.SignedTexturesResponse) +} + +type Storage interface { + UuidsStorage + TexturesStorage +} + +// SeparatedStorage allows you to use separate storage engines to satisfy +// the Storage interface +type SeparatedStorage struct { + UuidsStorage + TexturesStorage +} + +func (s *SeparatedStorage) GetUuid(username string) (string, error) { + return s.UuidsStorage.GetUuid(username) +} + +func (s *SeparatedStorage) StoreUuid(username string, uuid string) error { + return s.UuidsStorage.StoreUuid(username, uuid) +} + +func (s *SeparatedStorage) GetTextures(uuid string) (*mojang.SignedTexturesResponse, error) { + return s.TexturesStorage.GetTextures(uuid) +} + +func (s *SeparatedStorage) StoreTextures(uuid string, textures *mojang.SignedTexturesResponse) { + s.TexturesStorage.StoreTextures(uuid, textures) +} + +// This error can be used to indicate, that requested +// value doesn't exists in the storage +type ValueNotFound struct { +} + +func (*ValueNotFound) Error() string { + return "value not found in the storage" +} diff --git a/api/mojang/queue/storage_test.go b/mojangtextures/storage_test.go similarity index 91% rename from api/mojang/queue/storage_test.go rename to mojangtextures/storage_test.go index b9ee64c..1b82349 100644 --- a/api/mojang/queue/storage_test.go +++ b/mojangtextures/storage_test.go @@ -1,4 +1,4 @@ -package queue +package mojangtextures import ( "github.com/elyby/chrly/api/mojang" @@ -41,11 +41,11 @@ func (m *texturesStorageMock) StoreTextures(uuid string, textures *mojang.Signed } func TestSplittedStorage(t *testing.T) { - createMockedStorage := func() (*SplittedStorage, *uuidsStorageMock, *texturesStorageMock) { + createMockedStorage := func() (*SeparatedStorage, *uuidsStorageMock, *texturesStorageMock) { uuidsStorage := &uuidsStorageMock{} texturesStorage := &texturesStorageMock{} - return &SplittedStorage{uuidsStorage, texturesStorage}, uuidsStorage, texturesStorage + return &SeparatedStorage{uuidsStorage, texturesStorage}, uuidsStorage, texturesStorage } t.Run("GetUuid", func(t *testing.T) { diff --git a/tests/mojang_textures_queue_mock.go b/tests/mojang_textures_queue_mock.go deleted file mode 100644 index 0494243..0000000 --- a/tests/mojang_textures_queue_mock.go +++ /dev/null @@ -1,33 +0,0 @@ -package tests - -import ( - "github.com/elyby/chrly/api/mojang" - - "github.com/stretchr/testify/mock" -) - -type MojangTexturesQueueMock struct { - mock.Mock -} - -func (m *MojangTexturesQueueMock) GetTexturesForUsername(username string) chan *mojang.SignedTexturesResponse { - args := m.Called(username) - result := make(chan *mojang.SignedTexturesResponse) - arg := args.Get(0) - switch arg.(type) { - case *mojang.SignedTexturesResponse: - go func() { - result <- arg.(*mojang.SignedTexturesResponse) - }() - case chan *mojang.SignedTexturesResponse: - return arg.(chan *mojang.SignedTexturesResponse) - case nil: - go func() { - result <- nil - }() - default: - panic("unsupported return value") - } - - return result -} From d27caa4922d35f99b955c50a9d375e39a21bc672 Mon Sep 17 00:00:00 2001 From: ErickSkrauch Date: Thu, 21 Nov 2019 19:33:05 +0300 Subject: [PATCH 02/12] Add sync channel to batch_uuids_provider_test --- mojangtextures/batch_uuids_provider_test.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/mojangtextures/batch_uuids_provider_test.go b/mojangtextures/batch_uuids_provider_test.go index f217dd0..e2a614c 100644 --- a/mojangtextures/batch_uuids_provider_test.go +++ b/mojangtextures/batch_uuids_provider_test.go @@ -118,7 +118,9 @@ func (suite *batchUuidsProviderTestSuite) SetupTest() { suite.GetUuidAsync = func(username string) chan *batchUuidsProviderGetUuidResult { c := make(chan *batchUuidsProviderGetUuidResult) + s := make(chan int) go func() { + s <- 0 profile, err := suite.Provider.GetUuid(username) c <- &batchUuidsProviderGetUuidResult{ Result: profile, @@ -126,6 +128,8 @@ func (suite *batchUuidsProviderTestSuite) SetupTest() { } }() + <-s + return c } @@ -178,9 +182,7 @@ func (suite *batchUuidsProviderTestSuite) TestGetUuidForTwoUsernames() { }, nil) resultChan1 := suite.GetUuidAsync("username1") - time.Sleep(time.Millisecond) // Just to keep order for the usernames resultChan2 := suite.GetUuidAsync("username2") - time.Sleep(time.Millisecond) // Allow to all goroutines begin suite.Iterate() @@ -212,7 +214,6 @@ func (suite *batchUuidsProviderTestSuite) TestGetUuidForMoreThan10Usernames() { channels := make([]chan *batchUuidsProviderGetUuidResult, 12) for i, username := range usernames { channels[i] = suite.GetUuidAsync(username) - time.Sleep(time.Millisecond) // Just to keep order for the usernames } suite.Iterate() @@ -257,9 +258,7 @@ func (suite *batchUuidsProviderTestSuite) TestGetUuidForTwoUsernamesWithAnError( suite.MojangApi.On("UsernamesToUuids", []string{"username1", "username2"}).Once().Return(nil, expectedError) resultChan1 := suite.GetUuidAsync("username1") - time.Sleep(time.Millisecond) // Just to keep order for the usernames resultChan2 := suite.GetUuidAsync("username2") - time.Sleep(time.Millisecond) // Allow to all goroutines begin suite.Iterate() From 1033069211b1a0cb3d264f4b17ea87f3cc6c23b1 Mon Sep 17 00:00:00 2001 From: ErickSkrauch Date: Sun, 24 Nov 2019 04:07:56 +0300 Subject: [PATCH 03/12] Implemented remote api mojang uuids provider --- CHANGELOG.md | 1 + cmd/serve.go | 32 +++- mojangtextures/remote_api_uuids_provider.go | 71 +++++++++ .../remote_api_uuids_provider_test.go | 149 ++++++++++++++++++ 4 files changed, 247 insertions(+), 6 deletions(-) create mode 100644 mojangtextures/remote_api_uuids_provider.go create mode 100644 mojangtextures/remote_api_uuids_provider_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 450d8e7..6867097 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] - xxxx-xx-xx ### Added +- Added remote mode for Mojang's textures queue. - New StatsD metrics: - Counters: - `ely.skinsystem.{hostname}.app.mojang_textures.usernames.textures_hit` diff --git a/cmd/serve.go b/cmd/serve.go index 2ca852d..e06a66b 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -3,8 +3,10 @@ package cmd import ( "fmt" "log" + "net/url" "time" + "github.com/mono83/slf/wd" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -19,6 +21,7 @@ var serveCmd = &cobra.Command{ Use: "serve", Short: "Starts http handler for the skins system", Run: func(cmd *cobra.Command, args []string) { + // TODO: this is a mess, need to organize this code somehow to make services initialization more compact logger, err := bootstrap.CreateLogger(viper.GetString("statsd.addr"), viper.GetString("sentry.dsn")) if err != nil { log.Fatal(fmt.Printf("Cannot initialize logger: %v", err)) @@ -52,15 +55,32 @@ var serveCmd = &cobra.Command{ return } - texturesStorage := mojangtextures.NewInMemoryTexturesStorage() - texturesStorage.Start() - mojangTexturesProvider := &mojangtextures.Provider{ - Logger: logger, - UuidsProvider: &mojangtextures.BatchUuidsProvider{ + var uuidsProvider mojangtextures.UuidsProvider + preferredUuidsProvider := viper.GetString("mojang_textures.uuids_provider.driver") + if preferredUuidsProvider == "remote" { + remoteUrl, err := url.Parse(viper.GetString("mojang_textures.uuids_provider.url")) + if err != nil { + logger.Emergency("Unable to parse remote url :err", wd.ErrParam(err)) + return + } + + uuidsProvider = &mojangtextures.RemoteApiUuidsProvider{ + Url: *remoteUrl, + Logger: logger, + } + } else { + uuidsProvider = &mojangtextures.BatchUuidsProvider{ IterationDelay: time.Duration(viper.GetInt("queue.loop_delay")) * time.Millisecond, IterationSize: viper.GetInt("queue.batch_size"), Logger: logger, - }, + } + } + + texturesStorage := mojangtextures.NewInMemoryTexturesStorage() + texturesStorage.Start() + mojangTexturesProvider := &mojangtextures.Provider{ + Logger: logger, + UuidsProvider: uuidsProvider, TexturesProvider: &mojangtextures.MojangApiTexturesProvider{ Logger: logger, }, diff --git a/mojangtextures/remote_api_uuids_provider.go b/mojangtextures/remote_api_uuids_provider.go new file mode 100644 index 0000000..0980176 --- /dev/null +++ b/mojangtextures/remote_api_uuids_provider.go @@ -0,0 +1,71 @@ +package mojangtextures + +import ( + "encoding/json" + "io/ioutil" + "net/http" + . "net/url" + "path" + "time" + + "github.com/mono83/slf/wd" + + "github.com/elyby/chrly/api/mojang" + "github.com/elyby/chrly/bootstrap" +) + +var HttpClient = &http.Client{ + Transport: &http.Transport{ + MaxIdleConnsPerHost: 1024, + }, +} + +type RemoteApiUuidsProvider struct { + Url URL + Logger wd.Watchdog +} + +func (ctx *RemoteApiUuidsProvider) GetUuid(username string) (*mojang.ProfileInfo, error) { + ctx.Logger.IncCounter("mojang_textures.usernames.request", 1) + + url := ctx.Url + url.Path = path.Join(url.Path, username) + + request, _ := http.NewRequest("GET", url.String(), nil) + request.Header.Add("Accept", "application/json") + // Change default User-Agent to allow specify "Username -> UUID at time" Mojang's api endpoint + request.Header.Add("User-Agent", "Chrly/"+bootstrap.GetVersion()) + + start := time.Now() + response, err := HttpClient.Do(request) + ctx.Logger.RecordTimer("mojang_textures.usernames.request_time", time.Since(start)) + if err != nil { + return nil, err + } + defer response.Body.Close() + + if response.StatusCode == 204 { + return nil, nil + } + + if response.StatusCode != 200 { + return nil, &UnexpectedRemoteApiResponse{response} + } + + var result *mojang.ProfileInfo + body, _ := ioutil.ReadAll(response.Body) + err = json.Unmarshal(body, &result) + if err != nil { + return nil, err + } + + return result, nil +} + +type UnexpectedRemoteApiResponse struct { + Response *http.Response +} + +func (*UnexpectedRemoteApiResponse) Error() string { + return "Unexpected remote api response" +} diff --git a/mojangtextures/remote_api_uuids_provider_test.go b/mojangtextures/remote_api_uuids_provider_test.go new file mode 100644 index 0000000..b47edca --- /dev/null +++ b/mojangtextures/remote_api_uuids_provider_test.go @@ -0,0 +1,149 @@ +package mojangtextures + +import ( + "net" + "net/http" + "net/url" + "testing" + + "github.com/h2non/gock" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" + + mocks "github.com/elyby/chrly/tests" +) + +type remoteApiUuidsProviderTestSuite struct { + suite.Suite + + Provider *RemoteApiUuidsProvider + Logger *mocks.WdMock +} + +func (suite *remoteApiUuidsProviderTestSuite) SetupSuite() { + client := &http.Client{} + gock.InterceptClient(client) + + HttpClient = client +} + +func (suite *remoteApiUuidsProviderTestSuite) SetupTest() { + suite.Logger = &mocks.WdMock{} + suite.Provider = &RemoteApiUuidsProvider{ + Logger: suite.Logger, + } +} + +func (suite *remoteApiUuidsProviderTestSuite) TearDownTest() { + suite.Logger.AssertExpectations(suite.T()) + gock.Off() +} + +func TestRemoteApiUuidsProvider(t *testing.T) { + suite.Run(t, new(remoteApiUuidsProviderTestSuite)) +} + +func (suite *remoteApiUuidsProviderTestSuite) TestGetUuidForValidUsername() { + suite.Logger.On("IncCounter", "mojang_textures.usernames.request", int64(1)).Once() + suite.Logger.On("RecordTimer", "mojang_textures.usernames.request_time", mock.Anything).Once() + + gock.New("http://example.com"). + Get("/subpath/username"). + Reply(200). + JSON(map[string]interface{}{ + "id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "name": "username", + }) + + suite.Provider.Url = shouldParseUrl("http://example.com/subpath") + result, err := suite.Provider.GetUuid("username") + + assert := suite.Assert() + if assert.NoError(err) { + assert.Equal("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", result.Id) + assert.Equal("username", result.Name) + assert.False(result.IsLegacy) + assert.False(result.IsDemo) + } +} + +func (suite *remoteApiUuidsProviderTestSuite) TestGetUuidForNotExistsUsername() { + suite.Logger.On("IncCounter", "mojang_textures.usernames.request", int64(1)).Once() + suite.Logger.On("RecordTimer", "mojang_textures.usernames.request_time", mock.Anything).Once() + + gock.New("http://example.com"). + Get("/subpath/username"). + Reply(204) + + suite.Provider.Url = shouldParseUrl("http://example.com/subpath") + result, err := suite.Provider.GetUuid("username") + + assert := suite.Assert() + assert.Nil(result) + assert.Nil(err) +} + +func (suite *remoteApiUuidsProviderTestSuite) TestGetUuidForNon20xResponse() { + suite.Logger.On("IncCounter", "mojang_textures.usernames.request", int64(1)).Once() + suite.Logger.On("RecordTimer", "mojang_textures.usernames.request_time", mock.Anything).Once() + + gock.New("http://example.com"). + Get("/subpath/username"). + Reply(504). + BodyString("504 Gateway Timeout") + + suite.Provider.Url = shouldParseUrl("http://example.com/subpath") + result, err := suite.Provider.GetUuid("username") + + assert := suite.Assert() + assert.Nil(result) + assert.EqualError(err, "Unexpected remote api response") +} + +func (suite *remoteApiUuidsProviderTestSuite) TestGetUuidForNotSuccessRequest() { + suite.Logger.On("IncCounter", "mojang_textures.usernames.request", int64(1)).Once() + suite.Logger.On("RecordTimer", "mojang_textures.usernames.request_time", mock.Anything).Once() + + expectedError := &net.OpError{Op: "dial"} + + gock.New("http://example.com"). + Get("/subpath/username"). + ReplyError(expectedError) + + suite.Provider.Url = shouldParseUrl("http://example.com/subpath") + result, err := suite.Provider.GetUuid("username") + + assert := suite.Assert() + assert.Nil(result) + if assert.Error(err) { + assert.IsType(&url.Error{}, err) + casterErr, _ := err.(*url.Error) + assert.Equal(expectedError, casterErr.Err) + } +} + +func (suite *remoteApiUuidsProviderTestSuite) TestGetUuidForInvalidSuccessResponse() { + suite.Logger.On("IncCounter", "mojang_textures.usernames.request", int64(1)).Once() + suite.Logger.On("RecordTimer", "mojang_textures.usernames.request_time", mock.Anything).Once() + + gock.New("http://example.com"). + Get("/subpath/username"). + Reply(200). + BodyString("completely not json") + + suite.Provider.Url = shouldParseUrl("http://example.com/subpath") + result, err := suite.Provider.GetUuid("username") + + assert := suite.Assert() + assert.Nil(result) + assert.Error(err) +} + +func shouldParseUrl(rawUrl string) url.URL { + url, err := url.Parse(rawUrl) + if err != nil { + panic(err) + } + + return *url +} From 1e91aef0a657517a519492992bb96a1706d66144 Mon Sep 17 00:00:00 2001 From: ErickSkrauch Date: Wed, 1 Jan 2020 23:42:45 +0300 Subject: [PATCH 04/12] Rework http app structure, get rid of the golang/mock package, rewrite http tests --- Gopkg.lock | 9 - Gopkg.toml | 4 - cmd/serve.go | 2 +- db/commons.go | 16 - db/factory.go | 6 +- db/filesystem.go | 10 +- db/redis.go | 16 +- http/api.go | 259 ---- http/api_test.go | 501 -------- http/cape.go | 51 - http/cape_test.go | 163 --- http/http.go | 110 +- http/http_test.go | 108 +- http/not_found.go | 17 - http/not_found_test.go | 27 - http/signed_textures.go | 52 - http/signed_textures_test.go | 141 --- http/skin.go | 49 - http/skin_test.go | 158 --- http/skinsystem.go | 503 ++++++++ http/skinsystem_test.go | 1092 +++++++++++++++++ http/textures.go | 61 - http/textures_test.go | 194 --- interfaces/auth.go | 7 - interfaces/mock_interfaces/mock_auth.go | 45 - interfaces/mock_interfaces/mock_interfaces.go | 131 -- interfaces/mock_wd/mock_wd.go | 218 ---- interfaces/repositories.go | 22 - 28 files changed, 1669 insertions(+), 2303 deletions(-) delete mode 100644 http/api.go delete mode 100644 http/api_test.go delete mode 100644 http/cape.go delete mode 100644 http/cape_test.go delete mode 100644 http/not_found.go delete mode 100644 http/not_found_test.go delete mode 100644 http/signed_textures.go delete mode 100644 http/signed_textures_test.go delete mode 100644 http/skin.go delete mode 100644 http/skin_test.go create mode 100644 http/skinsystem.go create mode 100644 http/skinsystem_test.go delete mode 100644 http/textures.go delete mode 100644 http/textures_test.go delete mode 100644 interfaces/auth.go delete mode 100644 interfaces/mock_interfaces/mock_auth.go delete mode 100644 interfaces/mock_interfaces/mock_interfaces.go delete mode 100644 interfaces/mock_wd/mock_wd.go delete mode 100644 interfaces/repositories.go diff --git a/Gopkg.lock b/Gopkg.lock index 5b022fc..8416f5d 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -46,14 +46,6 @@ pruneopts = "" revision = "919484f041ea21e7e27be291cee1d6af7bc98864" -[[projects]] - digest = "1:530233672f656641b365f8efb38ed9fba80e420baff2ce87633813ab3755ed6d" - name = "github.com/golang/mock" - packages = ["gomock"] - pruneopts = "" - revision = "51421b967af1f557f93a59e0057aaf15ca02e29c" - version = "v1.2.0" - [[projects]] digest = "1:65c7ed49d9f36dd4752e43013323fa9229db60b29aa4f5a75aaecda3130c74e2" name = "github.com/gorilla/mux" @@ -310,7 +302,6 @@ "github.com/SermoDigital/jose/crypto", "github.com/SermoDigital/jose/jws", "github.com/getsentry/raven-go", - "github.com/golang/mock/gomock", "github.com/gorilla/mux", "github.com/h2non/gock", "github.com/mediocregopher/radix.v2/pool", diff --git a/Gopkg.toml b/Gopkg.toml index dd2f585..a260730 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -42,10 +42,6 @@ ignored = ["github.com/elyby/chrly"] name = "github.com/stretchr/testify" version = "^1.3.0" -[[constraint]] - name = "github.com/golang/mock" - version = "^1.0.0" - [[constraint]] name = "github.com/h2non/gock" version = "^1.0.6" diff --git a/cmd/serve.go b/cmd/serve.go index e06a66b..481e219 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -91,7 +91,7 @@ var serveCmd = &cobra.Command{ } logger.Info("Mojang's textures queue is successfully initialized") - cfg := &http.Config{ + cfg := &http.Skinsystem{ ListenSpec: fmt.Sprintf("%s:%d", viper.GetString("server.host"), viper.GetInt("server.port")), SkinsRepo: skinsRepo, CapesRepo: capesRepo, diff --git a/db/commons.go b/db/commons.go index 55be43e..136137c 100644 --- a/db/commons.go +++ b/db/commons.go @@ -7,19 +7,3 @@ type ParamRequired struct { func (e ParamRequired) Error() string { return "Required parameter not provided" } - -type SkinNotFoundError struct { - Who string -} - -func (e SkinNotFoundError) Error() string { - return "Skin data not found." -} - -type CapeNotFoundError struct { - Who string -} - -func (e CapeNotFoundError) Error() string { - return "Cape file not found." -} diff --git a/db/factory.go b/db/factory.go index 337d8bf..03c9923 100644 --- a/db/factory.go +++ b/db/factory.go @@ -1,9 +1,9 @@ package db import ( + "github.com/elyby/chrly/http" "github.com/spf13/viper" - "github.com/elyby/chrly/interfaces" "github.com/elyby/chrly/mojangtextures" ) @@ -12,8 +12,8 @@ type StorageFactory struct { } type RepositoriesCreator interface { - CreateSkinsRepository() (interfaces.SkinsRepository, error) - CreateCapesRepository() (interfaces.CapesRepository, error) + CreateSkinsRepository() (http.SkinsRepository, error) + CreateCapesRepository() (http.CapesRepository, error) CreateMojangUuidsRepository() (mojangtextures.UuidsStorage, error) } diff --git a/db/filesystem.go b/db/filesystem.go index 4a996c3..a9c9030 100644 --- a/db/filesystem.go +++ b/db/filesystem.go @@ -1,11 +1,11 @@ package db import ( + "github.com/elyby/chrly/http" "os" "path" "strings" - "github.com/elyby/chrly/interfaces" "github.com/elyby/chrly/model" "github.com/elyby/chrly/mojangtextures" ) @@ -15,11 +15,11 @@ type FilesystemFactory struct { CapesDirName string } -func (f FilesystemFactory) CreateSkinsRepository() (interfaces.SkinsRepository, error) { +func (f FilesystemFactory) CreateSkinsRepository() (http.SkinsRepository, error) { panic("skins repository not supported for this storage type") } -func (f FilesystemFactory) CreateCapesRepository() (interfaces.CapesRepository, error) { +func (f FilesystemFactory) CreateCapesRepository() (http.CapesRepository, error) { if err := f.validateFactoryConfig(); err != nil { return nil, err } @@ -49,13 +49,13 @@ type filesStorage struct { func (repository *filesStorage) FindByUsername(username string) (*model.Cape, error) { if username == "" { - return nil, &CapeNotFoundError{username} + return nil, &http.CapeNotFoundError{username} } capePath := path.Join(repository.path, strings.ToLower(username)+".png") file, err := os.Open(capePath) if err != nil { - return nil, &CapeNotFoundError{username} + return nil, &http.CapeNotFoundError{username} } return &model.Cape{ diff --git a/db/redis.go b/db/redis.go index cb657b0..d07e5b6 100644 --- a/db/redis.go +++ b/db/redis.go @@ -5,6 +5,7 @@ import ( "compress/zlib" "encoding/json" "fmt" + "github.com/elyby/chrly/http" "io" "strconv" "strings" @@ -14,7 +15,6 @@ import ( "github.com/mediocregopher/radix.v2/redis" "github.com/mediocregopher/radix.v2/util" - "github.com/elyby/chrly/interfaces" "github.com/elyby/chrly/model" "github.com/elyby/chrly/mojangtextures" ) @@ -26,11 +26,11 @@ type RedisFactory struct { pool *pool.Pool } -func (f *RedisFactory) CreateSkinsRepository() (interfaces.SkinsRepository, error) { +func (f *RedisFactory) CreateSkinsRepository() (http.SkinsRepository, error) { return f.createInstance() } -func (f *RedisFactory) CreateCapesRepository() (interfaces.CapesRepository, error) { +func (f *RedisFactory) CreateCapesRepository() (http.CapesRepository, error) { panic("capes repository not supported for this storage type") } @@ -148,13 +148,13 @@ func (db *redisDb) StoreUuid(username string, uuid string) error { func findByUsername(username string, conn util.Cmder) (*model.Skin, error) { if username == "" { - return nil, &SkinNotFoundError{username} + return nil, &http.SkinNotFoundError{username} } redisKey := buildUsernameKey(username) response := conn.Cmd("GET", redisKey) if !response.IsType(redis.Str) { - return nil, &SkinNotFoundError{username} + return nil, &http.SkinNotFoundError{username} } encodedResult, err := response.Bytes() @@ -181,7 +181,7 @@ func findByUsername(username string, conn util.Cmder) (*model.Skin, error) { func findByUserId(id int, conn util.Cmder) (*model.Skin, error) { response := conn.Cmd("HGET", accountIdToUsernameKey, id) if !response.IsType(redis.Str) { - return nil, &SkinNotFoundError{"unknown"} + return nil, &http.SkinNotFoundError{"unknown"} } username, _ := response.Str() @@ -192,7 +192,7 @@ func findByUserId(id int, conn util.Cmder) (*model.Skin, error) { func removeByUserId(id int, conn util.Cmder) error { record, err := findByUserId(id, conn) if err != nil { - if _, ok := err.(*SkinNotFoundError); !ok { + if _, ok := err.(*http.SkinNotFoundError); !ok { return err } } @@ -212,7 +212,7 @@ func removeByUserId(id int, conn util.Cmder) error { func removeByUsername(username string, conn util.Cmder) error { record, err := findByUsername(username, conn) if err != nil { - if _, ok := err.(*SkinNotFoundError); ok { + if _, ok := err.(*http.SkinNotFoundError); ok { return nil } diff --git a/http/api.go b/http/api.go deleted file mode 100644 index 9a41401..0000000 --- a/http/api.go +++ /dev/null @@ -1,259 +0,0 @@ -package http - -import ( - "encoding/json" - "errors" - "fmt" - "net/http" - "regexp" - "strconv" - - "github.com/elyby/chrly/auth" - "github.com/elyby/chrly/db" - "github.com/elyby/chrly/interfaces" - "github.com/elyby/chrly/model" - - "github.com/gorilla/mux" - "github.com/mono83/slf/wd" - "github.com/thedevsaddam/govalidator" -) - -//noinspection GoSnakeCaseUsage -const UUID_ANY = "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$" - -var regexUuidAny = regexp.MustCompile(UUID_ANY) - -func init() { - govalidator.AddCustomRule("skinUploadingNotAvailable", func(field string, rule string, message string, value interface{}) error { - if message == "" { - message = "Skin uploading is temporary unavailable" - } - - return errors.New(message) - }) - - // Add ability to validate any possible uuid form - govalidator.AddCustomRule("uuid_any", func(field string, rule string, message string, value interface{}) error { - str := value.(string) - if !regexUuidAny.MatchString(str) { - if message == "" { - message = fmt.Sprintf("The %s field must contain valid UUID", field) - } - - return errors.New(message) - } - - return nil - }) -} - -func (cfg *Config) PostSkin(resp http.ResponseWriter, req *http.Request) { - cfg.Logger.IncCounter("api.skins.post.request", 1) - validationErrors := validatePostSkinRequest(req) - if validationErrors != nil { - cfg.Logger.IncCounter("api.skins.post.validation_failed", 1) - apiBadRequest(resp, validationErrors) - return - } - - identityId, _ := strconv.Atoi(req.Form.Get("identityId")) - username := req.Form.Get("username") - - record, err := findIdentity(cfg.SkinsRepo, identityId, username) - if err != nil { - cfg.Logger.Error("Error on requesting a skin from the repository: :err", wd.ErrParam(err)) - apiServerError(resp) - return - } - - skinId, _ := strconv.Atoi(req.Form.Get("skinId")) - is18, _ := strconv.ParseBool(req.Form.Get("is1_8")) - isSlim, _ := strconv.ParseBool(req.Form.Get("isSlim")) - - record.Uuid = req.Form.Get("uuid") - record.SkinId = skinId - record.Is1_8 = is18 - record.IsSlim = isSlim - record.Url = req.Form.Get("url") - record.MojangTextures = req.Form.Get("mojangTextures") - record.MojangSignature = req.Form.Get("mojangSignature") - - err = cfg.SkinsRepo.Save(record) - if err != nil { - cfg.Logger.Error("Unable to save record to the repository: :err", wd.ErrParam(err)) - apiServerError(resp) - return - } - - cfg.Logger.IncCounter("api.skins.post.success", 1) - resp.WriteHeader(http.StatusCreated) -} - -func (cfg *Config) DeleteSkinByUserId(resp http.ResponseWriter, req *http.Request) { - cfg.Logger.IncCounter("api.skins.delete.request", 1) - id, _ := strconv.Atoi(mux.Vars(req)["id"]) - skin, err := cfg.SkinsRepo.FindByUserId(id) - if err != nil { - cfg.Logger.IncCounter("api.skins.delete.not_found", 1) - apiNotFound(resp, "Cannot find record for requested user id") - return - } - - cfg.deleteSkin(skin, resp) -} - -func (cfg *Config) DeleteSkinByUsername(resp http.ResponseWriter, req *http.Request) { - cfg.Logger.IncCounter("api.skins.delete.request", 1) - username := mux.Vars(req)["username"] - skin, err := cfg.SkinsRepo.FindByUsername(username) - if err != nil { - cfg.Logger.IncCounter("api.skins.delete.not_found", 1) - apiNotFound(resp, "Cannot find record for requested username") - return - } - - cfg.deleteSkin(skin, resp) -} - -func (cfg *Config) AuthenticationMiddleware(handler http.Handler) http.Handler { - return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { - cfg.Logger.IncCounter("authentication.challenge", 1) - err := cfg.Auth.Check(req) - if err != nil { - if _, ok := err.(*auth.Unauthorized); ok { - cfg.Logger.IncCounter("authentication.failed", 1) - apiForbidden(resp, err.Error()) - } else { - cfg.Logger.Error("Unknown error on validating api request: :err", wd.ErrParam(err)) - apiServerError(resp) - } - - return - } - - cfg.Logger.IncCounter("authentication.success", 1) - handler.ServeHTTP(resp, req) - }) -} - -func (cfg *Config) deleteSkin(skin *model.Skin, resp http.ResponseWriter) { - err := cfg.SkinsRepo.RemoveByUserId(skin.UserId) - if err != nil { - cfg.Logger.Error("Cannot delete skin by error: :err", wd.ErrParam(err)) - apiServerError(resp) - return - } - - cfg.Logger.IncCounter("api.skins.delete.success", 1) - resp.WriteHeader(http.StatusNoContent) -} - -func validatePostSkinRequest(request *http.Request) map[string][]string { - const maxMultipartMemory int64 = 32 << 20 - const oneOfSkinOrUrlMessage = "One of url or skin should be provided, but not both" - - _ = request.ParseMultipartForm(maxMultipartMemory) - - validationRules := govalidator.MapData{ - "identityId": {"required", "numeric", "min:1"}, - "username": {"required"}, - "uuid": {"required", "uuid_any"}, - "skinId": {"required", "numeric", "min:1"}, - "url": {"url"}, - "file:skin": {"ext:png", "size:24576", "mime:image/png"}, - "is1_8": {"bool"}, - "isSlim": {"bool"}, - } - - shouldAppendSkinRequiredError := false - url := request.Form.Get("url") - _, _, skinErr := request.FormFile("skin") - if (url != "" && skinErr == nil) || (url == "" && skinErr != nil) { - shouldAppendSkinRequiredError = true - } else if skinErr == nil { - validationRules["file:skin"] = append(validationRules["file:skin"], "skinUploadingNotAvailable") - } else if url != "" { - validationRules["is1_8"] = append(validationRules["is1_8"], "required") - validationRules["isSlim"] = append(validationRules["isSlim"], "required") - } - - mojangTextures := request.Form.Get("mojangTextures") - if mojangTextures != "" { - validationRules["mojangSignature"] = []string{"required"} - } - - validator := govalidator.New(govalidator.Options{ - Request: request, - Rules: validationRules, - RequiredDefault: false, - FormSize: maxMultipartMemory, - }) - validationResults := validator.Validate() - if shouldAppendSkinRequiredError { - validationResults["url"] = append(validationResults["url"], oneOfSkinOrUrlMessage) - validationResults["skin"] = append(validationResults["skin"], oneOfSkinOrUrlMessage) - } - - if len(validationResults) != 0 { - return validationResults - } - - return nil -} - -func findIdentity(repo interfaces.SkinsRepository, identityId int, username string) (*model.Skin, error) { - var record *model.Skin - record, err := repo.FindByUserId(identityId) - if err != nil { - if _, isSkinNotFound := err.(*db.SkinNotFoundError); !isSkinNotFound { - return nil, err - } - - record, err = repo.FindByUsername(username) - if err == nil { - _ = repo.RemoveByUsername(username) - record.UserId = identityId - } else { - record = &model.Skin{ - UserId: identityId, - Username: username, - } - } - } else if record.Username != username { - _ = repo.RemoveByUserId(identityId) - record.Username = username - } - - return record, nil -} - -func apiBadRequest(resp http.ResponseWriter, errorsPerField map[string][]string) { - resp.WriteHeader(http.StatusBadRequest) - resp.Header().Set("Content-Type", "application/json") - result, _ := json.Marshal(map[string]interface{}{ - "errors": errorsPerField, - }) - _, _ = resp.Write(result) -} - -func apiForbidden(resp http.ResponseWriter, reason string) { - resp.WriteHeader(http.StatusForbidden) - resp.Header().Set("Content-Type", "application/json") - result, _ := json.Marshal(map[string]interface{}{ - "error": reason, - }) - _, _ = resp.Write(result) -} - -func apiNotFound(resp http.ResponseWriter, reason string) { - resp.WriteHeader(http.StatusNotFound) - resp.Header().Set("Content-Type", "application/json") - result, _ := json.Marshal([]interface{}{ - reason, - }) - _, _ = resp.Write(result) -} - -func apiServerError(resp http.ResponseWriter) { - resp.WriteHeader(http.StatusInternalServerError) -} diff --git a/http/api_test.go b/http/api_test.go deleted file mode 100644 index 01cd917..0000000 --- a/http/api_test.go +++ /dev/null @@ -1,501 +0,0 @@ -package http - -import ( - "bytes" - "encoding/base64" - "io/ioutil" - "mime/multipart" - "net/http" - "net/http/httptest" - "net/url" - "testing" - - "github.com/elyby/chrly/auth" - "github.com/elyby/chrly/db" - - "github.com/golang/mock/gomock" - testify "github.com/stretchr/testify/assert" -) - -func TestConfig_PostSkin(t *testing.T) { - t.Run("Upload new identity with textures info", func(t *testing.T) { - assert := testify.New(t) - - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - config, mocks := setupMocks(ctrl) - - resultModel := createSkinModel("mock_user", false) - resultModel.SkinId = 5 - resultModel.Url = "http://example.com/skin.png" - resultModel.MojangTextures = "" - resultModel.MojangSignature = "" - - form := url.Values{ - "identityId": {"1"}, - "username": {"mock_user"}, - "uuid": {"0f657aa8-bfbe-415d-b700-5750090d3af3"}, - "skinId": {"5"}, - "is1_8": {"0"}, - "isSlim": {"0"}, - "url": {"http://example.com/skin.png"}, - } - - req := httptest.NewRequest("POST", "http://chrly/api/skins", bytes.NewBufferString(form.Encode())) - req.Header.Add("Content-Type", "application/x-www-form-urlencoded") - w := httptest.NewRecorder() - - mocks.Auth.EXPECT().Check(gomock.Any()).Return(nil) - mocks.Skins.EXPECT().FindByUserId(1).Return(nil, &db.SkinNotFoundError{Who: "unknown"}) - mocks.Skins.EXPECT().FindByUsername("mock_user").Return(nil, &db.SkinNotFoundError{Who: "mock_user"}) - mocks.Skins.EXPECT().Save(resultModel).Return(nil) - mocks.Log.EXPECT().IncCounter("authentication.challenge", int64(1)) - mocks.Log.EXPECT().IncCounter("authentication.success", int64(1)) - mocks.Log.EXPECT().IncCounter("api.skins.post.request", int64(1)) - mocks.Log.EXPECT().IncCounter("api.skins.post.success", int64(1)) - - config.CreateHandler().ServeHTTP(w, req) - - resp := w.Result() - defer resp.Body.Close() - assert.Equal(201, resp.StatusCode) - response, _ := ioutil.ReadAll(resp.Body) - assert.Empty(response) - }) - - t.Run("Upload new identity with skin file", func(t *testing.T) { - assert := testify.New(t) - - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - config, mocks := setupMocks(ctrl) - - body := &bytes.Buffer{} - writer := multipart.NewWriter(body) - - part, _ := writer.CreateFormFile("skin", "char.png") - _, _ = part.Write(loadSkinFile()) - - _ = writer.WriteField("identityId", "1") - _ = writer.WriteField("username", "mock_user") - _ = writer.WriteField("uuid", "0f657aa8-bfbe-415d-b700-5750090d3af3") - _ = writer.WriteField("skinId", "5") - - err := writer.Close() - if err != nil { - panic(err) - } - - req := httptest.NewRequest("POST", "http://chrly/api/skins", body) - req.Header.Add("Content-Type", writer.FormDataContentType()) - w := httptest.NewRecorder() - - mocks.Auth.EXPECT().Check(gomock.Any()).Return(nil) - mocks.Log.EXPECT().IncCounter("authentication.challenge", int64(1)) - mocks.Log.EXPECT().IncCounter("authentication.success", int64(1)) - mocks.Log.EXPECT().IncCounter("api.skins.post.request", int64(1)) - mocks.Log.EXPECT().IncCounter("api.skins.post.validation_failed", int64(1)) - - config.CreateHandler().ServeHTTP(w, req) - - resp := w.Result() - defer resp.Body.Close() - assert.Equal(400, resp.StatusCode) - response, _ := ioutil.ReadAll(resp.Body) - assert.JSONEq(`{ - "errors": { - "skin": [ - "Skin uploading is temporary unavailable" - ] - } - }`, string(response)) - }) - - t.Run("Keep the same identityId, uuid and username, but change textures information", func(t *testing.T) { - assert := testify.New(t) - - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - config, mocks := setupMocks(ctrl) - - resultModel := createSkinModel("mock_user", false) - resultModel.SkinId = 5 - resultModel.Url = "http://textures-server.com/skin.png" - resultModel.MojangTextures = "" - resultModel.MojangSignature = "" - - mocks.Auth.EXPECT().Check(gomock.Any()).Return(nil) - mocks.Skins.EXPECT().FindByUserId(1).Return(createSkinModel("mock_user", false), nil) - mocks.Skins.EXPECT().Save(resultModel).Return(nil) - mocks.Log.EXPECT().IncCounter("authentication.challenge", int64(1)) - mocks.Log.EXPECT().IncCounter("authentication.success", int64(1)) - mocks.Log.EXPECT().IncCounter("api.skins.post.request", int64(1)) - mocks.Log.EXPECT().IncCounter("api.skins.post.success", int64(1)) - - form := url.Values{ - "identityId": {"1"}, - "username": {"mock_user"}, - "uuid": {"0f657aa8-bfbe-415d-b700-5750090d3af3"}, - "skinId": {"5"}, - "is1_8": {"0"}, - "isSlim": {"0"}, - "url": {"http://textures-server.com/skin.png"}, - } - - req := httptest.NewRequest("POST", "http://chrly/api/skins", bytes.NewBufferString(form.Encode())) - req.Header.Add("Content-Type", "application/x-www-form-urlencoded") - w := httptest.NewRecorder() - - config.CreateHandler().ServeHTTP(w, req) - - resp := w.Result() - defer resp.Body.Close() - assert.Equal(201, resp.StatusCode) - response, _ := ioutil.ReadAll(resp.Body) - assert.Empty(response) - }) - - t.Run("Keep the same uuid and username, but change identityId", func(t *testing.T) { - assert := testify.New(t) - - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - config, mocks := setupMocks(ctrl) - - resultModel := createSkinModel("mock_user", false) - resultModel.UserId = 2 - resultModel.SkinId = 5 - resultModel.Url = "http://example.com/skin.png" - resultModel.MojangTextures = "" - resultModel.MojangSignature = "" - - form := url.Values{ - "identityId": {"2"}, - "username": {"mock_user"}, - "uuid": {"0f657aa8-bfbe-415d-b700-5750090d3af3"}, - "skinId": {"5"}, - "is1_8": {"0"}, - "isSlim": {"0"}, - "url": {"http://example.com/skin.png"}, - } - - req := httptest.NewRequest("POST", "http://chrly/api/skins", bytes.NewBufferString(form.Encode())) - req.Header.Add("Content-Type", "application/x-www-form-urlencoded") - w := httptest.NewRecorder() - - mocks.Auth.EXPECT().Check(gomock.Any()).Return(nil) - mocks.Skins.EXPECT().FindByUserId(2).Return(nil, &db.SkinNotFoundError{Who: "unknown"}) - mocks.Skins.EXPECT().FindByUsername("mock_user").Return(createSkinModel("mock_user", false), nil) - mocks.Skins.EXPECT().RemoveByUsername("mock_user").Return(nil) - mocks.Skins.EXPECT().Save(resultModel).Return(nil) - mocks.Log.EXPECT().IncCounter("authentication.challenge", int64(1)) - mocks.Log.EXPECT().IncCounter("authentication.success", int64(1)) - mocks.Log.EXPECT().IncCounter("api.skins.post.request", int64(1)) - mocks.Log.EXPECT().IncCounter("api.skins.post.success", int64(1)) - - config.CreateHandler().ServeHTTP(w, req) - - resp := w.Result() - defer resp.Body.Close() - assert.Equal(201, resp.StatusCode) - response, _ := ioutil.ReadAll(resp.Body) - assert.Empty(response) - }) - - t.Run("Keep the same identityId and uuid, but change username", func(t *testing.T) { - assert := testify.New(t) - - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - config, mocks := setupMocks(ctrl) - - resultModel := createSkinModel("changed_username", false) - resultModel.SkinId = 5 - resultModel.Url = "http://example.com/skin.png" - resultModel.MojangTextures = "" - resultModel.MojangSignature = "" - - form := url.Values{ - "identityId": {"1"}, - "username": {"changed_username"}, - "uuid": {"0f657aa8-bfbe-415d-b700-5750090d3af3"}, - "skinId": {"5"}, - "is1_8": {"0"}, - "isSlim": {"0"}, - "url": {"http://example.com/skin.png"}, - } - - req := httptest.NewRequest("POST", "http://chrly/api/skins", bytes.NewBufferString(form.Encode())) - req.Header.Add("Content-Type", "application/x-www-form-urlencoded") - w := httptest.NewRecorder() - - mocks.Auth.EXPECT().Check(gomock.Any()).Return(nil) - mocks.Skins.EXPECT().FindByUserId(1).Return(createSkinModel("mock_user", false), nil) - mocks.Skins.EXPECT().RemoveByUserId(1).Return(nil) - mocks.Skins.EXPECT().Save(resultModel).Return(nil) - mocks.Log.EXPECT().IncCounter("authentication.challenge", int64(1)) - mocks.Log.EXPECT().IncCounter("authentication.success", int64(1)) - mocks.Log.EXPECT().IncCounter("api.skins.post.request", int64(1)) - mocks.Log.EXPECT().IncCounter("api.skins.post.success", int64(1)) - - config.CreateHandler().ServeHTTP(w, req) - - resp := w.Result() - defer resp.Body.Close() - assert.Equal(201, resp.StatusCode) - response, _ := ioutil.ReadAll(resp.Body) - assert.Empty(response) - }) - - t.Run("Get errors about required fields", func(t *testing.T) { - assert := testify.New(t) - - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - config, mocks := setupMocks(ctrl) - - form := url.Values{ - "mojangTextures": {"someBase64EncodedString"}, - } - - req := httptest.NewRequest("POST", "http://chrly/api/skins", bytes.NewBufferString(form.Encode())) - req.Header.Add("Content-Type", "application/x-www-form-urlencoded") - w := httptest.NewRecorder() - - mocks.Auth.EXPECT().Check(gomock.Any()).Return(nil) - mocks.Log.EXPECT().IncCounter("authentication.challenge", int64(1)) - mocks.Log.EXPECT().IncCounter("authentication.success", int64(1)) - mocks.Log.EXPECT().IncCounter("api.skins.post.request", int64(1)) - mocks.Log.EXPECT().IncCounter("api.skins.post.validation_failed", int64(1)) - - config.CreateHandler().ServeHTTP(w, req) - - resp := w.Result() - defer resp.Body.Close() - assert.Equal(400, resp.StatusCode) - response, _ := ioutil.ReadAll(resp.Body) - assert.JSONEq(`{ - "errors": { - "identityId": [ - "The identityId field is required", - "The identityId field must be numeric", - "The identityId field must be minimum 1 char" - ], - "skinId": [ - "The skinId field is required", - "The skinId field must be numeric", - "The skinId field must be minimum 1 char" - ], - "username": [ - "The username field is required" - ], - "uuid": [ - "The uuid field is required", - "The uuid field must contain valid UUID" - ], - "url": [ - "One of url or skin should be provided, but not both" - ], - "skin": [ - "One of url or skin should be provided, but not both" - ], - "mojangSignature": [ - "The mojangSignature field is required" - ] - } - }`, string(response)) - }) - - t.Run("Perform request without authorization", func(t *testing.T) { - assert := testify.New(t) - - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - config, mocks := setupMocks(ctrl) - - req := httptest.NewRequest("POST", "http://chrly/api/skins", nil) - req.Header.Add("Authorization", "Bearer invalid.jwt.token") - w := httptest.NewRecorder() - - mocks.Auth.EXPECT().Check(gomock.Any()).Return(&auth.Unauthorized{Reason: "Cannot parse passed JWT token"}) - mocks.Log.EXPECT().IncCounter("authentication.challenge", int64(1)) - mocks.Log.EXPECT().IncCounter("authentication.failed", int64(1)) - - config.CreateHandler().ServeHTTP(w, req) - - resp := w.Result() - defer resp.Body.Close() - assert.Equal(403, resp.StatusCode) - response, _ := ioutil.ReadAll(resp.Body) - assert.JSONEq(`{ - "error": "Cannot parse passed JWT token" - }`, string(response)) - }) -} - -func TestConfig_DeleteSkinByUserId(t *testing.T) { - t.Run("Delete skin by its identity id", func(t *testing.T) { - assert := testify.New(t) - - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - config, mocks := setupMocks(ctrl) - - req := httptest.NewRequest("DELETE", "http://chrly/api/skins/id:1", nil) - w := httptest.NewRecorder() - - mocks.Auth.EXPECT().Check(gomock.Any()).Return(nil) - mocks.Skins.EXPECT().FindByUserId(1).Return(createSkinModel("mock_user", false), nil) - mocks.Skins.EXPECT().RemoveByUserId(1).Return(nil) - mocks.Log.EXPECT().IncCounter("authentication.challenge", int64(1)) - mocks.Log.EXPECT().IncCounter("authentication.success", int64(1)) - mocks.Log.EXPECT().IncCounter("api.skins.delete.request", int64(1)) - mocks.Log.EXPECT().IncCounter("api.skins.delete.success", int64(1)) - - config.CreateHandler().ServeHTTP(w, req) - - resp := w.Result() - defer resp.Body.Close() - assert.Equal(204, resp.StatusCode) - response, _ := ioutil.ReadAll(resp.Body) - assert.Empty(response) - }) - - t.Run("Try to remove not exists identity id", func(t *testing.T) { - assert := testify.New(t) - - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - config, mocks := setupMocks(ctrl) - - req := httptest.NewRequest("DELETE", "http://chrly/api/skins/id:2", nil) - w := httptest.NewRecorder() - - mocks.Auth.EXPECT().Check(gomock.Any()).Return(nil) - mocks.Skins.EXPECT().FindByUserId(2).Return(nil, &db.SkinNotFoundError{Who: "unknown"}) - mocks.Log.EXPECT().IncCounter("authentication.challenge", int64(1)) - mocks.Log.EXPECT().IncCounter("authentication.success", int64(1)) - mocks.Log.EXPECT().IncCounter("api.skins.delete.request", int64(1)) - mocks.Log.EXPECT().IncCounter("api.skins.delete.not_found", int64(1)) - - config.CreateHandler().ServeHTTP(w, req) - - resp := w.Result() - defer resp.Body.Close() - assert.Equal(404, resp.StatusCode) - response, _ := ioutil.ReadAll(resp.Body) - assert.JSONEq(`[ - "Cannot find record for requested user id" - ]`, string(response)) - }) -} - -func TestConfig_DeleteSkinByUsername(t *testing.T) { - t.Run("Delete skin by its identity username", func(t *testing.T) { - assert := testify.New(t) - - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - config, mocks := setupMocks(ctrl) - - req := httptest.NewRequest("DELETE", "http://chrly/api/skins/mock_user", nil) - w := httptest.NewRecorder() - - mocks.Auth.EXPECT().Check(gomock.Any()).Return(nil) - mocks.Skins.EXPECT().FindByUsername("mock_user").Return(createSkinModel("mock_user", false), nil) - mocks.Skins.EXPECT().RemoveByUserId(1).Return(nil) - mocks.Log.EXPECT().IncCounter("authentication.challenge", int64(1)) - mocks.Log.EXPECT().IncCounter("authentication.success", int64(1)) - mocks.Log.EXPECT().IncCounter("api.skins.delete.request", int64(1)) - mocks.Log.EXPECT().IncCounter("api.skins.delete.success", int64(1)) - - config.CreateHandler().ServeHTTP(w, req) - - resp := w.Result() - defer resp.Body.Close() - assert.Equal(204, resp.StatusCode) - response, _ := ioutil.ReadAll(resp.Body) - assert.Empty(response) - }) - - t.Run("Try to remove not exists identity username", func(t *testing.T) { - assert := testify.New(t) - - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - config, mocks := setupMocks(ctrl) - - req := httptest.NewRequest("DELETE", "http://chrly/api/skins/mock_user_2", nil) - w := httptest.NewRecorder() - - mocks.Auth.EXPECT().Check(gomock.Any()).Return(nil) - mocks.Skins.EXPECT().FindByUsername("mock_user_2").Return(nil, &db.SkinNotFoundError{Who: "mock_user_2"}) - mocks.Log.EXPECT().IncCounter("authentication.challenge", int64(1)) - mocks.Log.EXPECT().IncCounter("authentication.success", int64(1)) - mocks.Log.EXPECT().IncCounter("api.skins.delete.request", int64(1)) - mocks.Log.EXPECT().IncCounter("api.skins.delete.not_found", int64(1)) - - config.CreateHandler().ServeHTTP(w, req) - - resp := w.Result() - defer resp.Body.Close() - assert.Equal(404, resp.StatusCode) - response, _ := ioutil.ReadAll(resp.Body) - assert.JSONEq(`[ - "Cannot find record for requested username" - ]`, string(response)) - }) -} - -func TestConfig_Authenticate(t *testing.T) { - t.Run("Test behavior when signing key is not set", func(t *testing.T) { - assert := testify.New(t) - - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - config, mocks := setupMocks(ctrl) - - req := httptest.NewRequest("POST", "http://localhost", nil) - w := httptest.NewRecorder() - - mocks.Auth.EXPECT().Check(gomock.Any()).Return(&auth.Unauthorized{Reason: "signing key not available"}) - mocks.Log.EXPECT().IncCounter("authentication.challenge", int64(1)) - mocks.Log.EXPECT().IncCounter("authentication.failed", int64(1)) - - res := config.AuthenticationMiddleware(http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {})) - res.ServeHTTP(w, req) - - resp := w.Result() - defer resp.Body.Close() - assert.Equal(403, resp.StatusCode) - response, _ := ioutil.ReadAll(resp.Body) - assert.JSONEq(`{ - "error": "signing key not available" - }`, string(response)) - }) -} - -// base64 https://github.com/mathiasbynens/small/blob/0ca3c51/png-transparent.png -var OnePxPng = []byte("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAACklEQVR4nGMAAQAABQABDQottAAAAABJRU5ErkJggg==") - -func loadSkinFile() []byte { - result := make([]byte, 92) - _, err := base64.StdEncoding.Decode(result, OnePxPng) - if err != nil { - panic(err) - } - - return result -} diff --git a/http/cape.go b/http/cape.go deleted file mode 100644 index 9bba411..0000000 --- a/http/cape.go +++ /dev/null @@ -1,51 +0,0 @@ -package http - -import ( - "io" - "net/http" - - "github.com/gorilla/mux" -) - -func (cfg *Config) Cape(response http.ResponseWriter, request *http.Request) { - if mux.Vars(request)["converted"] == "" { - cfg.Logger.IncCounter("capes.request", 1) - } - - username := parseUsername(mux.Vars(request)["username"]) - rec, err := cfg.CapesRepo.FindByUsername(username) - if err == nil { - request.Header.Set("Content-Type", "image/png") - _, _ = io.Copy(response, rec.File) - return - } - - mojangTextures, err := cfg.MojangTexturesProvider.GetForUsername(username) - if err != nil || mojangTextures == nil { - response.WriteHeader(http.StatusNotFound) - return - } - - texturesProp := mojangTextures.DecodeTextures() - cape := texturesProp.Textures.Cape - if cape == nil { - response.WriteHeader(http.StatusNotFound) - return - } - - http.Redirect(response, request, cape.Url, 301) -} - -func (cfg *Config) CapeGET(response http.ResponseWriter, request *http.Request) { - cfg.Logger.IncCounter("capes.get_request", 1) - username := request.URL.Query().Get("name") - if username == "" { - response.WriteHeader(http.StatusBadRequest) - return - } - - mux.Vars(request)["username"] = username - mux.Vars(request)["converted"] = "1" - - cfg.Cape(response, request) -} diff --git a/http/cape_test.go b/http/cape_test.go deleted file mode 100644 index ededa15..0000000 --- a/http/cape_test.go +++ /dev/null @@ -1,163 +0,0 @@ -package http - -import ( - "bytes" - "image" - "image/png" - "io/ioutil" - "net/http" - "net/http/httptest" - "testing" - - "github.com/golang/mock/gomock" - testify "github.com/stretchr/testify/assert" - - "github.com/elyby/chrly/db" - "github.com/elyby/chrly/model" -) - -type capesTestCase struct { - Name string - RequestUrl string - ExpectedLogKey string - ExistsInLocalStorage bool - ExistsInMojang bool - HasCapeInMojangResp bool - AssertResponse func(assert *testify.Assertions, resp *http.Response) -} - -var capesTestCases = []*capesTestCase{ - { - Name: "Obtain cape for known username", - ExistsInLocalStorage: true, - AssertResponse: func(assert *testify.Assertions, resp *http.Response) { - assert.Equal(200, resp.StatusCode) - responseData, _ := ioutil.ReadAll(resp.Body) - assert.Equal(createCape(), responseData) - assert.Equal("image/png", resp.Header.Get("Content-Type")) - }, - }, - { - Name: "Obtain cape for unknown username that exists in Mojang and has a cape", - ExistsInLocalStorage: false, - ExistsInMojang: true, - HasCapeInMojangResp: true, - AssertResponse: func(assert *testify.Assertions, resp *http.Response) { - assert.Equal(301, resp.StatusCode) - assert.Equal("http://mojang/cape.png", resp.Header.Get("Location")) - }, - }, - { - Name: "Obtain cape for unknown username that exists in Mojang, but don't has a cape", - ExistsInLocalStorage: false, - ExistsInMojang: true, - HasCapeInMojangResp: false, - AssertResponse: func(assert *testify.Assertions, resp *http.Response) { - assert.Equal(404, resp.StatusCode) - }, - }, - { - Name: "Obtain cape for unknown username that doesn't exists in Mojang", - ExistsInLocalStorage: false, - ExistsInMojang: false, - AssertResponse: func(assert *testify.Assertions, resp *http.Response) { - assert.Equal(404, resp.StatusCode) - }, - }, -} - -func TestConfig_Cape(t *testing.T) { - performTest := func(t *testing.T, testCase *capesTestCase) { - assert := testify.New(t) - - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - config, mocks := setupMocks(ctrl) - - mocks.Log.EXPECT().IncCounter(testCase.ExpectedLogKey, int64(1)) - if testCase.ExistsInLocalStorage { - mocks.Capes.EXPECT().FindByUsername("mock_username").Return(&model.Cape{ - File: bytes.NewReader(createCape()), - }, nil) - } else { - mocks.Capes.EXPECT().FindByUsername("mock_username").Return(nil, &db.CapeNotFoundError{Who: "mock_username"}) - } - - if testCase.ExistsInMojang { - textures := createTexturesResponse(false, testCase.HasCapeInMojangResp) - mocks.MojangProvider.On("GetForUsername", "mock_username").Return(textures, nil) - } else { - mocks.MojangProvider.On("GetForUsername", "mock_username").Return(nil, nil) - } - - req := httptest.NewRequest("GET", testCase.RequestUrl, nil) - w := httptest.NewRecorder() - - config.CreateHandler().ServeHTTP(w, req) - - resp := w.Result() - testCase.AssertResponse(assert, resp) - } - - t.Run("Normal API", func(t *testing.T) { - for _, testCase := range capesTestCases { - testCase.RequestUrl = "http://chrly/cloaks/mock_username" - testCase.ExpectedLogKey = "capes.request" - t.Run(testCase.Name, func(t *testing.T) { - performTest(t, testCase) - }) - } - }) - - t.Run("GET fallback API", func(t *testing.T) { - for _, testCase := range capesTestCases { - testCase.RequestUrl = "http://chrly/cloaks?name=mock_username" - testCase.ExpectedLogKey = "capes.get_request" - t.Run(testCase.Name, func(t *testing.T) { - performTest(t, testCase) - }) - } - - t.Run("Should trim trailing slash", func(t *testing.T) { - assert := testify.New(t) - - req := httptest.NewRequest("GET", "http://chrly/cloaks/?name=notch", nil) - w := httptest.NewRecorder() - - (&Config{}).CreateHandler().ServeHTTP(w, req) - - resp := w.Result() - assert.Equal(301, resp.StatusCode) - assert.Equal("http://chrly/cloaks?name=notch", resp.Header.Get("Location")) - }) - - t.Run("Return error when name is not provided", func(t *testing.T) { - assert := testify.New(t) - - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - config, mocks := setupMocks(ctrl) - mocks.Log.EXPECT().IncCounter("capes.get_request", int64(1)) - - req := httptest.NewRequest("GET", "http://chrly/cloaks", nil) - w := httptest.NewRecorder() - - config.CreateHandler().ServeHTTP(w, req) - - resp := w.Result() - assert.Equal(400, resp.StatusCode) - }) - }) -} - -// Cape md5: 424ff79dce9940af89c28ad80de8aaad -func createCape() []byte { - img := image.NewAlpha(image.Rect(0, 0, 64, 32)) - writer := &bytes.Buffer{} - _ = png.Encode(writer, img) - pngBytes, _ := ioutil.ReadAll(writer) - - return pngBytes -} diff --git a/http/http.go b/http/http.go index bc1edc4..8286b28 100644 --- a/http/http.go +++ b/http/http.go @@ -1,83 +1,22 @@ package http import ( - "fmt" - "net" + "encoding/json" "net/http" "os" "os/signal" - "strings" "syscall" - "time" - - "github.com/gorilla/mux" - "github.com/mono83/slf/wd" - - "github.com/elyby/chrly/interfaces" ) -type Config struct { - ListenSpec string +func NotFound(response http.ResponseWriter, _ *http.Request) { + data, _ := json.Marshal(map[string]string{ + "status": "404", + "message": "Not Found", + }) - SkinsRepo interfaces.SkinsRepository - CapesRepo interfaces.CapesRepository - MojangTexturesProvider interfaces.MojangTexturesProvider - Logger wd.Watchdog - Auth interfaces.AuthChecker -} - -func (cfg *Config) Run() error { - cfg.Logger.Info(fmt.Sprintf("Starting, HTTP on: %s\n", cfg.ListenSpec)) - - listener, err := net.Listen("tcp", cfg.ListenSpec) - if err != nil { - return err - } - - server := &http.Server{ - ReadTimeout: 60 * time.Second, - WriteTimeout: 60 * time.Second, - MaxHeaderBytes: 1 << 16, - Handler: cfg.CreateHandler(), - } - - go server.Serve(listener) - - s := waitForSignal() - cfg.Logger.Info(fmt.Sprintf("Got signal: %v, exiting.", s)) - - return nil -} - -func (cfg *Config) CreateHandler() http.Handler { - router := mux.NewRouter().StrictSlash(true) - - router.HandleFunc("/skins/{username}", cfg.Skin).Methods("GET") - router.HandleFunc("/cloaks/{username}", cfg.Cape).Methods("GET").Name("cloaks") - router.HandleFunc("/textures/{username}", cfg.Textures).Methods("GET") - router.HandleFunc("/textures/signed/{username}", cfg.SignedTextures).Methods("GET") - // Legacy - router.HandleFunc("/skins", cfg.SkinGET).Methods("GET") - router.HandleFunc("/cloaks", cfg.CapeGET).Methods("GET") - // API - apiRouter := router.PathPrefix("/api").Subrouter() - apiRouter.Use(cfg.AuthenticationMiddleware) - apiRouter.Handle("/skins", http.HandlerFunc(cfg.PostSkin)).Methods("POST") - apiRouter.Handle("/skins/id:{id:[0-9]+}", http.HandlerFunc(cfg.DeleteSkinByUserId)).Methods("DELETE") - apiRouter.Handle("/skins/{username}", http.HandlerFunc(cfg.DeleteSkinByUsername)).Methods("DELETE") - // 404 - router.NotFoundHandler = http.HandlerFunc(cfg.NotFound) - - return router -} - -func parseUsername(username string) string { - const suffix = ".png" - if strings.HasSuffix(username, suffix) { - username = strings.TrimSuffix(username, suffix) - } - - return username + response.Header().Set("Content-Type", "application/json") + response.WriteHeader(http.StatusNotFound) + _, _ = response.Write(data) } func waitForSignal() os.Signal { @@ -86,3 +25,34 @@ func waitForSignal() os.Signal { return <-ch } + +func apiBadRequest(resp http.ResponseWriter, errorsPerField map[string][]string) { + resp.WriteHeader(http.StatusBadRequest) + resp.Header().Set("Content-Type", "application/json") + result, _ := json.Marshal(map[string]interface{}{ + "errors": errorsPerField, + }) + _, _ = resp.Write(result) +} + +func apiForbidden(resp http.ResponseWriter, reason string) { + resp.WriteHeader(http.StatusForbidden) + resp.Header().Set("Content-Type", "application/json") + result, _ := json.Marshal(map[string]interface{}{ + "error": reason, + }) + _, _ = resp.Write(result) +} + +func apiNotFound(resp http.ResponseWriter, reason string) { + resp.WriteHeader(http.StatusNotFound) + resp.Header().Set("Content-Type", "application/json") + result, _ := json.Marshal([]interface{}{ + reason, + }) + _, _ = resp.Write(result) +} + +func apiServerError(resp http.ResponseWriter) { + resp.WriteHeader(http.StatusInternalServerError) +} diff --git a/http/http_test.go b/http/http_test.go index ec74497..2b6d93d 100644 --- a/http/http_test.go +++ b/http/http_test.go @@ -1,101 +1,27 @@ package http import ( + "io/ioutil" + "net/http/httptest" "testing" - "time" - "github.com/stretchr/testify/mock" - - "github.com/elyby/chrly/api/mojang" - - "github.com/golang/mock/gomock" testify "github.com/stretchr/testify/assert" - - "github.com/elyby/chrly/interfaces/mock_interfaces" - "github.com/elyby/chrly/interfaces/mock_wd" ) -func TestParseUsername(t *testing.T) { +func TestConfig_NotFound(t *testing.T) { assert := testify.New(t) - assert.Equal("test", parseUsername("test.png"), "Function should trim .png at end") - assert.Equal("test", parseUsername("test"), "Function should return string itself, if it not contains .png at end") -} - -type mojangTexturesProviderMock struct { - mock.Mock -} - -func (m *mojangTexturesProviderMock) GetForUsername(username string) (*mojang.SignedTexturesResponse, error) { - args := m.Called(username) - var result *mojang.SignedTexturesResponse - if casted, ok := args.Get(0).(*mojang.SignedTexturesResponse); ok { - result = casted - } - - return result, args.Error(1) -} - -type mocks struct { - Skins *mock_interfaces.MockSkinsRepository - Capes *mock_interfaces.MockCapesRepository - MojangProvider *mojangTexturesProviderMock - Auth *mock_interfaces.MockAuthChecker - Log *mock_wd.MockWatchdog -} - -func setupMocks(ctrl *gomock.Controller) (*Config, *mocks) { - skinsRepo := mock_interfaces.NewMockSkinsRepository(ctrl) - capesRepo := mock_interfaces.NewMockCapesRepository(ctrl) - authChecker := mock_interfaces.NewMockAuthChecker(ctrl) - wd := mock_wd.NewMockWatchdog(ctrl) - texturesProvider := &mojangTexturesProviderMock{} - - return &Config{ - SkinsRepo: skinsRepo, - CapesRepo: capesRepo, - Auth: authChecker, - MojangTexturesProvider: texturesProvider, - Logger: wd, - }, &mocks{ - Skins: skinsRepo, - Capes: capesRepo, - Auth: authChecker, - MojangProvider: texturesProvider, - Log: wd, - } -} - -func createTexturesResponse(includeSkin bool, includeCape bool) *mojang.SignedTexturesResponse { - timeZone, _ := time.LoadLocation("Europe/Minsk") - textures := &mojang.TexturesProp{ - Timestamp: time.Date(2019, 4, 27, 23, 56, 12, 0, timeZone).Unix(), - ProfileID: "00000000000000000000000000000000", - ProfileName: "mock_user", - Textures: &mojang.TexturesResponse{}, - } - - if includeSkin { - textures.Textures.Skin = &mojang.SkinTexturesResponse{ - Url: "http://mojang/skin.png", - } - } - - if includeCape { - textures.Textures.Cape = &mojang.CapeTexturesResponse{ - Url: "http://mojang/cape.png", - } - } - - response := &mojang.SignedTexturesResponse{ - Id: "00000000000000000000000000000000", - Name: "mock_user", - Props: []*mojang.Property{ - { - Name: "textures", - Value: mojang.EncodeTextures(textures), - }, - }, - } - - return response + + req := httptest.NewRequest("GET", "http://example.com", nil) + w := httptest.NewRecorder() + + NotFound(w, req) + + resp := w.Result() + assert.Equal(404, resp.StatusCode) + assert.Equal("application/json", resp.Header.Get("Content-Type")) + response, _ := ioutil.ReadAll(resp.Body) + assert.JSONEq(`{ + "status": "404", + "message": "Not Found" + }`, string(response)) } diff --git a/http/not_found.go b/http/not_found.go deleted file mode 100644 index 33e4705..0000000 --- a/http/not_found.go +++ /dev/null @@ -1,17 +0,0 @@ -package http - -import ( - "encoding/json" - "net/http" -) - -func (cfg *Config) NotFound(response http.ResponseWriter, request *http.Request) { - data, _ := json.Marshal(map[string]string{ - "status": "404", - "message": "Not Found", - }) - - response.Header().Set("Content-Type", "application/json") - response.WriteHeader(http.StatusNotFound) - response.Write(data) -} diff --git a/http/not_found_test.go b/http/not_found_test.go deleted file mode 100644 index dfab394..0000000 --- a/http/not_found_test.go +++ /dev/null @@ -1,27 +0,0 @@ -package http - -import ( - "io/ioutil" - "net/http/httptest" - "testing" - - testify "github.com/stretchr/testify/assert" -) - -func TestConfig_NotFound(t *testing.T) { - assert := testify.New(t) - - req := httptest.NewRequest("GET", "http://skinsystem.ely.by/", nil) - w := httptest.NewRecorder() - - (&Config{}).CreateHandler().ServeHTTP(w, req) - - resp := w.Result() - assert.Equal(404, resp.StatusCode) - assert.Equal("application/json", resp.Header.Get("Content-Type")) - response, _ := ioutil.ReadAll(resp.Body) - assert.JSONEq(`{ - "status": "404", - "message": "Not Found" - }`, string(response)) -} diff --git a/http/signed_textures.go b/http/signed_textures.go deleted file mode 100644 index a3b1a20..0000000 --- a/http/signed_textures.go +++ /dev/null @@ -1,52 +0,0 @@ -package http - -import ( - "encoding/json" - "net/http" - "strings" - - "github.com/gorilla/mux" - - "github.com/elyby/chrly/api/mojang" -) - -func (cfg *Config) SignedTextures(response http.ResponseWriter, request *http.Request) { - cfg.Logger.IncCounter("signed_textures.request", 1) - username := parseUsername(mux.Vars(request)["username"]) - - var responseData *mojang.SignedTexturesResponse - - rec, err := cfg.SkinsRepo.FindByUsername(username) - if err == nil && rec.SkinId != 0 && rec.MojangTextures != "" { - responseData = &mojang.SignedTexturesResponse{ - Id: strings.Replace(rec.Uuid, "-", "", -1), - Name: rec.Username, - Props: []*mojang.Property{ - { - Name: "textures", - Signature: rec.MojangSignature, - Value: rec.MojangTextures, - }, - }, - } - } else if request.URL.Query().Get("proxy") != "" { - mojangTextures, err := cfg.MojangTexturesProvider.GetForUsername(username) - if err == nil && mojangTextures != nil { - responseData = mojangTextures - } - } - - if responseData == nil { - response.WriteHeader(http.StatusNoContent) - return - } - - responseData.Props = append(responseData.Props, &mojang.Property{ - Name: "chrly", - Value: "how do you tame a horse in Minecraft?", - }) - - responseJson, _ := json.Marshal(responseData) - response.Header().Set("Content-Type", "application/json") - response.Write(responseJson) -} diff --git a/http/signed_textures_test.go b/http/signed_textures_test.go deleted file mode 100644 index c07a969..0000000 --- a/http/signed_textures_test.go +++ /dev/null @@ -1,141 +0,0 @@ -package http - -import ( - "io/ioutil" - "net/http/httptest" - "testing" - - "github.com/golang/mock/gomock" - testify "github.com/stretchr/testify/assert" - - "github.com/elyby/chrly/db" -) - -func TestConfig_SignedTextures(t *testing.T) { - t.Run("Obtain signed textures for exists user", func(t *testing.T) { - assert := testify.New(t) - - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - config, mocks := setupMocks(ctrl) - - mocks.Log.EXPECT().IncCounter("signed_textures.request", int64(1)) - mocks.Skins.EXPECT().FindByUsername("mock_user").Return(createSkinModel("mock_user", false), nil) - - req := httptest.NewRequest("GET", "http://chrly/textures/signed/mock_user", nil) - w := httptest.NewRecorder() - - config.CreateHandler().ServeHTTP(w, req) - - resp := w.Result() - assert.Equal(200, resp.StatusCode) - assert.Equal("application/json", resp.Header.Get("Content-Type")) - response, _ := ioutil.ReadAll(resp.Body) - assert.JSONEq(`{ - "id": "0f657aa8bfbe415db7005750090d3af3", - "name": "mock_user", - "properties": [ - { - "name": "textures", - "signature": "mocked signature", - "value": "mocked textures base64" - }, - { - "name": "chrly", - "value": "how do you tame a horse in Minecraft?" - } - ] - }`, string(response)) - }) - - t.Run("Obtain signed textures for not exists user", func(t *testing.T) { - assert := testify.New(t) - - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - config, mocks := setupMocks(ctrl) - - mocks.Log.EXPECT().IncCounter("signed_textures.request", int64(1)) - - mocks.Skins.EXPECT().FindByUsername("mock_user").Return(nil, &db.SkinNotFoundError{}) - - req := httptest.NewRequest("GET", "http://chrly/textures/signed/mock_user", nil) - w := httptest.NewRecorder() - - config.CreateHandler().ServeHTTP(w, req) - - resp := w.Result() - assert.Equal(204, resp.StatusCode) - response, _ := ioutil.ReadAll(resp.Body) - assert.Equal("", string(response)) - }) - - t.Run("Obtain signed textures for exists user, but without signed textures", func(t *testing.T) { - assert := testify.New(t) - - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - config, mocks := setupMocks(ctrl) - - skinModel := createSkinModel("mock_user", false) - skinModel.MojangTextures = "" - skinModel.MojangSignature = "" - - mocks.Log.EXPECT().IncCounter("signed_textures.request", int64(1)) - mocks.Skins.EXPECT().FindByUsername("mock_user").Return(skinModel, nil) - - req := httptest.NewRequest("GET", "http://chrly/textures/signed/mock_user", nil) - w := httptest.NewRecorder() - - config.CreateHandler().ServeHTTP(w, req) - - resp := w.Result() - assert.Equal(204, resp.StatusCode) - response, _ := ioutil.ReadAll(resp.Body) - assert.Equal("", string(response)) - }) - - t.Run("Obtain signed textures for exists user, but without signed textures", func(t *testing.T) { - assert := testify.New(t) - - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - config, mocks := setupMocks(ctrl) - - skinModel := createSkinModel("mock_user", false) - skinModel.MojangTextures = "" - skinModel.MojangSignature = "" - - mocks.Log.EXPECT().IncCounter("signed_textures.request", int64(1)) - mocks.Skins.EXPECT().FindByUsername("mock_user").Return(skinModel, nil) - mocks.MojangProvider.On("GetForUsername", "mock_user").Once().Return(createTexturesResponse(true, false), nil) - - req := httptest.NewRequest("GET", "http://chrly/textures/signed/mock_user?proxy=true", nil) - w := httptest.NewRecorder() - - config.CreateHandler().ServeHTTP(w, req) - - resp := w.Result() - assert.Equal(200, resp.StatusCode) - assert.Equal("application/json", resp.Header.Get("Content-Type")) - response, _ := ioutil.ReadAll(resp.Body) - assert.JSONEq(`{ - "id": "00000000000000000000000000000000", - "name": "mock_user", - "properties": [ - { - "name": "textures", - "value": "eyJ0aW1lc3RhbXAiOjE1NTYzOTg1NzIsInByb2ZpbGVJZCI6IjAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwIiwicHJvZmlsZU5hbWUiOiJtb2NrX3VzZXIiLCJ0ZXh0dXJlcyI6eyJTS0lOIjp7InVybCI6Imh0dHA6Ly9tb2phbmcvc2tpbi5wbmcifX19" - }, - { - "name": "chrly", - "value": "how do you tame a horse in Minecraft?" - } - ] - }`, string(response)) - }) -} diff --git a/http/skin.go b/http/skin.go deleted file mode 100644 index a7f9397..0000000 --- a/http/skin.go +++ /dev/null @@ -1,49 +0,0 @@ -package http - -import ( - "net/http" - - "github.com/gorilla/mux" -) - -func (cfg *Config) Skin(response http.ResponseWriter, request *http.Request) { - if mux.Vars(request)["converted"] == "" { - cfg.Logger.IncCounter("skins.request", 1) - } - - username := parseUsername(mux.Vars(request)["username"]) - rec, err := cfg.SkinsRepo.FindByUsername(username) - if err == nil && rec.SkinId != 0 { - http.Redirect(response, request, rec.Url, 301) - return - } - - mojangTextures, err := cfg.MojangTexturesProvider.GetForUsername(username) - if err != nil || mojangTextures == nil { - response.WriteHeader(http.StatusNotFound) - return - } - - texturesProp := mojangTextures.DecodeTextures() - skin := texturesProp.Textures.Skin - if skin == nil { - response.WriteHeader(http.StatusNotFound) - return - } - - http.Redirect(response, request, skin.Url, 301) -} - -func (cfg *Config) SkinGET(response http.ResponseWriter, request *http.Request) { - cfg.Logger.IncCounter("skins.get_request", 1) - username := request.URL.Query().Get("name") - if username == "" { - response.WriteHeader(http.StatusBadRequest) - return - } - - mux.Vars(request)["username"] = username - mux.Vars(request)["converted"] = "1" - - cfg.Skin(response, request) -} diff --git a/http/skin_test.go b/http/skin_test.go deleted file mode 100644 index 23475ab..0000000 --- a/http/skin_test.go +++ /dev/null @@ -1,158 +0,0 @@ -package http - -import ( - "net/http" - "net/http/httptest" - "testing" - - "github.com/golang/mock/gomock" - testify "github.com/stretchr/testify/assert" - - "github.com/elyby/chrly/db" - "github.com/elyby/chrly/model" -) - -type skinsTestCase struct { - Name string - RequestUrl string - ExpectedLogKey string - ExistsInLocalStorage bool - ExistsInMojang bool - HasSkinInMojangResp bool - AssertResponse func(assert *testify.Assertions, resp *http.Response) -} - -var skinsTestCases = []*skinsTestCase{ - { - Name: "Obtain skin for known username", - ExistsInLocalStorage: true, - AssertResponse: func(assert *testify.Assertions, resp *http.Response) { - assert.Equal(301, resp.StatusCode) - assert.Equal("http://chrly/skin.png", resp.Header.Get("Location")) - }, - }, - { - Name: "Obtain skin for unknown username that exists in Mojang and has a cape", - ExistsInLocalStorage: false, - ExistsInMojang: true, - HasSkinInMojangResp: true, - AssertResponse: func(assert *testify.Assertions, resp *http.Response) { - assert.Equal(301, resp.StatusCode) - assert.Equal("http://mojang/skin.png", resp.Header.Get("Location")) - }, - }, - { - Name: "Obtain skin for unknown username that exists in Mojang, but don't has a cape", - ExistsInLocalStorage: false, - ExistsInMojang: true, - HasSkinInMojangResp: false, - AssertResponse: func(assert *testify.Assertions, resp *http.Response) { - assert.Equal(404, resp.StatusCode) - }, - }, - { - Name: "Obtain skin for unknown username that doesn't exists in Mojang", - ExistsInLocalStorage: false, - ExistsInMojang: false, - AssertResponse: func(assert *testify.Assertions, resp *http.Response) { - assert.Equal(404, resp.StatusCode) - }, - }, -} - -func TestConfig_Skin(t *testing.T) { - performTest := func(t *testing.T, testCase *skinsTestCase) { - assert := testify.New(t) - - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - config, mocks := setupMocks(ctrl) - - mocks.Log.EXPECT().IncCounter(testCase.ExpectedLogKey, int64(1)) - if testCase.ExistsInLocalStorage { - mocks.Skins.EXPECT().FindByUsername("mock_username").Return(createSkinModel("mock_username", false), nil) - } else { - mocks.Skins.EXPECT().FindByUsername("mock_username").Return(nil, &db.SkinNotFoundError{Who: "mock_username"}) - } - - if testCase.ExistsInMojang { - textures := createTexturesResponse(testCase.HasSkinInMojangResp, true) - mocks.MojangProvider.On("GetForUsername", "mock_username").Return(textures, nil) - } else { - mocks.MojangProvider.On("GetForUsername", "mock_username").Return(nil, nil) - } - - req := httptest.NewRequest("GET", testCase.RequestUrl, nil) - w := httptest.NewRecorder() - - config.CreateHandler().ServeHTTP(w, req) - - resp := w.Result() - testCase.AssertResponse(assert, resp) - } - - t.Run("Normal API", func(t *testing.T) { - for _, testCase := range skinsTestCases { - testCase.RequestUrl = "http://chrly/skins/mock_username" - testCase.ExpectedLogKey = "skins.request" - t.Run(testCase.Name, func(t *testing.T) { - performTest(t, testCase) - }) - } - }) - - t.Run("GET fallback API", func(t *testing.T) { - for _, testCase := range skinsTestCases { - testCase.RequestUrl = "http://chrly/skins?name=mock_username" - testCase.ExpectedLogKey = "skins.get_request" - t.Run(testCase.Name, func(t *testing.T) { - performTest(t, testCase) - }) - } - - t.Run("Should trim trailing slash", func(t *testing.T) { - assert := testify.New(t) - - req := httptest.NewRequest("GET", "http://chrly/skins/?name=notch", nil) - w := httptest.NewRecorder() - - (&Config{}).CreateHandler().ServeHTTP(w, req) - - resp := w.Result() - assert.Equal(301, resp.StatusCode) - assert.Equal("http://chrly/skins?name=notch", resp.Header.Get("Location")) - }) - - t.Run("Return error when name is not provided", func(t *testing.T) { - assert := testify.New(t) - - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - config, mocks := setupMocks(ctrl) - mocks.Log.EXPECT().IncCounter("skins.get_request", int64(1)) - - req := httptest.NewRequest("GET", "http://chrly/skins", nil) - w := httptest.NewRecorder() - - config.CreateHandler().ServeHTTP(w, req) - - resp := w.Result() - assert.Equal(400, resp.StatusCode) - }) - }) -} - -func createSkinModel(username string, isSlim bool) *model.Skin { - return &model.Skin{ - UserId: 1, - Username: username, - Uuid: "0f657aa8-bfbe-415d-b700-5750090d3af3", // Use non nil UUID to pass validation in api tests - SkinId: 1, - Url: "http://chrly/skin.png", - MojangTextures: "mocked textures base64", - MojangSignature: "mocked signature", - IsSlim: isSlim, - } -} diff --git a/http/skinsystem.go b/http/skinsystem.go new file mode 100644 index 0000000..c7fc2a9 --- /dev/null +++ b/http/skinsystem.go @@ -0,0 +1,503 @@ +package http + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "regexp" + "strconv" + "strings" + "time" + + "github.com/gorilla/mux" + "github.com/mono83/slf/wd" + "github.com/thedevsaddam/govalidator" + + "github.com/elyby/chrly/api/mojang" + "github.com/elyby/chrly/auth" + "github.com/elyby/chrly/model" +) + +//noinspection GoSnakeCaseUsage +const UUID_ANY = "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$" + +var regexUuidAny = regexp.MustCompile(UUID_ANY) + +func init() { + govalidator.AddCustomRule("skinUploadingNotAvailable", func(field string, rule string, message string, value interface{}) error { + if message == "" { + message = "Skin uploading is temporary unavailable" + } + + return errors.New(message) + }) + + // Add ability to validate any possible uuid form + govalidator.AddCustomRule("uuid_any", func(field string, rule string, message string, value interface{}) error { + str := value.(string) + if !regexUuidAny.MatchString(str) { + if message == "" { + message = fmt.Sprintf("The %s field must contain valid UUID", field) + } + + return errors.New(message) + } + + return nil + }) +} + +type SkinsRepository interface { + FindByUsername(username string) (*model.Skin, error) + FindByUserId(id int) (*model.Skin, error) + Save(skin *model.Skin) error + RemoveByUserId(id int) error + RemoveByUsername(username string) error +} + +type CapesRepository interface { + FindByUsername(username string) (*model.Cape, error) +} + +type SkinNotFoundError struct { + Who string +} + +func (e SkinNotFoundError) Error() string { + return "Skin data not found." +} + +type CapeNotFoundError struct { + Who string +} + +func (e CapeNotFoundError) Error() string { + return "Cape file not found." +} + +type MojangTexturesProvider interface { + GetForUsername(username string) (*mojang.SignedTexturesResponse, error) +} + +type AuthChecker interface { + Check(req *http.Request) error +} + +type Skinsystem struct { + ListenSpec string + + SkinsRepo SkinsRepository + CapesRepo CapesRepository + MojangTexturesProvider MojangTexturesProvider + Auth AuthChecker + Logger wd.Watchdog +} + +func (ctx *Skinsystem) Run() error { + ctx.Logger.Info(fmt.Sprintf("Starting, HTTP on: %s\n", ctx.ListenSpec)) + + listener, err := net.Listen("tcp", ctx.ListenSpec) + if err != nil { + return err + } + + server := &http.Server{ + ReadTimeout: 60 * time.Second, + WriteTimeout: 60 * time.Second, + MaxHeaderBytes: 1 << 16, + Handler: ctx.CreateHandler(), + } + + go server.Serve(listener) + + s := waitForSignal() + ctx.Logger.Info(fmt.Sprintf("Got signal: %v, exiting.", s)) + + return nil +} + +func (ctx *Skinsystem) CreateHandler() *mux.Router { + router := mux.NewRouter().StrictSlash(true) + + router.HandleFunc("/skins/{username}", ctx.Skin).Methods("GET") + router.HandleFunc("/cloaks/{username}", ctx.Cape).Methods("GET").Name("cloaks") + router.HandleFunc("/textures/{username}", ctx.Textures).Methods("GET") + router.HandleFunc("/textures/signed/{username}", ctx.SignedTextures).Methods("GET") + // Legacy + router.HandleFunc("/skins", ctx.SkinGET).Methods("GET") + router.HandleFunc("/cloaks", ctx.CapeGET).Methods("GET") + // API + apiRouter := router.PathPrefix("/api").Subrouter() + apiRouter.Use(ctx.AuthenticationMiddleware) + apiRouter.Handle("/skins", http.HandlerFunc(ctx.PostSkin)).Methods("POST") + apiRouter.Handle("/skins/id:{id:[0-9]+}", http.HandlerFunc(ctx.DeleteSkinByUserId)).Methods("DELETE") + apiRouter.Handle("/skins/{username}", http.HandlerFunc(ctx.DeleteSkinByUsername)).Methods("DELETE") + // 404 + router.NotFoundHandler = http.HandlerFunc(NotFound) + + return router +} + +func (ctx *Skinsystem) Skin(response http.ResponseWriter, request *http.Request) { + if mux.Vars(request)["converted"] == "" { + ctx.Logger.IncCounter("skins.request", 1) + } + + username := parseUsername(mux.Vars(request)["username"]) + rec, err := ctx.SkinsRepo.FindByUsername(username) + if err == nil && rec.SkinId != 0 { + http.Redirect(response, request, rec.Url, 301) + return + } + + mojangTextures, err := ctx.MojangTexturesProvider.GetForUsername(username) + if err != nil || mojangTextures == nil { + response.WriteHeader(http.StatusNotFound) + return + } + + texturesProp := mojangTextures.DecodeTextures() + skin := texturesProp.Textures.Skin + if skin == nil { + response.WriteHeader(http.StatusNotFound) + return + } + + http.Redirect(response, request, skin.Url, 301) +} + +func (ctx *Skinsystem) SkinGET(response http.ResponseWriter, request *http.Request) { + username := request.URL.Query().Get("name") + if username == "" { + response.WriteHeader(http.StatusBadRequest) + return + } + + ctx.Logger.IncCounter("skins.get_request", 1) + mux.Vars(request)["username"] = username + mux.Vars(request)["converted"] = "1" + + ctx.Skin(response, request) +} + +func (ctx *Skinsystem) Cape(response http.ResponseWriter, request *http.Request) { + if mux.Vars(request)["converted"] == "" { + ctx.Logger.IncCounter("capes.request", 1) + } + + username := parseUsername(mux.Vars(request)["username"]) + rec, err := ctx.CapesRepo.FindByUsername(username) + if err == nil { + request.Header.Set("Content-Type", "image/png") + _, _ = io.Copy(response, rec.File) + return + } + + mojangTextures, err := ctx.MojangTexturesProvider.GetForUsername(username) + if err != nil || mojangTextures == nil { + response.WriteHeader(http.StatusNotFound) + return + } + + texturesProp := mojangTextures.DecodeTextures() + cape := texturesProp.Textures.Cape + if cape == nil { + response.WriteHeader(http.StatusNotFound) + return + } + + http.Redirect(response, request, cape.Url, 301) +} + +func (ctx *Skinsystem) CapeGET(response http.ResponseWriter, request *http.Request) { + username := request.URL.Query().Get("name") + if username == "" { + response.WriteHeader(http.StatusBadRequest) + return + } + + ctx.Logger.IncCounter("capes.get_request", 1) + mux.Vars(request)["username"] = username + mux.Vars(request)["converted"] = "1" + + ctx.Cape(response, request) +} + +func (ctx *Skinsystem) Textures(response http.ResponseWriter, request *http.Request) { + ctx.Logger.IncCounter("textures.request", 1) + username := parseUsername(mux.Vars(request)["username"]) + + var textures *mojang.TexturesResponse + skin, skinErr := ctx.SkinsRepo.FindByUsername(username) + _, capeErr := ctx.CapesRepo.FindByUsername(username) + if (skinErr == nil && skin.SkinId != 0) || capeErr == nil { + textures = &mojang.TexturesResponse{} + + if skinErr == nil && skin.SkinId != 0 { + skinTextures := &mojang.SkinTexturesResponse{ + Url: skin.Url, + } + + if skin.IsSlim { + skinTextures.Metadata = &mojang.SkinTexturesMetadata{ + Model: "slim", + } + } + + textures.Skin = skinTextures + } + + if capeErr == nil { + textures.Cape = &mojang.CapeTexturesResponse{ + Url: request.URL.Scheme + "://" + request.Host + "/cloaks/" + username, + } + } + } else { + mojangTextures, err := ctx.MojangTexturesProvider.GetForUsername(username) + if err != nil || mojangTextures == nil { + response.WriteHeader(http.StatusNoContent) + return + } + + texturesProp := mojangTextures.DecodeTextures() + if texturesProp == nil { + response.WriteHeader(http.StatusInternalServerError) + ctx.Logger.Error("Unable to find textures property") + return + } + + textures = texturesProp.Textures + } + + responseData, _ := json.Marshal(textures) + response.Header().Set("Content-Type", "application/json") + response.Write(responseData) +} + +func (ctx *Skinsystem) SignedTextures(response http.ResponseWriter, request *http.Request) { + ctx.Logger.IncCounter("signed_textures.request", 1) + username := parseUsername(mux.Vars(request)["username"]) + + var responseData *mojang.SignedTexturesResponse + + rec, err := ctx.SkinsRepo.FindByUsername(username) + if err == nil && rec.SkinId != 0 && rec.MojangTextures != "" { + responseData = &mojang.SignedTexturesResponse{ + Id: strings.Replace(rec.Uuid, "-", "", -1), + Name: rec.Username, + Props: []*mojang.Property{ + { + Name: "textures", + Signature: rec.MojangSignature, + Value: rec.MojangTextures, + }, + }, + } + } else if request.URL.Query().Get("proxy") != "" { + mojangTextures, err := ctx.MojangTexturesProvider.GetForUsername(username) + if err == nil && mojangTextures != nil { + responseData = mojangTextures + } + } + + if responseData == nil { + response.WriteHeader(http.StatusNoContent) + return + } + + responseData.Props = append(responseData.Props, &mojang.Property{ + Name: "chrly", + Value: "how do you tame a horse in Minecraft?", + }) + + responseJson, _ := json.Marshal(responseData) + response.Header().Set("Content-Type", "application/json") + response.Write(responseJson) +} + +func (ctx *Skinsystem) PostSkin(resp http.ResponseWriter, req *http.Request) { + ctx.Logger.IncCounter("api.skins.post.request", 1) + validationErrors := validatePostSkinRequest(req) + if validationErrors != nil { + ctx.Logger.IncCounter("api.skins.post.validation_failed", 1) + apiBadRequest(resp, validationErrors) + return + } + + identityId, _ := strconv.Atoi(req.Form.Get("identityId")) + username := req.Form.Get("username") + + record, err := findIdentity(ctx.SkinsRepo, identityId, username) + if err != nil { + ctx.Logger.Error("Error on requesting a skin from the repository: :err", wd.ErrParam(err)) + apiServerError(resp) + return + } + + skinId, _ := strconv.Atoi(req.Form.Get("skinId")) + is18, _ := strconv.ParseBool(req.Form.Get("is1_8")) + isSlim, _ := strconv.ParseBool(req.Form.Get("isSlim")) + + record.Uuid = req.Form.Get("uuid") + record.SkinId = skinId + record.Is1_8 = is18 + record.IsSlim = isSlim + record.Url = req.Form.Get("url") + record.MojangTextures = req.Form.Get("mojangTextures") + record.MojangSignature = req.Form.Get("mojangSignature") + + err = ctx.SkinsRepo.Save(record) + if err != nil { + ctx.Logger.Error("Unable to save record to the repository: :err", wd.ErrParam(err)) + apiServerError(resp) + return + } + + ctx.Logger.IncCounter("api.skins.post.success", 1) + resp.WriteHeader(http.StatusCreated) +} + +func (ctx *Skinsystem) DeleteSkinByUserId(resp http.ResponseWriter, req *http.Request) { + ctx.Logger.IncCounter("api.skins.delete.request", 1) + id, _ := strconv.Atoi(mux.Vars(req)["id"]) + skin, err := ctx.SkinsRepo.FindByUserId(id) + if err != nil { + ctx.Logger.IncCounter("api.skins.delete.not_found", 1) + apiNotFound(resp, "Cannot find record for requested user id") + return + } + + ctx.deleteSkin(skin, resp) +} + +func (ctx *Skinsystem) DeleteSkinByUsername(resp http.ResponseWriter, req *http.Request) { + ctx.Logger.IncCounter("api.skins.delete.request", 1) + username := mux.Vars(req)["username"] + skin, err := ctx.SkinsRepo.FindByUsername(username) + if err != nil { + ctx.Logger.IncCounter("api.skins.delete.not_found", 1) + apiNotFound(resp, "Cannot find record for requested username") + return + } + + ctx.deleteSkin(skin, resp) +} + +func (ctx *Skinsystem) AuthenticationMiddleware(handler http.Handler) http.Handler { + return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { + ctx.Logger.IncCounter("authentication.challenge", 1) + err := ctx.Auth.Check(req) + if err != nil { + if _, ok := err.(*auth.Unauthorized); ok { + ctx.Logger.IncCounter("authentication.failed", 1) + apiForbidden(resp, err.Error()) + } else { + ctx.Logger.Error("Unknown error on validating api request: :err", wd.ErrParam(err)) + apiServerError(resp) + } + + return + } + + ctx.Logger.IncCounter("authentication.success", 1) + handler.ServeHTTP(resp, req) + }) +} + +func (ctx *Skinsystem) deleteSkin(skin *model.Skin, resp http.ResponseWriter) { + err := ctx.SkinsRepo.RemoveByUserId(skin.UserId) + if err != nil { + ctx.Logger.Error("Cannot delete skin by error: :err", wd.ErrParam(err)) + apiServerError(resp) + return + } + + ctx.Logger.IncCounter("api.skins.delete.success", 1) + resp.WriteHeader(http.StatusNoContent) +} + +func validatePostSkinRequest(request *http.Request) map[string][]string { + const maxMultipartMemory int64 = 32 << 20 + const oneOfSkinOrUrlMessage = "One of url or skin should be provided, but not both" + + _ = request.ParseMultipartForm(maxMultipartMemory) + + validationRules := govalidator.MapData{ + "identityId": {"required", "numeric", "min:1"}, + "username": {"required"}, + "uuid": {"required", "uuid_any"}, + "skinId": {"required", "numeric", "min:1"}, + "url": {"url"}, + "file:skin": {"ext:png", "size:24576", "mime:image/png"}, + "is1_8": {"bool"}, + "isSlim": {"bool"}, + } + + shouldAppendSkinRequiredError := false + url := request.Form.Get("url") + _, _, skinErr := request.FormFile("skin") + if (url != "" && skinErr == nil) || (url == "" && skinErr != nil) { + shouldAppendSkinRequiredError = true + } else if skinErr == nil { + validationRules["file:skin"] = append(validationRules["file:skin"], "skinUploadingNotAvailable") + } else if url != "" { + validationRules["is1_8"] = append(validationRules["is1_8"], "required") + validationRules["isSlim"] = append(validationRules["isSlim"], "required") + } + + mojangTextures := request.Form.Get("mojangTextures") + if mojangTextures != "" { + validationRules["mojangSignature"] = []string{"required"} + } + + validator := govalidator.New(govalidator.Options{ + Request: request, + Rules: validationRules, + RequiredDefault: false, + FormSize: maxMultipartMemory, + }) + validationResults := validator.Validate() + if shouldAppendSkinRequiredError { + validationResults["url"] = append(validationResults["url"], oneOfSkinOrUrlMessage) + validationResults["skin"] = append(validationResults["skin"], oneOfSkinOrUrlMessage) + } + + if len(validationResults) != 0 { + return validationResults + } + + return nil +} + +func findIdentity(repo SkinsRepository, identityId int, username string) (*model.Skin, error) { + var record *model.Skin + record, err := repo.FindByUserId(identityId) + if err != nil { + if _, isSkinNotFound := err.(*SkinNotFoundError); !isSkinNotFound { + return nil, err + } + + record, err = repo.FindByUsername(username) + if err == nil { + _ = repo.RemoveByUsername(username) + record.UserId = identityId + } else { + record = &model.Skin{ + UserId: identityId, + Username: username, + } + } + } else if record.Username != username { + _ = repo.RemoveByUserId(identityId) + record.Username = username + } + + return record, nil +} + +func parseUsername(username string) string { + return strings.TrimSuffix(username, ".png") +} diff --git a/http/skinsystem_test.go b/http/skinsystem_test.go new file mode 100644 index 0000000..463eeb2 --- /dev/null +++ b/http/skinsystem_test.go @@ -0,0 +1,1092 @@ +package http + +import ( + "bytes" + "encoding/base64" + "github.com/elyby/chrly/auth" + testify "github.com/stretchr/testify/assert" + "image" + "image/png" + "io" + "io/ioutil" + "mime/multipart" + "net/http" + "net/http/httptest" + "net/url" + "testing" + "time" + + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" + + "github.com/elyby/chrly/api/mojang" + "github.com/elyby/chrly/model" + "github.com/elyby/chrly/tests" +) + +/*************** + * Setup mocks * + ***************/ + +type skinsRepositoryMock struct { + mock.Mock +} + +func (m *skinsRepositoryMock) FindByUsername(username string) (*model.Skin, error) { + args := m.Called(username) + var result *model.Skin + if casted, ok := args.Get(0).(*model.Skin); ok { + result = casted + } + + return result, args.Error(1) +} + +func (m *skinsRepositoryMock) FindByUserId(id int) (*model.Skin, error) { + args := m.Called(id) + var result *model.Skin + if casted, ok := args.Get(0).(*model.Skin); ok { + result = casted + } + + return result, args.Error(1) +} + +func (m *skinsRepositoryMock) Save(skin *model.Skin) error { + args := m.Called(skin) + return args.Error(0) +} + +func (m *skinsRepositoryMock) RemoveByUserId(id int) error { + args := m.Called(id) + return args.Error(0) +} + +func (m *skinsRepositoryMock) RemoveByUsername(username string) error { + args := m.Called(username) + return args.Error(0) +} + +type capesRepositoryMock struct { + mock.Mock +} + +func (m *capesRepositoryMock) FindByUsername(username string) (*model.Cape, error) { + args := m.Called(username) + var result *model.Cape + if casted, ok := args.Get(0).(*model.Cape); ok { + result = casted + } + + return result, args.Error(1) +} + +type mojangTexturesProviderMock struct { + mock.Mock +} + +func (m *mojangTexturesProviderMock) GetForUsername(username string) (*mojang.SignedTexturesResponse, error) { + args := m.Called(username) + var result *mojang.SignedTexturesResponse + if casted, ok := args.Get(0).(*mojang.SignedTexturesResponse); ok { + result = casted + } + + return result, args.Error(1) +} + +type authCheckerMock struct { + mock.Mock +} + +func (m *authCheckerMock) Check(req *http.Request) error { + args := m.Called(req) + return args.Error(0) +} + +type skinsystemTestSuite struct { + suite.Suite + + App *Skinsystem + + SkinsRepository *skinsRepositoryMock + CapesRepository *capesRepositoryMock + MojangTexturesProvider *mojangTexturesProviderMock + Auth *authCheckerMock + Logger *tests.WdMock +} + +/******************** + * Setup test suite * + ********************/ + +func (suite *skinsystemTestSuite) SetupTest() { + suite.SkinsRepository = &skinsRepositoryMock{} + suite.CapesRepository = &capesRepositoryMock{} + suite.MojangTexturesProvider = &mojangTexturesProviderMock{} + suite.Auth = &authCheckerMock{} + suite.Logger = &tests.WdMock{} + + suite.App = &Skinsystem{ + SkinsRepo: suite.SkinsRepository, + CapesRepo: suite.CapesRepository, + MojangTexturesProvider: suite.MojangTexturesProvider, + Auth: suite.Auth, + Logger: suite.Logger, + } +} + +func (suite *skinsystemTestSuite) TearDownTest() { + suite.SkinsRepository.AssertExpectations(suite.T()) + suite.CapesRepository.AssertExpectations(suite.T()) + suite.MojangTexturesProvider.AssertExpectations(suite.T()) + suite.Auth.AssertExpectations(suite.T()) + suite.Logger.AssertExpectations(suite.T()) +} + +func (suite *skinsystemTestSuite) RunSubTest(name string, subTest func()) { + suite.SetupTest() + suite.Run(name, subTest) + suite.TearDownTest() +} + +/************* + * Run tests * + *************/ + +func TestSkinsystem(t *testing.T) { + suite.Run(t, new(skinsystemTestSuite)) +} + +type skinsystemTestCase struct { + Name string + BeforeTest func(suite *skinsystemTestSuite) + AfterTest func(suite *skinsystemTestSuite, response *http.Response) +} + +/************************ + * Get skin tests cases * + ************************/ + +var skinsTestsCases = []*skinsystemTestCase{ + { + Name: "Username exists in the local storage", + BeforeTest: func(suite *skinsystemTestSuite) { + suite.SkinsRepository.On("FindByUsername", "mock_username").Return(createSkinModel("mock_username", false), nil) + }, + AfterTest: func(suite *skinsystemTestSuite, response *http.Response) { + suite.Equal(301, response.StatusCode) + suite.Equal("http://chrly/skin.png", response.Header.Get("Location")) + }, + }, + { + Name: "Username doesn't exists on the local storage, but exists on Mojang and has textures", + BeforeTest: func(suite *skinsystemTestSuite) { + suite.SkinsRepository.On("FindByUsername", "mock_username").Return(nil, &SkinNotFoundError{Who: "mock_username"}) + suite.MojangTexturesProvider.On("GetForUsername", "mock_username").Return(createMojangResponse(true, false), nil) + }, + AfterTest: func(suite *skinsystemTestSuite, response *http.Response) { + suite.Equal(301, response.StatusCode) + suite.Equal("http://mojang/skin.png", response.Header.Get("Location")) + }, + }, + { + Name: "Username doesn't exists on the local storage, but exists on Mojang and has no textures", + BeforeTest: func(suite *skinsystemTestSuite) { + suite.SkinsRepository.On("FindByUsername", "mock_username").Return(nil, &SkinNotFoundError{Who: "mock_username"}) + suite.MojangTexturesProvider.On("GetForUsername", "mock_username").Return(createMojangResponse(false, false), nil) + }, + AfterTest: func(suite *skinsystemTestSuite, response *http.Response) { + suite.Equal(404, response.StatusCode) + }, + }, + { + Name: "Username doesn't exists on the local storage and doesn't exists on Mojang", + BeforeTest: func(suite *skinsystemTestSuite) { + suite.SkinsRepository.On("FindByUsername", "mock_username").Return(nil, &SkinNotFoundError{Who: "mock_username"}) + suite.MojangTexturesProvider.On("GetForUsername", "mock_username").Return(nil, nil) + }, + AfterTest: func(suite *skinsystemTestSuite, response *http.Response) { + suite.Equal(404, response.StatusCode) + }, + }, +} + +func (suite *skinsystemTestSuite) TestSkin() { + for _, testCase := range skinsTestsCases { + suite.RunSubTest(testCase.Name, func() { + suite.Logger.On("IncCounter", "skins.request", int64(1)).Once() + testCase.BeforeTest(suite) + + req := httptest.NewRequest("GET", "http://chrly/skins/mock_username", nil) + w := httptest.NewRecorder() + + suite.App.CreateHandler().ServeHTTP(w, req) + + testCase.AfterTest(suite, w.Result()) + }) + } + + suite.RunSubTest("Pass username with png extension", func() { + suite.Logger.On("IncCounter", "skins.request", int64(1)).Once() + suite.SkinsRepository.On("FindByUsername", "mock_username").Return(createSkinModel("mock_username", false), nil) + + req := httptest.NewRequest("GET", "http://chrly/skins/mock_username.png", nil) + w := httptest.NewRecorder() + + suite.App.CreateHandler().ServeHTTP(w, req) + + resp := w.Result() + suite.Equal(301, resp.StatusCode) + suite.Equal("http://chrly/skin.png", resp.Header.Get("Location")) + }) +} + +func (suite *skinsystemTestSuite) TestSkinGET() { + for _, testCase := range skinsTestsCases { + suite.RunSubTest(testCase.Name, func() { + suite.Logger.On("IncCounter", "skins.get_request", int64(1)).Once() + testCase.BeforeTest(suite) + + req := httptest.NewRequest("GET", "http://chrly/skins?name=mock_username", nil) + w := httptest.NewRecorder() + + suite.App.CreateHandler().ServeHTTP(w, req) + + testCase.AfterTest(suite, w.Result()) + }) + } + + suite.RunSubTest("Do not pass name param", func() { + req := httptest.NewRequest("GET", "http://chrly/skins", nil) + w := httptest.NewRecorder() + + suite.App.CreateHandler().ServeHTTP(w, req) + + resp := w.Result() + suite.Equal(400, resp.StatusCode) + }) +} + +/************************ + * Get cape tests cases * + ************************/ + +var capesTestsCases = []*skinsystemTestCase{ + { + Name: "Username exists in the local storage", + BeforeTest: func(suite *skinsystemTestSuite) { + suite.CapesRepository.On("FindByUsername", "mock_username").Return(createCapeModel(), nil) + }, + AfterTest: func(suite *skinsystemTestSuite, response *http.Response) { + suite.Equal(200, response.StatusCode) + responseData, _ := ioutil.ReadAll(response.Body) + suite.Equal(createCape(), responseData) + suite.Equal("image/png", response.Header.Get("Content-Type")) + }, + }, + { + Name: "Username doesn't exists on the local storage, but exists on Mojang and has textures", + BeforeTest: func(suite *skinsystemTestSuite) { + suite.CapesRepository.On("FindByUsername", "mock_username").Return(nil, &CapeNotFoundError{Who: "mock_username"}) + suite.MojangTexturesProvider.On("GetForUsername", "mock_username").Return(createMojangResponse(true, true), nil) + }, + AfterTest: func(suite *skinsystemTestSuite, response *http.Response) { + suite.Equal(301, response.StatusCode) + suite.Equal("http://mojang/cape.png", response.Header.Get("Location")) + }, + }, + { + Name: "Username doesn't exists on the local storage, but exists on Mojang and has no textures", + BeforeTest: func(suite *skinsystemTestSuite) { + suite.CapesRepository.On("FindByUsername", "mock_username").Return(nil, &CapeNotFoundError{Who: "mock_username"}) + suite.MojangTexturesProvider.On("GetForUsername", "mock_username").Return(createMojangResponse(false, false), nil) + }, + AfterTest: func(suite *skinsystemTestSuite, response *http.Response) { + suite.Equal(404, response.StatusCode) + }, + }, + { + Name: "Username doesn't exists on the local storage and doesn't exists on Mojang", + BeforeTest: func(suite *skinsystemTestSuite) { + suite.CapesRepository.On("FindByUsername", "mock_username").Return(nil, &CapeNotFoundError{Who: "mock_username"}) + suite.MojangTexturesProvider.On("GetForUsername", "mock_username").Return(nil, nil) + }, + AfterTest: func(suite *skinsystemTestSuite, response *http.Response) { + suite.Equal(404, response.StatusCode) + }, + }, +} + +func (suite *skinsystemTestSuite) TestCape() { + for _, testCase := range capesTestsCases { + suite.RunSubTest(testCase.Name, func() { + suite.Logger.On("IncCounter", "capes.request", int64(1)).Once() + testCase.BeforeTest(suite) + + req := httptest.NewRequest("GET", "http://chrly/cloaks/mock_username", nil) + w := httptest.NewRecorder() + + suite.App.CreateHandler().ServeHTTP(w, req) + + testCase.AfterTest(suite, w.Result()) + }) + } + + suite.RunSubTest("Pass username with png extension", func() { + suite.Logger.On("IncCounter", "capes.request", int64(1)).Once() + suite.CapesRepository.On("FindByUsername", "mock_username").Return(createCapeModel(), nil) + + req := httptest.NewRequest("GET", "http://chrly/cloaks/mock_username.png", nil) + w := httptest.NewRecorder() + + suite.App.CreateHandler().ServeHTTP(w, req) + + resp := w.Result() + suite.Equal(200, resp.StatusCode) + responseData, _ := ioutil.ReadAll(resp.Body) + suite.Equal(createCape(), responseData) + suite.Equal("image/png", resp.Header.Get("Content-Type")) + }) +} + +func (suite *skinsystemTestSuite) TestCapeGET() { + for _, testCase := range capesTestsCases { + suite.RunSubTest(testCase.Name, func() { + suite.Logger.On("IncCounter", "capes.get_request", int64(1)).Once() + testCase.BeforeTest(suite) + + req := httptest.NewRequest("GET", "http://chrly/cloaks?name=mock_username", nil) + w := httptest.NewRecorder() + + suite.App.CreateHandler().ServeHTTP(w, req) + + testCase.AfterTest(suite, w.Result()) + }) + } + + suite.RunSubTest("Do not pass name param", func() { + req := httptest.NewRequest("GET", "http://chrly/cloaks", nil) + w := httptest.NewRecorder() + + suite.App.CreateHandler().ServeHTTP(w, req) + + resp := w.Result() + suite.Equal(400, resp.StatusCode) + }) +} + +/**************************** + * Get textures tests cases * + ****************************/ + +var texturesTestsCases = []*skinsystemTestCase{ + { + Name: "Username exists and has skin, no cape", + BeforeTest: func(suite *skinsystemTestSuite) { + suite.SkinsRepository.On("FindByUsername", "mock_username").Return(createSkinModel("mock_username", false), nil) + suite.CapesRepository.On("FindByUsername", "mock_username").Return(nil, &CapeNotFoundError{Who: "mock_username"}) + }, + AfterTest: func(suite *skinsystemTestSuite, response *http.Response) { + suite.Equal(200, response.StatusCode) + suite.Equal("application/json", response.Header.Get("Content-Type")) + body, _ := ioutil.ReadAll(response.Body) + suite.JSONEq(`{ + "SKIN": { + "url": "http://chrly/skin.png" + } + }`, string(body)) + }, + }, + { + Name: "Username exists and has slim skin, no cape", + BeforeTest: func(suite *skinsystemTestSuite) { + suite.SkinsRepository.On("FindByUsername", "mock_username").Return(createSkinModel("mock_username", true), nil) + suite.CapesRepository.On("FindByUsername", "mock_username").Return(nil, &CapeNotFoundError{Who: "mock_username"}) + }, + AfterTest: func(suite *skinsystemTestSuite, response *http.Response) { + suite.Equal(200, response.StatusCode) + suite.Equal("application/json", response.Header.Get("Content-Type")) + body, _ := ioutil.ReadAll(response.Body) + suite.JSONEq(`{ + "SKIN": { + "url": "http://chrly/skin.png", + "metadata": { + "model": "slim" + } + } + }`, string(body)) + }, + }, + { + Name: "Username exists and has cape, no skin", + BeforeTest: func(suite *skinsystemTestSuite) { + suite.SkinsRepository.On("FindByUsername", "mock_username").Return(nil, &SkinNotFoundError{Who: "mock_username"}) + suite.CapesRepository.On("FindByUsername", "mock_username").Return(createCapeModel(), nil) + }, + AfterTest: func(suite *skinsystemTestSuite, response *http.Response) { + suite.Equal(200, response.StatusCode) + suite.Equal("application/json", response.Header.Get("Content-Type")) + body, _ := ioutil.ReadAll(response.Body) + suite.JSONEq(`{ + "CAPE": { + "url": "http://chrly/cloaks/mock_username" + } + }`, string(body)) + }, + }, + { + Name: "Username exists and has both skin and cape", + BeforeTest: func(suite *skinsystemTestSuite) { + suite.SkinsRepository.On("FindByUsername", "mock_username").Return(createSkinModel("mock_username", false), nil) + suite.CapesRepository.On("FindByUsername", "mock_username").Return(createCapeModel(), nil) + }, + AfterTest: func(suite *skinsystemTestSuite, response *http.Response) { + suite.Equal(200, response.StatusCode) + suite.Equal("application/json", response.Header.Get("Content-Type")) + body, _ := ioutil.ReadAll(response.Body) + suite.JSONEq(`{ + "SKIN": { + "url": "http://chrly/skin.png" + }, + "CAPE": { + "url": "http://chrly/cloaks/mock_username" + } + }`, string(body)) + }, + }, + { + Name: "Username not exists, but Mojang profile available", + BeforeTest: func(suite *skinsystemTestSuite) { + suite.SkinsRepository.On("FindByUsername", "mock_username").Return(nil, &SkinNotFoundError{}) + suite.CapesRepository.On("FindByUsername", "mock_username").Return(nil, &CapeNotFoundError{}) + suite.MojangTexturesProvider.On("GetForUsername", "mock_username").Once().Return(createMojangResponse(true, true), nil) + }, + AfterTest: func(suite *skinsystemTestSuite, response *http.Response) { + suite.Equal(200, response.StatusCode) + suite.Equal("application/json", response.Header.Get("Content-Type")) + body, _ := ioutil.ReadAll(response.Body) + suite.JSONEq(`{ + "SKIN": { + "url": "http://mojang/skin.png" + }, + "CAPE": { + "url": "http://mojang/cape.png" + } + }`, string(body)) + }, + }, + { + Name: "Username not exists and Mojang profile unavailable", + BeforeTest: func(suite *skinsystemTestSuite) { + suite.SkinsRepository.On("FindByUsername", "mock_username").Return(nil, &SkinNotFoundError{}) + suite.CapesRepository.On("FindByUsername", "mock_username").Return(nil, &CapeNotFoundError{}) + suite.MojangTexturesProvider.On("GetForUsername", "mock_username").Once().Return(nil, nil) + }, + AfterTest: func(suite *skinsystemTestSuite, response *http.Response) { + suite.Equal(204, response.StatusCode) + }, + }, +} + +func (suite *skinsystemTestSuite) TestTextures() { + for _, testCase := range texturesTestsCases { + suite.RunSubTest(testCase.Name, func() { + suite.Logger.On("IncCounter", "textures.request", int64(1)).Once() + testCase.BeforeTest(suite) + + req := httptest.NewRequest("GET", "http://chrly/textures/mock_username", nil) + w := httptest.NewRecorder() + + suite.App.CreateHandler().ServeHTTP(w, req) + + testCase.AfterTest(suite, w.Result()) + }) + } +} + +/*********************************** + * Get signed textures tests cases * + ***********************************/ + +type signedTexturesTestCase struct { + Name string + AllowProxy bool + BeforeTest func(suite *skinsystemTestSuite) + AfterTest func(suite *skinsystemTestSuite, response *http.Response) +} + +var signedTexturesTestsCases = []*signedTexturesTestCase{ + { + Name: "Username exists", + AllowProxy: false, + BeforeTest: func(suite *skinsystemTestSuite) { + suite.SkinsRepository.On("FindByUsername", "mock_username").Return(createSkinModel("mock_username", true), nil) + }, + AfterTest: func(suite *skinsystemTestSuite, response *http.Response) { + suite.Equal(200, response.StatusCode) + suite.Equal("application/json", response.Header.Get("Content-Type")) + body, _ := ioutil.ReadAll(response.Body) + suite.JSONEq(`{ + "id": "0f657aa8bfbe415db7005750090d3af3", + "name": "mock_username", + "properties": [ + { + "name": "textures", + "signature": "mocked signature", + "value": "mocked textures base64" + }, + { + "name": "chrly", + "value": "how do you tame a horse in Minecraft?" + } + ] + }`, string(body)) + }, + }, + { + Name: "Username not exists", + AllowProxy: false, + BeforeTest: func(suite *skinsystemTestSuite) { + suite.SkinsRepository.On("FindByUsername", "mock_username").Return(nil, &SkinNotFoundError{Who: "mock_username"}) + }, + AfterTest: func(suite *skinsystemTestSuite, response *http.Response) { + suite.Equal(204, response.StatusCode) + body, _ := ioutil.ReadAll(response.Body) + suite.Equal("", string(body)) + }, + }, + { + Name: "Username exists, but has no signed textures", + AllowProxy: false, + BeforeTest: func(suite *skinsystemTestSuite) { + skinModel := createSkinModel("mock_username", true) + skinModel.MojangTextures = "" + skinModel.MojangSignature = "" + suite.SkinsRepository.On("FindByUsername", "mock_username").Return(skinModel, nil) + }, + AfterTest: func(suite *skinsystemTestSuite, response *http.Response) { + suite.Equal(204, response.StatusCode) + body, _ := ioutil.ReadAll(response.Body) + suite.Equal("", string(body)) + }, + }, + { + Name: "Username not exists, but Mojang profile is available and proxying is enabled", + AllowProxy: true, + BeforeTest: func(suite *skinsystemTestSuite) { + suite.SkinsRepository.On("FindByUsername", "mock_username").Return(nil, &SkinNotFoundError{Who: "mock_username"}) + suite.MojangTexturesProvider.On("GetForUsername", "mock_username").Return(createMojangResponse(true, false), nil) + }, + AfterTest: func(suite *skinsystemTestSuite, response *http.Response) { + suite.Equal(200, response.StatusCode) + suite.Equal("application/json", response.Header.Get("Content-Type")) + body, _ := ioutil.ReadAll(response.Body) + suite.JSONEq(`{ + "id": "00000000000000000000000000000000", + "name": "mock_username", + "properties": [ + { + "name": "textures", + "value": "eyJ0aW1lc3RhbXAiOjE1NTYzOTg1NzIsInByb2ZpbGVJZCI6IjAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwIiwicHJvZmlsZU5hbWUiOiJtb2NrX3VzZXJuYW1lIiwidGV4dHVyZXMiOnsiU0tJTiI6eyJ1cmwiOiJodHRwOi8vbW9qYW5nL3NraW4ucG5nIn19fQ==" + }, + { + "name": "chrly", + "value": "how do you tame a horse in Minecraft?" + } + ] + }`, string(body)) + }, + }, + { + Name: "Username not exists, Mojang profile is unavailable too and proxying is enabled", + AllowProxy: true, + BeforeTest: func(suite *skinsystemTestSuite) { + suite.SkinsRepository.On("FindByUsername", "mock_username").Return(nil, &SkinNotFoundError{Who: "mock_username"}) + suite.MojangTexturesProvider.On("GetForUsername", "mock_username").Return(nil, nil) + }, + AfterTest: func(suite *skinsystemTestSuite, response *http.Response) { + suite.Equal(204, response.StatusCode) + body, _ := ioutil.ReadAll(response.Body) + suite.Equal("", string(body)) + }, + }, +} + +func (suite *skinsystemTestSuite) TestSignedTextures() { + for _, testCase := range signedTexturesTestsCases { + suite.RunSubTest(testCase.Name, func() { + suite.Logger.On("IncCounter", "signed_textures.request", int64(1)).Once() + testCase.BeforeTest(suite) + + var target string + if testCase.AllowProxy { + target = "http://chrly/textures/signed/mock_username?proxy=true" + } else { + target = "http://chrly/textures/signed/mock_username" + } + + req := httptest.NewRequest("GET", target, nil) + w := httptest.NewRecorder() + + suite.App.CreateHandler().ServeHTTP(w, req) + + testCase.AfterTest(suite, w.Result()) + }) + } +} + +/************************* + * Post skin tests cases * + *************************/ + +type postSkinTestCase struct { + Name string + Form io.Reader + ExpectSuccess bool + BeforeTest func(suite *skinsystemTestSuite) + AfterTest func(suite *skinsystemTestSuite, response *http.Response) +} + +var postSkinTestsCases = []*postSkinTestCase{ + { + Name: "Upload new identity with textures data", + Form: bytes.NewBufferString(url.Values{ + "identityId": {"1"}, + "username": {"mock_username"}, + "uuid": {"0f657aa8-bfbe-415d-b700-5750090d3af3"}, + "skinId": {"5"}, + "is1_8": {"0"}, + "isSlim": {"0"}, + "url": {"http://example.com/skin.png"}, + }.Encode()), + BeforeTest: func(suite *skinsystemTestSuite) { + suite.SkinsRepository.On("FindByUserId", 1).Return(nil, &SkinNotFoundError{Who: "unknown"}) + suite.SkinsRepository.On("FindByUsername", "mock_username").Return(nil, &SkinNotFoundError{Who: "mock_username"}) + suite.SkinsRepository.On("Save", mock.MatchedBy(func(model *model.Skin) bool { + suite.Equal(1, model.UserId) + suite.Equal("mock_username", model.Username) + suite.Equal("0f657aa8-bfbe-415d-b700-5750090d3af3", model.Uuid) + suite.Equal(5, model.SkinId) + suite.False(model.Is1_8) + suite.False(model.IsSlim) + suite.Equal("http://example.com/skin.png", model.Url) + + return true + })).Times(1).Return(nil) + }, + AfterTest: func(suite *skinsystemTestSuite, response *http.Response) { + suite.Equal(201, response.StatusCode) + body, _ := ioutil.ReadAll(response.Body) + suite.Empty(body) + }, + }, + { + Name: "Update exists identity by changing only textures data", + Form: bytes.NewBufferString(url.Values{ + "identityId": {"1"}, + "username": {"mock_username"}, + "uuid": {"0f657aa8-bfbe-415d-b700-5750090d3af3"}, + "skinId": {"5"}, + "is1_8": {"1"}, + "isSlim": {"1"}, + "url": {"http://textures-server.com/skin.png"}, + }.Encode()), + BeforeTest: func(suite *skinsystemTestSuite) { + suite.SkinsRepository.On("FindByUserId", 1).Return(createSkinModel("mock_username", false), nil) + suite.SkinsRepository.On("Save", mock.MatchedBy(func(model *model.Skin) bool { + suite.Equal(1, model.UserId) + suite.Equal("mock_username", model.Username) + suite.Equal("0f657aa8-bfbe-415d-b700-5750090d3af3", model.Uuid) + suite.Equal(5, model.SkinId) + suite.True(model.Is1_8) + suite.True(model.IsSlim) + suite.Equal("http://textures-server.com/skin.png", model.Url) + + return true + })).Times(1).Return(nil) + }, + AfterTest: func(suite *skinsystemTestSuite, response *http.Response) { + suite.Equal(201, response.StatusCode) + body, _ := ioutil.ReadAll(response.Body) + suite.Empty(body) + }, + }, + { + Name: "Update exists identity by changing its identityId", + Form: bytes.NewBufferString(url.Values{ + "identityId": {"2"}, + "username": {"mock_username"}, + "uuid": {"0f657aa8-bfbe-415d-b700-5750090d3af3"}, + "skinId": {"5"}, + "is1_8": {"0"}, + "isSlim": {"0"}, + "url": {"http://example.com/skin.png"}, + }.Encode()), + BeforeTest: func(suite *skinsystemTestSuite) { + suite.SkinsRepository.On("FindByUserId", 2).Return(nil, &SkinNotFoundError{Who: "unknown"}) + suite.SkinsRepository.On("FindByUsername", "mock_username").Return(createSkinModel("mock_username", false), nil) + suite.SkinsRepository.On("RemoveByUsername", "mock_username").Times(1).Return(nil) + suite.SkinsRepository.On("Save", mock.MatchedBy(func(model *model.Skin) bool { + suite.Equal(2, model.UserId) + suite.Equal("mock_username", model.Username) + suite.Equal("0f657aa8-bfbe-415d-b700-5750090d3af3", model.Uuid) + + return true + })).Times(1).Return(nil) + }, + AfterTest: func(suite *skinsystemTestSuite, response *http.Response) { + suite.Equal(201, response.StatusCode) + body, _ := ioutil.ReadAll(response.Body) + suite.Empty(body) + }, + }, + { + Name: "Update exists identity by changing its username", + Form: bytes.NewBufferString(url.Values{ + "identityId": {"1"}, + "username": {"changed_username"}, + "uuid": {"0f657aa8-bfbe-415d-b700-5750090d3af3"}, + "skinId": {"5"}, + "is1_8": {"0"}, + "isSlim": {"0"}, + "url": {"http://example.com/skin.png"}, + }.Encode()), + BeforeTest: func(suite *skinsystemTestSuite) { + suite.SkinsRepository.On("FindByUserId", 1).Return(createSkinModel("mock_username", false), nil) + suite.SkinsRepository.On("RemoveByUserId", 1).Times(1).Return(nil) + suite.SkinsRepository.On("Save", mock.MatchedBy(func(model *model.Skin) bool { + suite.Equal(1, model.UserId) + suite.Equal("changed_username", model.Username) + suite.Equal("0f657aa8-bfbe-415d-b700-5750090d3af3", model.Uuid) + + return true + })).Times(1).Return(nil) + }, + AfterTest: func(suite *skinsystemTestSuite, response *http.Response) { + suite.Equal(201, response.StatusCode) + body, _ := ioutil.ReadAll(response.Body) + suite.Empty(body) + }, + }, +} + +func (suite *skinsystemTestSuite) TestPostSkin() { + for _, testCase := range postSkinTestsCases { + suite.RunSubTest(testCase.Name, func() { + suite.Logger.On("IncCounter", "authentication.challenge", int64(1)).Once() + suite.Logger.On("IncCounter", "authentication.success", int64(1)).Once() + suite.Logger.On("IncCounter", "api.skins.post.request", int64(1)).Once() + suite.Logger.On("IncCounter", "api.skins.post.success", int64(1)).Once() + suite.Auth.On("Check", mock.Anything).Return(nil) + testCase.BeforeTest(suite) + + req := httptest.NewRequest("POST", "http://chrly/api/skins", testCase.Form) + req.Header.Add("Content-Type", "application/x-www-form-urlencoded") + w := httptest.NewRecorder() + + suite.App.CreateHandler().ServeHTTP(w, req) + + testCase.AfterTest(suite, w.Result()) + }) + } + + suite.RunSubTest("Get errors about required fields", func() { + suite.Logger.On("IncCounter", "authentication.challenge", int64(1)).Once() + suite.Logger.On("IncCounter", "authentication.success", int64(1)).Once() + suite.Logger.On("IncCounter", "api.skins.post.request", int64(1)).Once() + suite.Logger.On("IncCounter", "api.skins.post.validation_failed", int64(1)).Once() + suite.Auth.On("Check", mock.Anything).Return(nil) + + req := httptest.NewRequest("POST", "http://chrly/api/skins", bytes.NewBufferString(url.Values{ + "mojangTextures": {"someBase64EncodedString"}, + }.Encode())) + req.Header.Add("Content-Type", "application/x-www-form-urlencoded") + w := httptest.NewRecorder() + + suite.App.CreateHandler().ServeHTTP(w, req) + + resp := w.Result() + defer resp.Body.Close() + suite.Equal(400, resp.StatusCode) + body, _ := ioutil.ReadAll(resp.Body) + suite.JSONEq(`{ + "errors": { + "identityId": [ + "The identityId field is required", + "The identityId field must be numeric", + "The identityId field must be minimum 1 char" + ], + "skinId": [ + "The skinId field is required", + "The skinId field must be numeric", + "The skinId field must be minimum 1 char" + ], + "username": [ + "The username field is required" + ], + "uuid": [ + "The uuid field is required", + "The uuid field must contain valid UUID" + ], + "url": [ + "One of url or skin should be provided, but not both" + ], + "skin": [ + "One of url or skin should be provided, but not both" + ], + "mojangSignature": [ + "The mojangSignature field is required" + ] + } + }`, string(body)) + }) + + suite.RunSubTest("Send request without authorization", func() { + req := httptest.NewRequest("POST", "http://chrly/api/skins", nil) + req.Header.Add("Authorization", "Bearer invalid.jwt.token") + w := httptest.NewRecorder() + + suite.Auth.On("Check", mock.Anything).Return(&auth.Unauthorized{Reason: "Cannot parse passed JWT token"}) + suite.Logger.On("IncCounter", "authentication.challenge", int64(1)).Once() + suite.Logger.On("IncCounter", "authentication.failed", int64(1)).Once() + + suite.App.CreateHandler().ServeHTTP(w, req) + + resp := w.Result() + defer resp.Body.Close() + suite.Equal(403, resp.StatusCode) + body, _ := ioutil.ReadAll(resp.Body) + suite.JSONEq(`{ + "error": "Cannot parse passed JWT token" + }`, string(body)) + }) + + suite.RunSubTest("Upload textures with skin as file", func() { + suite.Logger.On("IncCounter", "authentication.challenge", int64(1)).Once() + suite.Logger.On("IncCounter", "authentication.success", int64(1)).Once() + suite.Logger.On("IncCounter", "api.skins.post.request", int64(1)).Once() + suite.Logger.On("IncCounter", "api.skins.post.validation_failed", int64(1)).Once() + suite.Auth.On("Check", mock.Anything).Return(nil) + + inputBody := &bytes.Buffer{} + writer := multipart.NewWriter(inputBody) + + part, _ := writer.CreateFormFile("skin", "char.png") + _, _ = part.Write(loadSkinFile()) + + _ = writer.WriteField("identityId", "1") + _ = writer.WriteField("username", "mock_user") + _ = writer.WriteField("uuid", "0f657aa8-bfbe-415d-b700-5750090d3af3") + _ = writer.WriteField("skinId", "5") + + err := writer.Close() + if err != nil { + panic(err) + } + + req := httptest.NewRequest("POST", "http://chrly/api/skins", inputBody) + req.Header.Add("Content-Type", writer.FormDataContentType()) + w := httptest.NewRecorder() + + suite.App.CreateHandler().ServeHTTP(w, req) + + resp := w.Result() + defer resp.Body.Close() + suite.Equal(400, resp.StatusCode) + responseBody, _ := ioutil.ReadAll(resp.Body) + suite.JSONEq(`{ + "errors": { + "skin": [ + "Skin uploading is temporary unavailable" + ] + } + }`, string(responseBody)) + }) +} + +/************************************** + * Delete skin by user id tests cases * + **************************************/ + +func (suite *skinsystemTestSuite) TestDeleteByUserId() { + suite.RunSubTest("Delete skin by its identity id", func() { + suite.Auth.On("Check", mock.Anything).Return(nil) + suite.SkinsRepository.On("FindByUserId", 1).Return(createSkinModel("mock_username", false), nil) + suite.SkinsRepository.On("RemoveByUserId", 1).Once().Return(nil) + suite.Logger.On("IncCounter", "authentication.challenge", int64(1)).Once() + suite.Logger.On("IncCounter", "authentication.success", int64(1)).Once() + suite.Logger.On("IncCounter", "api.skins.delete.request", int64(1)).Once() + suite.Logger.On("IncCounter", "api.skins.delete.success", int64(1)).Once() + + req := httptest.NewRequest("DELETE", "http://chrly/api/skins/id:1", nil) + w := httptest.NewRecorder() + + suite.App.CreateHandler().ServeHTTP(w, req) + + resp := w.Result() + defer resp.Body.Close() + suite.Equal(204, resp.StatusCode) + body, _ := ioutil.ReadAll(resp.Body) + suite.Empty(body) + }) + + suite.RunSubTest("Try to remove not exists identity id", func() { + suite.Auth.On("Check", mock.Anything).Return(nil) + suite.SkinsRepository.On("FindByUserId", 1).Return(nil, &SkinNotFoundError{Who: "unknown"}) + suite.Logger.On("IncCounter", "authentication.challenge", int64(1)).Once() + suite.Logger.On("IncCounter", "authentication.success", int64(1)).Once() + suite.Logger.On("IncCounter", "api.skins.delete.request", int64(1)).Once() + suite.Logger.On("IncCounter", "api.skins.delete.not_found", int64(1)).Once() + + req := httptest.NewRequest("DELETE", "http://chrly/api/skins/id:1", nil) + w := httptest.NewRecorder() + + suite.App.CreateHandler().ServeHTTP(w, req) + + resp := w.Result() + defer resp.Body.Close() + suite.Equal(404, resp.StatusCode) + body, _ := ioutil.ReadAll(resp.Body) + suite.JSONEq(`[ + "Cannot find record for requested user id" + ]`, string(body)) + }) +} + +/*************************************** + * Delete skin by username tests cases * + ***************************************/ + +func (suite *skinsystemTestSuite) TestDeleteByUsername() { + suite.RunSubTest("Delete skin by its identity username", func() { + suite.Auth.On("Check", mock.Anything).Return(nil) + suite.SkinsRepository.On("FindByUsername", "mock_username").Return(createSkinModel("mock_username", false), nil) + suite.SkinsRepository.On("RemoveByUserId", 1).Once().Return(nil) + suite.Logger.On("IncCounter", "authentication.challenge", int64(1)).Once() + suite.Logger.On("IncCounter", "authentication.success", int64(1)).Once() + suite.Logger.On("IncCounter", "api.skins.delete.request", int64(1)).Once() + suite.Logger.On("IncCounter", "api.skins.delete.success", int64(1)).Once() + + req := httptest.NewRequest("DELETE", "http://chrly/api/skins/mock_username", nil) + w := httptest.NewRecorder() + + suite.App.CreateHandler().ServeHTTP(w, req) + + resp := w.Result() + defer resp.Body.Close() + suite.Equal(204, resp.StatusCode) + body, _ := ioutil.ReadAll(resp.Body) + suite.Empty(body) + }) + + suite.RunSubTest("Try to remove not exists identity username", func() { + suite.Auth.On("Check", mock.Anything).Return(nil) + suite.SkinsRepository.On("FindByUsername", "mock_username").Return(nil, &SkinNotFoundError{Who: "mock_username"}) + suite.Logger.On("IncCounter", "authentication.challenge", int64(1)).Once() + suite.Logger.On("IncCounter", "authentication.success", int64(1)).Once() + suite.Logger.On("IncCounter", "api.skins.delete.request", int64(1)).Once() + suite.Logger.On("IncCounter", "api.skins.delete.not_found", int64(1)).Once() + + req := httptest.NewRequest("DELETE", "http://chrly/api/skins/mock_username", nil) + w := httptest.NewRecorder() + + suite.App.CreateHandler().ServeHTTP(w, req) + + resp := w.Result() + defer resp.Body.Close() + suite.Equal(404, resp.StatusCode) + body, _ := ioutil.ReadAll(resp.Body) + suite.JSONEq(`[ + "Cannot find record for requested username" + ]`, string(body)) + }) +} + +/**************** + * Custom tests * + ****************/ + +func TestParseUsername(t *testing.T) { + assert := testify.New(t) + assert.Equal("test", parseUsername("test.png"), "Function should trim .png at end") + assert.Equal("test", parseUsername("test"), "Function should return string itself, if it not contains .png at end") +} + +/************* + * Utilities * + *************/ + +func createSkinModel(username string, isSlim bool) *model.Skin { + return &model.Skin{ + UserId: 1, + Username: username, + Uuid: "0f657aa8-bfbe-415d-b700-5750090d3af3", // Use non nil UUID to pass validation in api tests + SkinId: 1, + Url: "http://chrly/skin.png", + MojangTextures: "mocked textures base64", + MojangSignature: "mocked signature", + IsSlim: isSlim, + } +} + +func createCape() []byte { + img := image.NewAlpha(image.Rect(0, 0, 64, 32)) + writer := &bytes.Buffer{} + _ = png.Encode(writer, img) + pngBytes, _ := ioutil.ReadAll(writer) + + return pngBytes +} + +func createCapeModel() *model.Cape { + return &model.Cape{File: bytes.NewReader(createCape())} +} + +func createMojangResponse(includeSkin bool, includeCape bool) *mojang.SignedTexturesResponse { + timeZone, _ := time.LoadLocation("Europe/Minsk") + textures := &mojang.TexturesProp{ + Timestamp: time.Date(2019, 4, 27, 23, 56, 12, 0, timeZone).Unix(), + ProfileID: "00000000000000000000000000000000", + ProfileName: "mock_username", + Textures: &mojang.TexturesResponse{}, + } + + if includeSkin { + textures.Textures.Skin = &mojang.SkinTexturesResponse{ + Url: "http://mojang/skin.png", + } + } + + if includeCape { + textures.Textures.Cape = &mojang.CapeTexturesResponse{ + Url: "http://mojang/cape.png", + } + } + + response := &mojang.SignedTexturesResponse{ + Id: "00000000000000000000000000000000", + Name: "mock_username", + Props: []*mojang.Property{ + { + Name: "textures", + Value: mojang.EncodeTextures(textures), + }, + }, + } + + return response +} + +// base64 https://github.com/mathiasbynens/small/blob/0ca3c51/png-transparent.png +var OnePxPng = []byte("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAACklEQVR4nGMAAQAABQABDQottAAAAABJRU5ErkJggg==") + +func loadSkinFile() []byte { + result := make([]byte, 92) + _, err := base64.StdEncoding.Decode(result, OnePxPng) + if err != nil { + panic(err) + } + + return result +} diff --git a/http/textures.go b/http/textures.go deleted file mode 100644 index 244cd25..0000000 --- a/http/textures.go +++ /dev/null @@ -1,61 +0,0 @@ -package http - -import ( - "encoding/json" - "net/http" - - "github.com/gorilla/mux" - - "github.com/elyby/chrly/api/mojang" -) - -func (cfg *Config) Textures(response http.ResponseWriter, request *http.Request) { - cfg.Logger.IncCounter("textures.request", 1) - username := parseUsername(mux.Vars(request)["username"]) - - var textures *mojang.TexturesResponse - skin, skinErr := cfg.SkinsRepo.FindByUsername(username) - _, capeErr := cfg.CapesRepo.FindByUsername(username) - if (skinErr == nil && skin.SkinId != 0) || capeErr == nil { - textures = &mojang.TexturesResponse{} - - if skinErr == nil && skin.SkinId != 0 { - skinTextures := &mojang.SkinTexturesResponse{ - Url: skin.Url, - } - - if skin.IsSlim { - skinTextures.Metadata = &mojang.SkinTexturesMetadata{ - Model: "slim", - } - } - - textures.Skin = skinTextures - } - - if capeErr == nil { - textures.Cape = &mojang.CapeTexturesResponse{ - Url: request.URL.Scheme + "://" + request.Host + "/cloaks/" + username, - } - } - } else { - mojangTextures, err := cfg.MojangTexturesProvider.GetForUsername(username) - if err != nil || mojangTextures == nil { - response.WriteHeader(http.StatusNoContent) - return - } - - texturesProp := mojangTextures.DecodeTextures() - if texturesProp == nil { - response.WriteHeader(http.StatusInternalServerError) - cfg.Logger.Error("Unable to find textures property") - return - } - - textures = texturesProp.Textures - } - - responseData, _ := json.Marshal(textures) - response.Header().Set("Content-Type", "application/json") - response.Write(responseData) -} diff --git a/http/textures_test.go b/http/textures_test.go deleted file mode 100644 index 9ffa91c..0000000 --- a/http/textures_test.go +++ /dev/null @@ -1,194 +0,0 @@ -package http - -import ( - "bytes" - "io/ioutil" - "net/http/httptest" - "testing" - - "github.com/golang/mock/gomock" - testify "github.com/stretchr/testify/assert" - - "github.com/elyby/chrly/db" - "github.com/elyby/chrly/model" -) - -func TestConfig_Textures(t *testing.T) { - t.Run("Obtain textures for exists user with only default skin", func(t *testing.T) { - assert := testify.New(t) - - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - config, mocks := setupMocks(ctrl) - - mocks.Log.EXPECT().IncCounter("textures.request", int64(1)) - - mocks.Skins.EXPECT().FindByUsername("mock_user").Return(createSkinModel("mock_user", false), nil) - mocks.Capes.EXPECT().FindByUsername("mock_user").Return(nil, &db.CapeNotFoundError{Who: "mock_user"}) - - req := httptest.NewRequest("GET", "http://chrly/textures/mock_user", nil) - w := httptest.NewRecorder() - - config.CreateHandler().ServeHTTP(w, req) - - resp := w.Result() - assert.Equal(200, resp.StatusCode) - assert.Equal("application/json", resp.Header.Get("Content-Type")) - response, _ := ioutil.ReadAll(resp.Body) - assert.JSONEq(`{ - "SKIN": { - "url": "http://chrly/skin.png" - } - }`, string(response)) - }) - - t.Run("Obtain textures for exists user with only slim skin", func(t *testing.T) { - assert := testify.New(t) - - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - config, mocks := setupMocks(ctrl) - - mocks.Log.EXPECT().IncCounter("textures.request", int64(1)) - - mocks.Skins.EXPECT().FindByUsername("mock_user").Return(createSkinModel("mock_user", true), nil) - mocks.Capes.EXPECT().FindByUsername("mock_user").Return(nil, &db.CapeNotFoundError{Who: "mock_user"}) - - req := httptest.NewRequest("GET", "http://chrly/textures/mock_user", nil) - w := httptest.NewRecorder() - - config.CreateHandler().ServeHTTP(w, req) - - resp := w.Result() - assert.Equal(200, resp.StatusCode) - assert.Equal("application/json", resp.Header.Get("Content-Type")) - response, _ := ioutil.ReadAll(resp.Body) - assert.JSONEq(`{ - "SKIN": { - "url": "http://chrly/skin.png", - "metadata": { - "model": "slim" - } - } - }`, string(response)) - }) - - t.Run("Obtain textures for exists user with only cape", func(t *testing.T) { - assert := testify.New(t) - - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - config, mocks := setupMocks(ctrl) - - mocks.Log.EXPECT().IncCounter("textures.request", int64(1)) - - mocks.Skins.EXPECT().FindByUsername("mock_user").Return(nil, &db.SkinNotFoundError{Who: "mock_user"}) - mocks.Capes.EXPECT().FindByUsername("mock_user").Return(&model.Cape{File: bytes.NewReader(createCape())}, nil) - - req := httptest.NewRequest("GET", "http://chrly/textures/mock_user", nil) - w := httptest.NewRecorder() - - config.CreateHandler().ServeHTTP(w, req) - - resp := w.Result() - assert.Equal(200, resp.StatusCode) - assert.Equal("application/json", resp.Header.Get("Content-Type")) - response, _ := ioutil.ReadAll(resp.Body) - assert.JSONEq(`{ - "CAPE": { - "url": "http://chrly/cloaks/mock_user" - } - }`, string(response)) - }) - - t.Run("Obtain textures for exists user with skin and cape", func(t *testing.T) { - assert := testify.New(t) - - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - config, mocks := setupMocks(ctrl) - - mocks.Log.EXPECT().IncCounter("textures.request", int64(1)) - - mocks.Skins.EXPECT().FindByUsername("mock_user").Return(createSkinModel("mock_user", false), nil) - mocks.Capes.EXPECT().FindByUsername("mock_user").Return(&model.Cape{File: bytes.NewReader(createCape())}, nil) - - req := httptest.NewRequest("GET", "http://chrly/textures/mock_user", nil) - w := httptest.NewRecorder() - - config.CreateHandler().ServeHTTP(w, req) - - resp := w.Result() - assert.Equal(200, resp.StatusCode) - assert.Equal("application/json", resp.Header.Get("Content-Type")) - response, _ := ioutil.ReadAll(resp.Body) - assert.JSONEq(`{ - "SKIN": { - "url": "http://chrly/skin.png" - }, - "CAPE": { - "url": "http://chrly/cloaks/mock_user" - } - }`, string(response)) - }) - - t.Run("Obtain textures for not exists user that exists in Mojang", func(t *testing.T) { - assert := testify.New(t) - - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - config, mocks := setupMocks(ctrl) - - mocks.Log.EXPECT().IncCounter("textures.request", int64(1)) - - mocks.Skins.EXPECT().FindByUsername("mock_username").Return(nil, &db.SkinNotFoundError{}) - mocks.Capes.EXPECT().FindByUsername("mock_username").Return(nil, &db.CapeNotFoundError{}) - mocks.MojangProvider.On("GetForUsername", "mock_username").Once().Return(createTexturesResponse(true, true), nil) - - req := httptest.NewRequest("GET", "http://chrly/textures/mock_username", nil) - w := httptest.NewRecorder() - - config.CreateHandler().ServeHTTP(w, req) - - resp := w.Result() - assert.Equal(200, resp.StatusCode) - assert.Equal("application/json", resp.Header.Get("Content-Type")) - response, _ := ioutil.ReadAll(resp.Body) - assert.JSONEq(`{ - "SKIN": { - "url": "http://mojang/skin.png" - }, - "CAPE": { - "url": "http://mojang/cape.png" - } - }`, string(response)) - }) - - t.Run("Obtain textures for not exists user that not exists in Mojang too", func(t *testing.T) { - assert := testify.New(t) - - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - config, mocks := setupMocks(ctrl) - - mocks.Log.EXPECT().IncCounter("textures.request", int64(1)) - - mocks.Skins.EXPECT().FindByUsername("mock_username").Return(nil, &db.SkinNotFoundError{}) - mocks.Capes.EXPECT().FindByUsername("mock_username").Return(nil, &db.CapeNotFoundError{}) - mocks.MojangProvider.On("GetForUsername", "mock_username").Once().Return(nil, nil) - - req := httptest.NewRequest("GET", "http://chrly/textures/mock_username", nil) - w := httptest.NewRecorder() - - config.CreateHandler().ServeHTTP(w, req) - - resp := w.Result() - assert.Equal(204, resp.StatusCode) - }) -} diff --git a/interfaces/auth.go b/interfaces/auth.go deleted file mode 100644 index 3f645f7..0000000 --- a/interfaces/auth.go +++ /dev/null @@ -1,7 +0,0 @@ -package interfaces - -import "net/http" - -type AuthChecker interface { - Check(req *http.Request) error -} diff --git a/interfaces/mock_interfaces/mock_auth.go b/interfaces/mock_interfaces/mock_auth.go deleted file mode 100644 index 6b78454..0000000 --- a/interfaces/mock_interfaces/mock_auth.go +++ /dev/null @@ -1,45 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: interfaces/auth.go - -package mock_interfaces - -import ( - gomock "github.com/golang/mock/gomock" - http "net/http" - reflect "reflect" -) - -// MockAuthChecker is a mock of AuthChecker interface -type MockAuthChecker struct { - ctrl *gomock.Controller - recorder *MockAuthCheckerMockRecorder -} - -// MockAuthCheckerMockRecorder is the mock recorder for MockAuthChecker -type MockAuthCheckerMockRecorder struct { - mock *MockAuthChecker -} - -// NewMockAuthChecker creates a new mock instance -func NewMockAuthChecker(ctrl *gomock.Controller) *MockAuthChecker { - mock := &MockAuthChecker{ctrl: ctrl} - mock.recorder = &MockAuthCheckerMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use -func (_m *MockAuthChecker) EXPECT() *MockAuthCheckerMockRecorder { - return _m.recorder -} - -// Check mocks base method -func (_m *MockAuthChecker) Check(req *http.Request) error { - ret := _m.ctrl.Call(_m, "Check", req) - ret0, _ := ret[0].(error) - return ret0 -} - -// Check indicates an expected call of Check -func (_mr *MockAuthCheckerMockRecorder) Check(arg0 interface{}) *gomock.Call { - return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "Check", reflect.TypeOf((*MockAuthChecker)(nil).Check), arg0) -} diff --git a/interfaces/mock_interfaces/mock_interfaces.go b/interfaces/mock_interfaces/mock_interfaces.go deleted file mode 100644 index 846ec92..0000000 --- a/interfaces/mock_interfaces/mock_interfaces.go +++ /dev/null @@ -1,131 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: interfaces/repositories.go - -package mock_interfaces - -import ( - model "github.com/elyby/chrly/model" - gomock "github.com/golang/mock/gomock" - reflect "reflect" -) - -// MockSkinsRepository is a mock of SkinsRepository interface -type MockSkinsRepository struct { - ctrl *gomock.Controller - recorder *MockSkinsRepositoryMockRecorder -} - -// MockSkinsRepositoryMockRecorder is the mock recorder for MockSkinsRepository -type MockSkinsRepositoryMockRecorder struct { - mock *MockSkinsRepository -} - -// NewMockSkinsRepository creates a new mock instance -func NewMockSkinsRepository(ctrl *gomock.Controller) *MockSkinsRepository { - mock := &MockSkinsRepository{ctrl: ctrl} - mock.recorder = &MockSkinsRepositoryMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use -func (_m *MockSkinsRepository) EXPECT() *MockSkinsRepositoryMockRecorder { - return _m.recorder -} - -// FindByUsername mocks base method -func (_m *MockSkinsRepository) FindByUsername(username string) (*model.Skin, error) { - ret := _m.ctrl.Call(_m, "FindByUsername", username) - ret0, _ := ret[0].(*model.Skin) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// FindByUsername indicates an expected call of FindByUsername -func (_mr *MockSkinsRepositoryMockRecorder) FindByUsername(arg0 interface{}) *gomock.Call { - return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "FindByUsername", reflect.TypeOf((*MockSkinsRepository)(nil).FindByUsername), arg0) -} - -// FindByUserId mocks base method -func (_m *MockSkinsRepository) FindByUserId(id int) (*model.Skin, error) { - ret := _m.ctrl.Call(_m, "FindByUserId", id) - ret0, _ := ret[0].(*model.Skin) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// FindByUserId indicates an expected call of FindByUserId -func (_mr *MockSkinsRepositoryMockRecorder) FindByUserId(arg0 interface{}) *gomock.Call { - return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "FindByUserId", reflect.TypeOf((*MockSkinsRepository)(nil).FindByUserId), arg0) -} - -// Save mocks base method -func (_m *MockSkinsRepository) Save(skin *model.Skin) error { - ret := _m.ctrl.Call(_m, "Save", skin) - ret0, _ := ret[0].(error) - return ret0 -} - -// Save indicates an expected call of Save -func (_mr *MockSkinsRepositoryMockRecorder) Save(arg0 interface{}) *gomock.Call { - return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "Save", reflect.TypeOf((*MockSkinsRepository)(nil).Save), arg0) -} - -// RemoveByUserId mocks base method -func (_m *MockSkinsRepository) RemoveByUserId(id int) error { - ret := _m.ctrl.Call(_m, "RemoveByUserId", id) - ret0, _ := ret[0].(error) - return ret0 -} - -// RemoveByUserId indicates an expected call of RemoveByUserId -func (_mr *MockSkinsRepositoryMockRecorder) RemoveByUserId(arg0 interface{}) *gomock.Call { - return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "RemoveByUserId", reflect.TypeOf((*MockSkinsRepository)(nil).RemoveByUserId), arg0) -} - -// RemoveByUsername mocks base method -func (_m *MockSkinsRepository) RemoveByUsername(username string) error { - ret := _m.ctrl.Call(_m, "RemoveByUsername", username) - ret0, _ := ret[0].(error) - return ret0 -} - -// RemoveByUsername indicates an expected call of RemoveByUsername -func (_mr *MockSkinsRepositoryMockRecorder) RemoveByUsername(arg0 interface{}) *gomock.Call { - return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "RemoveByUsername", reflect.TypeOf((*MockSkinsRepository)(nil).RemoveByUsername), arg0) -} - -// MockCapesRepository is a mock of CapesRepository interface -type MockCapesRepository struct { - ctrl *gomock.Controller - recorder *MockCapesRepositoryMockRecorder -} - -// MockCapesRepositoryMockRecorder is the mock recorder for MockCapesRepository -type MockCapesRepositoryMockRecorder struct { - mock *MockCapesRepository -} - -// NewMockCapesRepository creates a new mock instance -func NewMockCapesRepository(ctrl *gomock.Controller) *MockCapesRepository { - mock := &MockCapesRepository{ctrl: ctrl} - mock.recorder = &MockCapesRepositoryMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use -func (_m *MockCapesRepository) EXPECT() *MockCapesRepositoryMockRecorder { - return _m.recorder -} - -// FindByUsername mocks base method -func (_m *MockCapesRepository) FindByUsername(username string) (*model.Cape, error) { - ret := _m.ctrl.Call(_m, "FindByUsername", username) - ret0, _ := ret[0].(*model.Cape) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// FindByUsername indicates an expected call of FindByUsername -func (_mr *MockCapesRepositoryMockRecorder) FindByUsername(arg0 interface{}) *gomock.Call { - return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "FindByUsername", reflect.TypeOf((*MockCapesRepository)(nil).FindByUsername), arg0) -} diff --git a/interfaces/mock_wd/mock_wd.go b/interfaces/mock_wd/mock_wd.go deleted file mode 100644 index 0bdde12..0000000 --- a/interfaces/mock_wd/mock_wd.go +++ /dev/null @@ -1,218 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: github.com/mono83/slf/wd (interfaces: Watchdog) - -package mock_wd - -import ( - gomock "github.com/golang/mock/gomock" - slf "github.com/mono83/slf" - wd "github.com/mono83/slf/wd" - reflect "reflect" - time "time" -) - -// MockWatchdog is a mock of Watchdog interface -type MockWatchdog struct { - ctrl *gomock.Controller - recorder *MockWatchdogMockRecorder -} - -// MockWatchdogMockRecorder is the mock recorder for MockWatchdog -type MockWatchdogMockRecorder struct { - mock *MockWatchdog -} - -// NewMockWatchdog creates a new mock instance -func NewMockWatchdog(ctrl *gomock.Controller) *MockWatchdog { - mock := &MockWatchdog{ctrl: ctrl} - mock.recorder = &MockWatchdogMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use -func (_m *MockWatchdog) EXPECT() *MockWatchdogMockRecorder { - return _m.recorder -} - -// Alert mocks base method -func (_m *MockWatchdog) Alert(_param0 string, _param1 ...slf.Param) { - _s := []interface{}{_param0} - for _, _x := range _param1 { - _s = append(_s, _x) - } - _m.ctrl.Call(_m, "Alert", _s...) -} - -// Alert indicates an expected call of Alert -func (_mr *MockWatchdogMockRecorder) Alert(arg0 interface{}, arg1 ...interface{}) *gomock.Call { - _s := append([]interface{}{arg0}, arg1...) - return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "Alert", reflect.TypeOf((*MockWatchdog)(nil).Alert), _s...) -} - -// Debug mocks base method -func (_m *MockWatchdog) Debug(_param0 string, _param1 ...slf.Param) { - _s := []interface{}{_param0} - for _, _x := range _param1 { - _s = append(_s, _x) - } - _m.ctrl.Call(_m, "Debug", _s...) -} - -// Debug indicates an expected call of Debug -func (_mr *MockWatchdogMockRecorder) Debug(arg0 interface{}, arg1 ...interface{}) *gomock.Call { - _s := append([]interface{}{arg0}, arg1...) - return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "Debug", reflect.TypeOf((*MockWatchdog)(nil).Debug), _s...) -} - -// Emergency mocks base method -func (_m *MockWatchdog) Emergency(_param0 string, _param1 ...slf.Param) { - _s := []interface{}{_param0} - for _, _x := range _param1 { - _s = append(_s, _x) - } - _m.ctrl.Call(_m, "Emergency", _s...) -} - -// Emergency indicates an expected call of Emergency -func (_mr *MockWatchdogMockRecorder) Emergency(arg0 interface{}, arg1 ...interface{}) *gomock.Call { - _s := append([]interface{}{arg0}, arg1...) - return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "Emergency", reflect.TypeOf((*MockWatchdog)(nil).Emergency), _s...) -} - -// Error mocks base method -func (_m *MockWatchdog) Error(_param0 string, _param1 ...slf.Param) { - _s := []interface{}{_param0} - for _, _x := range _param1 { - _s = append(_s, _x) - } - _m.ctrl.Call(_m, "Error", _s...) -} - -// Error indicates an expected call of Error -func (_mr *MockWatchdogMockRecorder) Error(arg0 interface{}, arg1 ...interface{}) *gomock.Call { - _s := append([]interface{}{arg0}, arg1...) - return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "Error", reflect.TypeOf((*MockWatchdog)(nil).Error), _s...) -} - -// IncCounter mocks base method -func (_m *MockWatchdog) IncCounter(_param0 string, _param1 int64, _param2 ...slf.Param) { - _s := []interface{}{_param0, _param1} - for _, _x := range _param2 { - _s = append(_s, _x) - } - _m.ctrl.Call(_m, "IncCounter", _s...) -} - -// IncCounter indicates an expected call of IncCounter -func (_mr *MockWatchdogMockRecorder) IncCounter(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { - _s := append([]interface{}{arg0, arg1}, arg2...) - return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "IncCounter", reflect.TypeOf((*MockWatchdog)(nil).IncCounter), _s...) -} - -// Info mocks base method -func (_m *MockWatchdog) Info(_param0 string, _param1 ...slf.Param) { - _s := []interface{}{_param0} - for _, _x := range _param1 { - _s = append(_s, _x) - } - _m.ctrl.Call(_m, "Info", _s...) -} - -// Info indicates an expected call of Info -func (_mr *MockWatchdogMockRecorder) Info(arg0 interface{}, arg1 ...interface{}) *gomock.Call { - _s := append([]interface{}{arg0}, arg1...) - return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "Info", reflect.TypeOf((*MockWatchdog)(nil).Info), _s...) -} - -// RecordTimer mocks base method -func (_m *MockWatchdog) RecordTimer(_param0 string, _param1 time.Duration, _param2 ...slf.Param) { - _s := []interface{}{_param0, _param1} - for _, _x := range _param2 { - _s = append(_s, _x) - } - _m.ctrl.Call(_m, "RecordTimer", _s...) -} - -// RecordTimer indicates an expected call of RecordTimer -func (_mr *MockWatchdogMockRecorder) RecordTimer(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { - _s := append([]interface{}{arg0, arg1}, arg2...) - return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "RecordTimer", reflect.TypeOf((*MockWatchdog)(nil).RecordTimer), _s...) -} - -// Timer mocks base method -func (_m *MockWatchdog) Timer(_param0 string, _param1 ...slf.Param) slf.Timer { - _s := []interface{}{_param0} - for _, _x := range _param1 { - _s = append(_s, _x) - } - ret := _m.ctrl.Call(_m, "Timer", _s...) - ret0, _ := ret[0].(slf.Timer) - return ret0 -} - -// Timer indicates an expected call of Timer -func (_mr *MockWatchdogMockRecorder) Timer(arg0 interface{}, arg1 ...interface{}) *gomock.Call { - _s := append([]interface{}{arg0}, arg1...) - return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "Timer", reflect.TypeOf((*MockWatchdog)(nil).Timer), _s...) -} - -// Trace mocks base method -func (_m *MockWatchdog) Trace(_param0 string, _param1 ...slf.Param) { - _s := []interface{}{_param0} - for _, _x := range _param1 { - _s = append(_s, _x) - } - _m.ctrl.Call(_m, "Trace", _s...) -} - -// Trace indicates an expected call of Trace -func (_mr *MockWatchdogMockRecorder) Trace(arg0 interface{}, arg1 ...interface{}) *gomock.Call { - _s := append([]interface{}{arg0}, arg1...) - return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "Trace", reflect.TypeOf((*MockWatchdog)(nil).Trace), _s...) -} - -// UpdateGauge mocks base method -func (_m *MockWatchdog) UpdateGauge(_param0 string, _param1 int64, _param2 ...slf.Param) { - _s := []interface{}{_param0, _param1} - for _, _x := range _param2 { - _s = append(_s, _x) - } - _m.ctrl.Call(_m, "UpdateGauge", _s...) -} - -// UpdateGauge indicates an expected call of UpdateGauge -func (_mr *MockWatchdogMockRecorder) UpdateGauge(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { - _s := append([]interface{}{arg0, arg1}, arg2...) - return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "UpdateGauge", reflect.TypeOf((*MockWatchdog)(nil).UpdateGauge), _s...) -} - -// Warning mocks base method -func (_m *MockWatchdog) Warning(_param0 string, _param1 ...slf.Param) { - _s := []interface{}{_param0} - for _, _x := range _param1 { - _s = append(_s, _x) - } - _m.ctrl.Call(_m, "Warning", _s...) -} - -// Warning indicates an expected call of Warning -func (_mr *MockWatchdogMockRecorder) Warning(arg0 interface{}, arg1 ...interface{}) *gomock.Call { - _s := append([]interface{}{arg0}, arg1...) - return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "Warning", reflect.TypeOf((*MockWatchdog)(nil).Warning), _s...) -} - -// WithParams mocks base method -func (_m *MockWatchdog) WithParams(_param0 ...slf.Param) wd.Watchdog { - _s := []interface{}{} - for _, _x := range _param0 { - _s = append(_s, _x) - } - ret := _m.ctrl.Call(_m, "WithParams", _s...) - ret0, _ := ret[0].(wd.Watchdog) - return ret0 -} - -// WithParams indicates an expected call of WithParams -func (_mr *MockWatchdogMockRecorder) WithParams(arg0 ...interface{}) *gomock.Call { - return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "WithParams", reflect.TypeOf((*MockWatchdog)(nil).WithParams), arg0...) -} diff --git a/interfaces/repositories.go b/interfaces/repositories.go deleted file mode 100644 index 848045a..0000000 --- a/interfaces/repositories.go +++ /dev/null @@ -1,22 +0,0 @@ -package interfaces - -import ( - "github.com/elyby/chrly/api/mojang" - "github.com/elyby/chrly/model" -) - -type SkinsRepository interface { - FindByUsername(username string) (*model.Skin, error) - FindByUserId(id int) (*model.Skin, error) - Save(skin *model.Skin) error - RemoveByUserId(id int) error - RemoveByUsername(username string) error -} - -type CapesRepository interface { - FindByUsername(username string) (*model.Cape, error) -} - -type MojangTexturesProvider interface { - GetForUsername(username string) (*mojang.SignedTexturesResponse, error) -} From 5a0c10c1a1750f172878cb2bf35afb5680b7b907 Mon Sep 17 00:00:00 2001 From: ErickSkrauch Date: Fri, 3 Jan 2020 00:51:57 +0300 Subject: [PATCH 05/12] Implemented worker command --- .travis.yml | 2 +- bootstrap/bootstrap.go | 50 +++++-- cmd/root.go | 6 +- cmd/serve.go | 31 +--- cmd/version.go | 10 +- cmd/worker.go | 45 ++++++ http/uuids_worker.go | 89 +++++++++++ http/uuids_worker_test.go | 157 ++++++++++++++++++++ mojangtextures/mojang_textures.go | 6 +- mojangtextures/mojang_textures_test.go | 2 +- mojangtextures/remote_api_uuids_provider.go | 4 +- script/mocks | 4 - version/version.go | 14 ++ 13 files changed, 367 insertions(+), 53 deletions(-) create mode 100644 cmd/worker.go create mode 100644 http/uuids_worker.go create mode 100644 http/uuids_worker_test.go delete mode 100755 script/mocks create mode 100644 version/version.go diff --git a/.travis.yml b/.travis.yml index b91a909..339b236 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,7 +31,7 @@ jobs: env CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o release/chrly - -ldflags '-extldflags "-static" -X github.com/elyby/chrly/bootstrap.version=$APP_VERSION' + -ldflags '-extldflags "-static" -X github.com/elyby/chrly/version.version=$APP_VERSION -X github.com/elyby/chrly/version.commit=$TRAVIS_COMMIT' main.go - docker build -t elyby/chrly:$DOCKER_TAG . - docker push elyby/chrly:$DOCKER_TAG diff --git a/bootstrap/bootstrap.go b/bootstrap/bootstrap.go index d8774ee..4455e17 100644 --- a/bootstrap/bootstrap.go +++ b/bootstrap/bootstrap.go @@ -1,7 +1,9 @@ package bootstrap import ( + "net/url" "os" + "time" "github.com/getsentry/raven-go" "github.com/mono83/slf/rays" @@ -9,24 +11,23 @@ import ( "github.com/mono83/slf/recievers/statsd" "github.com/mono83/slf/recievers/writer" "github.com/mono83/slf/wd" + "github.com/spf13/viper" + + "github.com/elyby/chrly/mojangtextures" + "github.com/elyby/chrly/version" ) -var version = "" - -func GetVersion() string { - return version -} - func CreateLogger(statsdAddr string, sentryAddr string) (wd.Watchdog, error) { wd.AddReceiver(writer.New(writer.Options{ - Marker: false, + Marker: false, TimeFormat: "15:04:05.000", })) + if statsdAddr != "" { hostname, _ := os.Hostname() statsdReceiver, err := statsd.NewReceiver(statsd.Config{ - Address: statsdAddr, - Prefix: "ely.skinsystem." + hostname + ".app.", + Address: statsdAddr, + Prefix: "ely.skinsystem." + hostname + ".app.", FlushEvery: 1, }) @@ -45,7 +46,7 @@ func CreateLogger(statsdAddr string, sentryAddr string) (wd.Watchdog, error) { ravenClient.SetEnvironment("production") ravenClient.SetDefaultLoggerName("sentry-watchdog-receiver") - programVersion := GetVersion() + programVersion := version.Version() if programVersion != "" { raven.SetRelease(programVersion) } @@ -62,3 +63,32 @@ func CreateLogger(statsdAddr string, sentryAddr string) (wd.Watchdog, error) { return wd.New("", "").WithParams(rays.Host), nil } + +func init() { + viper.SetDefault("queue.loop_delay", 2*time.Second+500*time.Millisecond) + viper.SetDefault("queue.batch_size", 10) +} + +func CreateMojangUUIDsProvider(logger wd.Watchdog) (mojangtextures.UUIDsProvider, error) { + var uuidsProvider mojangtextures.UUIDsProvider + preferredUuidsProvider := viper.GetString("mojang_textures.uuids_provider.driver") + if preferredUuidsProvider == "remote" { + remoteUrl, err := url.Parse(viper.GetString("mojang_textures.uuids_provider.url")) + if err != nil { + return nil, err + } + + uuidsProvider = &mojangtextures.RemoteApiUuidsProvider{ + Url: *remoteUrl, + Logger: logger, + } + } else { + uuidsProvider = &mojangtextures.BatchUuidsProvider{ + IterationDelay: viper.GetDuration("queue.loop_delay"), + IterationSize: viper.GetInt("queue.batch_size"), + Logger: logger, + } + } + + return uuidsProvider, nil +} diff --git a/cmd/root.go b/cmd/root.go index 5b973bd..038161c 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -5,16 +5,16 @@ import ( "os" "strings" - "github.com/elyby/chrly/bootstrap" - "github.com/spf13/cobra" "github.com/spf13/viper" + + "github.com/elyby/chrly/version" ) var RootCmd = &cobra.Command{ Use: "chrly", Short: "Implementation of Minecraft skins system server", - Version: bootstrap.GetVersion(), + Version: version.Version(), } // Execute adds all child commands to the root command and sets flags appropriately. diff --git a/cmd/serve.go b/cmd/serve.go index 481e219..da05120 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -3,8 +3,6 @@ package cmd import ( "fmt" "log" - "net/url" - "time" "github.com/mono83/slf/wd" "github.com/spf13/cobra" @@ -19,7 +17,7 @@ import ( var serveCmd = &cobra.Command{ Use: "serve", - Short: "Starts http handler for the skins system", + Short: "Starts HTTP handler for the skins system", Run: func(cmd *cobra.Command, args []string) { // TODO: this is a mess, need to organize this code somehow to make services initialization more compact logger, err := bootstrap.CreateLogger(viper.GetString("statsd.addr"), viper.GetString("sentry.dsn")) @@ -55,32 +53,17 @@ var serveCmd = &cobra.Command{ return } - var uuidsProvider mojangtextures.UuidsProvider - preferredUuidsProvider := viper.GetString("mojang_textures.uuids_provider.driver") - if preferredUuidsProvider == "remote" { - remoteUrl, err := url.Parse(viper.GetString("mojang_textures.uuids_provider.url")) - if err != nil { - logger.Emergency("Unable to parse remote url :err", wd.ErrParam(err)) - return - } - - uuidsProvider = &mojangtextures.RemoteApiUuidsProvider{ - Url: *remoteUrl, - Logger: logger, - } - } else { - uuidsProvider = &mojangtextures.BatchUuidsProvider{ - IterationDelay: time.Duration(viper.GetInt("queue.loop_delay")) * time.Millisecond, - IterationSize: viper.GetInt("queue.batch_size"), - Logger: logger, - } + uuidsProvider, err := bootstrap.CreateMojangUUIDsProvider(logger) + if err != nil { + logger.Emergency("Unable to parse remote url :err", wd.ErrParam(err)) + return } texturesStorage := mojangtextures.NewInMemoryTexturesStorage() texturesStorage.Start() mojangTexturesProvider := &mojangtextures.Provider{ Logger: logger, - UuidsProvider: uuidsProvider, + UUIDsProvider: uuidsProvider, TexturesProvider: &mojangtextures.MojangApiTexturesProvider{ Logger: logger, }, @@ -115,6 +98,4 @@ func init() { viper.SetDefault("storage.redis.poll", 10) viper.SetDefault("storage.filesystem.basePath", "data") viper.SetDefault("storage.filesystem.capesDirName", "capes") - viper.SetDefault("queue.loop_delay", 2_500) - viper.SetDefault("queue.batch_size", 10) } diff --git a/cmd/version.go b/cmd/version.go index e1196fe..59cd038 100644 --- a/cmd/version.go +++ b/cmd/version.go @@ -2,18 +2,20 @@ package cmd import ( "fmt" + "runtime" "github.com/spf13/cobra" - "github.com/elyby/chrly/bootstrap" - "runtime" + + "github.com/elyby/chrly/version" ) var versionCmd = &cobra.Command{ Use: "version", Short: "Show the Chrly version information", Run: func(cmd *cobra.Command, args []string) { - fmt.Printf("Version: %s\n", bootstrap.GetVersion()) - fmt.Printf("Go version: %s\n", runtime.Version()) + fmt.Printf("Version: %s\n", version.Version()) + fmt.Printf("Commit: %s\n", version.Commit()) + fmt.Printf("Go version: %s\n", runtime.Version()) fmt.Printf("OS/Arch: %s/%s\n", runtime.GOOS, runtime.GOARCH) }, } diff --git a/cmd/worker.go b/cmd/worker.go new file mode 100644 index 0000000..13c1adb --- /dev/null +++ b/cmd/worker.go @@ -0,0 +1,45 @@ +package cmd + +import ( + "fmt" + "log" + + "github.com/mono83/slf/wd" + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "github.com/elyby/chrly/bootstrap" + "github.com/elyby/chrly/http" +) + +var workerCmd = &cobra.Command{ + Use: "worker", + Short: "Starts HTTP handler for the Mojang usernames to UUIDs worker", + Run: func(cmd *cobra.Command, args []string) { + logger, err := bootstrap.CreateLogger(viper.GetString("statsd.addr"), viper.GetString("sentry.dsn")) + if err != nil { + log.Fatal(fmt.Printf("Cannot initialize logger: %v", err)) + } + logger.Info("Logger successfully initialized") + + uuidsProvider, err := bootstrap.CreateMojangUUIDsProvider(logger) + if err != nil { + logger.Emergency("Unable to parse remote url :err", wd.ErrParam(err)) + return + } + + cfg := &http.UUIDsWorker{ + ListenSpec: fmt.Sprintf("%s:%d", viper.GetString("server.host"), viper.GetInt("server.port")), + UUIDsProvider: uuidsProvider, + Logger: logger, + } + + if err := cfg.Run(); err != nil { + logger.Error(fmt.Sprintf("Error in main(): %v", err)) + } + }, +} + +func init() { + RootCmd.AddCommand(workerCmd) +} diff --git a/http/uuids_worker.go b/http/uuids_worker.go new file mode 100644 index 0000000..5d08df5 --- /dev/null +++ b/http/uuids_worker.go @@ -0,0 +1,89 @@ +package http + +import ( + "encoding/json" + "fmt" + "net" + "net/http" + "time" + + "github.com/gorilla/mux" + "github.com/mono83/slf/wd" + + "github.com/elyby/chrly/api/mojang" + "github.com/elyby/chrly/mojangtextures" +) + +type UuidsProvider interface { + GetUuid(username string) (*mojang.ProfileInfo, error) +} + +type UUIDsWorker struct { + ListenSpec string + + UUIDsProvider mojangtextures.UUIDsProvider + Logger wd.Watchdog +} + +func (ctx *UUIDsWorker) Run() error { + ctx.Logger.Info(fmt.Sprintf("Starting, HTTP on: %s\n", ctx.ListenSpec)) + + listener, err := net.Listen("tcp", ctx.ListenSpec) + if err != nil { + return err + } + + server := &http.Server{ + ReadTimeout: 60 * time.Second, + WriteTimeout: 60 * time.Second, // TODO: should I adjust this values? + MaxHeaderBytes: 1 << 16, + Handler: ctx.CreateHandler(), + } + + // noinspection GoUnhandledErrorResult + go server.Serve(listener) + + s := waitForSignal() + ctx.Logger.Info(fmt.Sprintf("Got signal: %v, exiting.", s)) + + return nil +} + +func (ctx *UUIDsWorker) CreateHandler() http.Handler { + router := mux.NewRouter().StrictSlash(true) + router.NotFoundHandler = http.HandlerFunc(NotFound) + + router.Handle("/api/worker/mojang-uuid/{username}", http.HandlerFunc(ctx.GetUUID)).Methods("GET") + + return router +} + +func (ctx *UUIDsWorker) GetUUID(response http.ResponseWriter, request *http.Request) { + username := parseUsername(mux.Vars(request)["username"]) + profile, err := ctx.UUIDsProvider.GetUuid(username) + if err != nil { + if _, ok := err.(*mojang.TooManyRequestsError); ok { + ctx.Logger.Warning("Got 429 Too Many Requests") + response.WriteHeader(http.StatusTooManyRequests) + return + } + + ctx.Logger.Warning("Got non success response: :err", wd.ErrParam(err)) + response.Header().Set("Content-Type", "application/json") + response.WriteHeader(http.StatusInternalServerError) + result, _ := json.Marshal(map[string]interface{}{ + "provider": err.Error(), + }) + _, _ = response.Write(result) + return + } + + if profile == nil { + response.WriteHeader(http.StatusNoContent) + return + } + + response.Header().Set("Content-Type", "application/json") + responseData, _ := json.Marshal(profile) + _, _ = response.Write(responseData) +} diff --git a/http/uuids_worker_test.go b/http/uuids_worker_test.go new file mode 100644 index 0000000..ea0e64e --- /dev/null +++ b/http/uuids_worker_test.go @@ -0,0 +1,157 @@ +package http + +import ( + "errors" + "github.com/elyby/chrly/api/mojang" + "github.com/elyby/chrly/tests" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" + "io/ioutil" + "net/http" + "net/http/httptest" + "testing" +) + +/*************** + * Setup mocks * + ***************/ + +type uuidsProviderMock struct { + mock.Mock +} + +func (m *uuidsProviderMock) GetUuid(username string) (*mojang.ProfileInfo, error) { + args := m.Called(username) + var result *mojang.ProfileInfo + if casted, ok := args.Get(0).(*mojang.ProfileInfo); ok { + result = casted + } + + return result, args.Error(1) +} + +type uuidsWorkerTestSuite struct { + suite.Suite + + App *UUIDsWorker + + UuidsProvider *uuidsProviderMock + Logger *tests.WdMock +} + +/******************** + * Setup test suite * + ********************/ + +func (suite *uuidsWorkerTestSuite) SetupTest() { + suite.UuidsProvider = &uuidsProviderMock{} + suite.Logger = &tests.WdMock{} + + suite.App = &UUIDsWorker{ + UUIDsProvider: suite.UuidsProvider, + Logger: suite.Logger, + } +} + +func (suite *uuidsWorkerTestSuite) TearDownTest() { + suite.UuidsProvider.AssertExpectations(suite.T()) + suite.Logger.AssertExpectations(suite.T()) +} + +func (suite *uuidsWorkerTestSuite) RunSubTest(name string, subTest func()) { + suite.SetupTest() + suite.Run(name, subTest) + suite.TearDownTest() +} + +/************* + * Run tests * + *************/ + +func TestUUIDsWorker(t *testing.T) { + suite.Run(t, new(uuidsWorkerTestSuite)) +} + +type uuidsWorkerTestCase struct { + Name string + BeforeTest func(suite *uuidsWorkerTestSuite) + AfterTest func(suite *uuidsWorkerTestSuite, response *http.Response) +} + +/************************ + * Get UUID tests cases * + ************************/ + +var getUuidTestsCases = []*uuidsWorkerTestCase{ + { + Name: "Success provider response", + BeforeTest: func(suite *uuidsWorkerTestSuite) { + suite.UuidsProvider.On("GetUuid", "mock_username").Return(&mojang.ProfileInfo{ + Id: "0fcc38620f1845f3a54e1b523c1bd1c7", + Name: "mock_username", + }, nil) + }, + AfterTest: func(suite *uuidsWorkerTestSuite, response *http.Response) { + suite.Equal(200, response.StatusCode) + suite.Equal("application/json", response.Header.Get("Content-Type")) + body, _ := ioutil.ReadAll(response.Body) + suite.JSONEq(`{ + "id": "0fcc38620f1845f3a54e1b523c1bd1c7", + "name": "mock_username" + }`, string(body)) + }, + }, + { + Name: "Receive empty response from UUIDs provider", + BeforeTest: func(suite *uuidsWorkerTestSuite) { + suite.UuidsProvider.On("GetUuid", "mock_username").Return(nil, nil) + }, + AfterTest: func(suite *uuidsWorkerTestSuite, response *http.Response) { + suite.Equal(204, response.StatusCode) + body, _ := ioutil.ReadAll(response.Body) + suite.Assert().Empty(body) + }, + }, + { + Name: "Receive error from UUIDs provider", + BeforeTest: func(suite *uuidsWorkerTestSuite) { + suite.UuidsProvider.On("GetUuid", "mock_username").Return(nil, errors.New("this is an error")) + suite.Logger.On("Warning", "Got non success response: :err", mock.Anything).Times(1) + }, + AfterTest: func(suite *uuidsWorkerTestSuite, response *http.Response) { + suite.Equal(500, response.StatusCode) + suite.Equal("application/json", response.Header.Get("Content-Type")) + body, _ := ioutil.ReadAll(response.Body) + suite.JSONEq(`{ + "provider": "this is an error" + }`, string(body)) + }, + }, + { + Name: "Receive Too Many Requests from UUIDs provider", + BeforeTest: func(suite *uuidsWorkerTestSuite) { + suite.UuidsProvider.On("GetUuid", "mock_username").Return(nil, &mojang.TooManyRequestsError{}) + suite.Logger.On("Warning", "Got 429 Too Many Requests").Times(1) + }, + AfterTest: func(suite *uuidsWorkerTestSuite, response *http.Response) { + suite.Equal(429, response.StatusCode) + body, _ := ioutil.ReadAll(response.Body) + suite.Empty(body) + }, + }, +} + +func (suite *uuidsWorkerTestSuite) TestGetUUID() { + for _, testCase := range getUuidTestsCases { + suite.RunSubTest(testCase.Name, func() { + testCase.BeforeTest(suite) + + req := httptest.NewRequest("GET", "http://chrly/api/worker/mojang-uuid/mock_username", nil) + w := httptest.NewRecorder() + + suite.App.CreateHandler().ServeHTTP(w, req) + + testCase.AfterTest(suite, w.Result()) + }) + } +} diff --git a/mojangtextures/mojang_textures.go b/mojangtextures/mojang_textures.go index 33919c6..2212cd7 100644 --- a/mojangtextures/mojang_textures.go +++ b/mojangtextures/mojang_textures.go @@ -69,7 +69,7 @@ func (c *broadcaster) BroadcastAndRemove(username string, result *broadcastResul // https://help.mojang.com/customer/portal/articles/928638 var allowedUsernamesRegex = regexp.MustCompile(`^[\w_]{3,16}$`) -type UuidsProvider interface { +type UUIDsProvider interface { GetUuid(username string) (*mojang.ProfileInfo, error) } @@ -78,7 +78,7 @@ type TexturesProvider interface { } type Provider struct { - UuidsProvider + UUIDsProvider TexturesProvider Storage Logger wd.Watchdog @@ -139,7 +139,7 @@ func (ctx *Provider) getResultAndBroadcast(username string, uuid string) { func (ctx *Provider) getResult(username string, uuid string) *broadcastResult { if uuid == "" { - profile, err := ctx.UuidsProvider.GetUuid(username) + profile, err := ctx.UUIDsProvider.GetUuid(username) if err != nil { ctx.handleMojangApiResponseError(err, "usernames") return &broadcastResult{nil, err} diff --git a/mojangtextures/mojang_textures_test.go b/mojangtextures/mojang_textures_test.go index fabdee6..e9c2ce2 100644 --- a/mojangtextures/mojang_textures_test.go +++ b/mojangtextures/mojang_textures_test.go @@ -158,7 +158,7 @@ func (suite *providerTestSuite) SetupTest() { suite.Logger = &mocks.WdMock{} suite.Provider = &Provider{ - UuidsProvider: suite.UuidsProvider, + UUIDsProvider: suite.UuidsProvider, TexturesProvider: suite.TexturesProvider, Storage: suite.Storage, Logger: suite.Logger, diff --git a/mojangtextures/remote_api_uuids_provider.go b/mojangtextures/remote_api_uuids_provider.go index 0980176..4d05b49 100644 --- a/mojangtextures/remote_api_uuids_provider.go +++ b/mojangtextures/remote_api_uuids_provider.go @@ -2,6 +2,7 @@ package mojangtextures import ( "encoding/json" + "github.com/elyby/chrly/version" "io/ioutil" "net/http" . "net/url" @@ -11,7 +12,6 @@ import ( "github.com/mono83/slf/wd" "github.com/elyby/chrly/api/mojang" - "github.com/elyby/chrly/bootstrap" ) var HttpClient = &http.Client{ @@ -34,7 +34,7 @@ func (ctx *RemoteApiUuidsProvider) GetUuid(username string) (*mojang.ProfileInfo request, _ := http.NewRequest("GET", url.String(), nil) request.Header.Add("Accept", "application/json") // Change default User-Agent to allow specify "Username -> UUID at time" Mojang's api endpoint - request.Header.Add("User-Agent", "Chrly/"+bootstrap.GetVersion()) + request.Header.Add("User-Agent", "Chrly/"+version.Version()) start := time.Now() response, err := HttpClient.Do(request) diff --git a/script/mocks b/script/mocks deleted file mode 100755 index 66a61ef..0000000 --- a/script/mocks +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh - -mockgen -source=interfaces/repositories.go -destination=interfaces/mock_interfaces/mock_interfaces.go -mockgen -source=interfaces/auth.go -destination=interfaces/mock_interfaces/mock_auth.go diff --git a/version/version.go b/version/version.go new file mode 100644 index 0000000..89e6def --- /dev/null +++ b/version/version.go @@ -0,0 +1,14 @@ +package version + +var ( + version = "" + commit = "" +) + +func Version() string { + return version +} + +func Commit() string { + return commit +} From 7f9b60ab3ae1b4ad8b1fd526ccd77ec698a578be Mon Sep 17 00:00:00 2001 From: ErickSkrauch Date: Fri, 3 Jan 2020 01:04:23 +0300 Subject: [PATCH 06/12] Fix race condition error --- mojangtextures/batch_uuids_provider.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/mojangtextures/batch_uuids_provider.go b/mojangtextures/batch_uuids_provider.go index d3259c1..b0b1826 100644 --- a/mojangtextures/batch_uuids_provider.go +++ b/mojangtextures/batch_uuids_provider.go @@ -41,8 +41,8 @@ func (s *jobsQueue) Dequeue(n int) []*jobItem { s.lock.Lock() defer s.lock.Unlock() - if n > s.Size() { - n = s.Size() + if n > s.size() { + n = s.size() } items := s.items[0:n] @@ -52,6 +52,13 @@ func (s *jobsQueue) Dequeue(n int) []*jobItem { } func (s *jobsQueue) Size() int { + s.lock.Lock() + defer s.lock.Unlock() + + return s.size() +} + +func (s *jobsQueue) size() int { return len(s.items) } From a4a92010346d2e85c5cc0a0321dc3600396da141 Mon Sep 17 00:00:00 2001 From: ErickSkrauch Date: Fri, 3 Jan 2020 01:41:51 +0300 Subject: [PATCH 07/12] Add additional synchronization layer for bath_uuids_provider_test --- mojangtextures/batch_uuids_provider_test.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/mojangtextures/batch_uuids_provider_test.go b/mojangtextures/batch_uuids_provider_test.go index e2a614c..24bfc82 100644 --- a/mojangtextures/batch_uuids_provider_test.go +++ b/mojangtextures/batch_uuids_provider_test.go @@ -4,6 +4,7 @@ import ( "crypto/rand" "encoding/base64" "strings" + "sync" "testing" "time" @@ -116,7 +117,11 @@ func (suite *batchUuidsProviderTestSuite) SetupTest() { suite.iterateChan <- false } + var lock sync.Mutex suite.GetUuidAsync = func(username string) chan *batchUuidsProviderGetUuidResult { + lock.Lock() + defer lock.Unlock() + c := make(chan *batchUuidsProviderGetUuidResult) s := make(chan int) go func() { @@ -211,7 +216,7 @@ func (suite *batchUuidsProviderTestSuite) TestGetUuidForMoreThan10Usernames() { suite.MojangApi.On("UsernamesToUuids", usernames[0:10]).Once().Return([]*mojang.ProfileInfo{}, nil) suite.MojangApi.On("UsernamesToUuids", usernames[10:12]).Once().Return([]*mojang.ProfileInfo{}, nil) - channels := make([]chan *batchUuidsProviderGetUuidResult, 12) + channels := make([]chan *batchUuidsProviderGetUuidResult, len(usernames)) for i, username := range usernames { channels[i] = suite.GetUuidAsync(username) } From 9946eae73ba8603e839764c094261d0510e5d35e Mon Sep 17 00:00:00 2001 From: ErickSkrauch Date: Sun, 5 Jan 2020 20:19:41 +0300 Subject: [PATCH 08/12] Update docs --- CHANGELOG.md | 20 +++++++++-- README.md | 80 +++++++++++++++++++++++++++++++++++++---- docker-compose.prod.yml | 9 +++++ docker-entrypoint.sh | 2 +- http/skinsystem.go | 2 +- http/uuids_worker.go | 2 +- 6 files changed, 103 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6867097..51e18fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] - xxxx-xx-xx ### Added -- Added remote mode for Mojang's textures queue. +- Remote mode for Mojang's textures queue with a new configuration params: `MOJANG_TEXTURES_UUIDS_PROVIDER_DRIVER` and + `MOJANG_TEXTURES_UUIDS_PROVIDER_URL`. + + For example, to send requests directly to [Mojang's APIs](https://wiki.vg/Mojang_API#Username_-.3E_UUID_at_time), + set the next configuration: + - `MOJANG_TEXTURES_UUIDS_PROVIDER_DRIVER=remote` + - `MOJANG_TEXTURES_UUIDS_PROVIDER_URL=https://api.mojang.com/users/profiles/minecraft/` +- Implemented worker mode. The app starts with the only one API endpoint: `/api/worker/mojang-uuid/{username}`, + which is compatible with [Mojang's endpoint](https://wiki.vg/Mojang_API#Username_-.3E_UUID_at_time) to exchange + username to its UUID. It can be used with some load balancing software to increase throughput of Mojang's textures + proxy by splitting the load across multiple servers with its own IPs. + - New StatsD metrics: - Counters: - `ely.skinsystem.{hostname}.app.mojang_textures.usernames.textures_hit` @@ -14,10 +25,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - `ely.skinsystem.{hostname}.app.mojang_textures.usernames.iteration_size` and - `ely.skinsystem.{hostname}.app.mojang_textures.usernames.queue_size` are now updated even if the queue is empty. + `ely.skinsystem.{hostname}.app.mojang_textures.usernames.queue_size` are now updates even if the queue is empty. ### Changed -- Event `ely.skinsystem.{hostname}.app.mojang_textures.already_in_queue` has been renamed into `ely.skinsystem.{hostname}.app.mojang_textures.already_scheduled`. +- **BREAKING**: `QUEUE_LOOP_DELAY` param is now sets as a Go duration, not milliseconds. + For example, default value is now `2s500ms`. +- **BREAKING**: Event `ely.skinsystem.{hostname}.app.mojang_textures.already_in_queue` has been renamed into + `ely.skinsystem.{hostname}.app.mojang_textures.already_scheduled`. ## [4.3.0] - 2019-11-08 ### Added diff --git a/README.md b/README.md index 60b2b88..98a14a2 100644 --- a/README.md +++ b/README.md @@ -57,12 +57,80 @@ docker-compose up -d app **Variables to adjust:** -| ENV | Description | Example | -|--------------------|-------------------------------------------------------------------------------------------------|-------------------------------------------| -| STORAGE_REDIS_POOL | By default, Chrly creates pool with 10 connection, but you may want to increase it | `20` | -| STATSD_ADDR | StatsD can be used to collect metrics | `localhost:8125` | -| SENTRY_DSN | Sentry can be used to collect app errors | `https://public:private@your.sentry.io/1` | -| QUEUE_LOOP_DELAY | Parameter is sets the delay before each iteration of the Mojang's textures queue (milliseconds) | `3200` | + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ENVDescriptionExample
STORAGE_REDIS_HOST + By default, Chrly tries to connect to the redis host + (by service name in docker-compose configuration). + localhost
STORAGE_REDIS_PORT + Specifies the Redis connection port. + 6379
STORAGE_REDIS_POOLBy default, Chrly creates pool with 10 connection, but you may want to increase it20
STATSD_ADDRStatsD can be used to collect metricslocalhost:8125
SENTRY_DSNSentry can be used to collect app errorshttps://public:private@your.sentry.io/1
QUEUE_LOOP_DELAY + Parameter is sets the delay before each iteration of the Mojang's textures queue + (Go's duration) + 3s200ms
QUEUE_BATCH_SIZE + Sets the count of usernames, which will be sent to the + Mojang's API to exchange them to their UUIDs. + The current limit is 10, but it may change in the future, so you may want to adjust it. + 10
MOJANG_TEXTURES_UUIDS_PROVIDER_DRIVER + Specifies the preferred provider of the Mojang's UUIDs. Takes remote value. + In any other case, the local queue will be used. + remote
MOJANG_TEXTURES_UUIDS_PROVIDER_URL + When the UUIDs driver set to remote, sets the remote URL. + The trailing slash won't cause any problems. + http://remote-provider.com/api/worker/mojang-uuid
If something goes wrong, you can always access logs by executing `docker-compose logs -f app`. diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index ca4e940..fa306ac 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -20,6 +20,15 @@ services: environment: CHRLY_SECRET: replace_this_value_in_production + # Use this configuration in case when you need a remote Mojang UUIDs provider + # worker: + # image: elyby/chrly + # hostname: chrly0 + # restart: always + # ports: + # - "8080:80" + # command: ["worker"] + redis: image: redis:4.0-32bit # 32-bit version is recommended to spare some memory restart: always diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index de3fa34..cefed94 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -5,7 +5,7 @@ if [ ! -d /data/capes ]; then mkdir -p /data/capes fi -if [ "$1" = "serve" ] || [ "$1" = "token" ] || [ "$1" = "version" ]; then +if [ "$1" = "serve" ] || [ "$1" = "worker" ] || [ "$1" = "token" ] || [ "$1" = "version" ]; then set -- /usr/local/bin/chrly "$@" fi diff --git a/http/skinsystem.go b/http/skinsystem.go index c7fc2a9..2e2d80f 100644 --- a/http/skinsystem.go +++ b/http/skinsystem.go @@ -97,7 +97,7 @@ type Skinsystem struct { } func (ctx *Skinsystem) Run() error { - ctx.Logger.Info(fmt.Sprintf("Starting, HTTP on: %s\n", ctx.ListenSpec)) + ctx.Logger.Info(fmt.Sprintf("Starting the app, HTTP on: %s\n", ctx.ListenSpec)) listener, err := net.Listen("tcp", ctx.ListenSpec) if err != nil { diff --git a/http/uuids_worker.go b/http/uuids_worker.go index 5d08df5..4765343 100644 --- a/http/uuids_worker.go +++ b/http/uuids_worker.go @@ -26,7 +26,7 @@ type UUIDsWorker struct { } func (ctx *UUIDsWorker) Run() error { - ctx.Logger.Info(fmt.Sprintf("Starting, HTTP on: %s\n", ctx.ListenSpec)) + ctx.Logger.Info(fmt.Sprintf("Starting the worker, HTTP on: %s\n", ctx.ListenSpec)) listener, err := net.Listen("tcp", ctx.ListenSpec) if err != nil { From 17f82ec6d371cfe093ae2c4f263567288ab9a9d6 Mon Sep 17 00:00:00 2001 From: ErickSkrauch Date: Sun, 5 Jan 2020 20:39:17 +0300 Subject: [PATCH 09/12] Resolve golangcibot issues --- cmd/root.go | 9 +++++++++ cmd/serve.go | 18 +++++++++++++++--- cmd/worker.go | 18 +++++++++++++++--- db/filesystem.go | 4 ++-- db/redis.go | 6 +++--- http/http.go | 10 ---------- http/skinsystem.go | 11 +++-------- http/uuids_worker.go | 8 +------- 8 files changed, 48 insertions(+), 36 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 038161c..eb17903 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -3,7 +3,9 @@ package cmd import ( "fmt" "os" + "os/signal" "strings" + "syscall" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -35,3 +37,10 @@ func initConfig() { replacer := strings.NewReplacer(".", "_") viper.SetEnvKeyReplacer(replacer) } + +func waitForExitSignal() os.Signal { + ch := make(chan os.Signal, 1) + signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM) + + return <-ch +} diff --git a/cmd/serve.go b/cmd/serve.go index da05120..01b7c75 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -83,9 +83,21 @@ var serveCmd = &cobra.Command{ Auth: &auth.JwtAuth{Key: []byte(viper.GetString("chrly.secret"))}, } - if err := cfg.Run(); err != nil { - logger.Error(fmt.Sprintf("Error in main(): %v", err)) - } + finishChan := make(chan bool) + go func() { + if err := cfg.Run(); err != nil { + logger.Error(fmt.Sprintf("Error in main(): %v", err)) + finishChan <- true + } + }() + + go func() { + s := waitForExitSignal() + logger.Info(fmt.Sprintf("Got signal: %v, exiting.", s)) + finishChan <- true + }() + + <-finishChan }, } diff --git a/cmd/worker.go b/cmd/worker.go index 13c1adb..79ef7ed 100644 --- a/cmd/worker.go +++ b/cmd/worker.go @@ -34,9 +34,21 @@ var workerCmd = &cobra.Command{ Logger: logger, } - if err := cfg.Run(); err != nil { - logger.Error(fmt.Sprintf("Error in main(): %v", err)) - } + finishChan := make(chan bool) + go func() { + if err := cfg.Run(); err != nil { + logger.Error(fmt.Sprintf("Error in main(): %v", err)) + finishChan <- true + } + }() + + go func() { + s := waitForExitSignal() + logger.Info(fmt.Sprintf("Got signal: %v, exiting.", s)) + finishChan <- true + }() + + <-finishChan }, } diff --git a/db/filesystem.go b/db/filesystem.go index a9c9030..d94e53f 100644 --- a/db/filesystem.go +++ b/db/filesystem.go @@ -49,13 +49,13 @@ type filesStorage struct { func (repository *filesStorage) FindByUsername(username string) (*model.Cape, error) { if username == "" { - return nil, &http.CapeNotFoundError{username} + return nil, &http.CapeNotFoundError{Who: username} } capePath := path.Join(repository.path, strings.ToLower(username)+".png") file, err := os.Open(capePath) if err != nil { - return nil, &http.CapeNotFoundError{username} + return nil, &http.CapeNotFoundError{Who: username} } return &model.Cape{ diff --git a/db/redis.go b/db/redis.go index d07e5b6..ec218b6 100644 --- a/db/redis.go +++ b/db/redis.go @@ -148,13 +148,13 @@ func (db *redisDb) StoreUuid(username string, uuid string) error { func findByUsername(username string, conn util.Cmder) (*model.Skin, error) { if username == "" { - return nil, &http.SkinNotFoundError{username} + return nil, &http.SkinNotFoundError{Who: username} } redisKey := buildUsernameKey(username) response := conn.Cmd("GET", redisKey) if !response.IsType(redis.Str) { - return nil, &http.SkinNotFoundError{username} + return nil, &http.SkinNotFoundError{Who: username} } encodedResult, err := response.Bytes() @@ -181,7 +181,7 @@ func findByUsername(username string, conn util.Cmder) (*model.Skin, error) { func findByUserId(id int, conn util.Cmder) (*model.Skin, error) { response := conn.Cmd("HGET", accountIdToUsernameKey, id) if !response.IsType(redis.Str) { - return nil, &http.SkinNotFoundError{"unknown"} + return nil, &http.SkinNotFoundError{Who: "unknown"} } username, _ := response.Str() diff --git a/http/http.go b/http/http.go index 8286b28..aec1182 100644 --- a/http/http.go +++ b/http/http.go @@ -3,9 +3,6 @@ package http import ( "encoding/json" "net/http" - "os" - "os/signal" - "syscall" ) func NotFound(response http.ResponseWriter, _ *http.Request) { @@ -19,13 +16,6 @@ func NotFound(response http.ResponseWriter, _ *http.Request) { _, _ = response.Write(data) } -func waitForSignal() os.Signal { - ch := make(chan os.Signal) - signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM) - - return <-ch -} - func apiBadRequest(resp http.ResponseWriter, errorsPerField map[string][]string) { resp.WriteHeader(http.StatusBadRequest) resp.Header().Set("Content-Type", "application/json") diff --git a/http/skinsystem.go b/http/skinsystem.go index 2e2d80f..659db29 100644 --- a/http/skinsystem.go +++ b/http/skinsystem.go @@ -111,12 +111,7 @@ func (ctx *Skinsystem) Run() error { Handler: ctx.CreateHandler(), } - go server.Serve(listener) - - s := waitForSignal() - ctx.Logger.Info(fmt.Sprintf("Got signal: %v, exiting.", s)) - - return nil + return server.Serve(listener) } func (ctx *Skinsystem) CreateHandler() *mux.Router { @@ -274,7 +269,7 @@ func (ctx *Skinsystem) Textures(response http.ResponseWriter, request *http.Requ responseData, _ := json.Marshal(textures) response.Header().Set("Content-Type", "application/json") - response.Write(responseData) + _, _ = response.Write(responseData) } func (ctx *Skinsystem) SignedTextures(response http.ResponseWriter, request *http.Request) { @@ -315,7 +310,7 @@ func (ctx *Skinsystem) SignedTextures(response http.ResponseWriter, request *htt responseJson, _ := json.Marshal(responseData) response.Header().Set("Content-Type", "application/json") - response.Write(responseJson) + _, _ = response.Write(responseJson) } func (ctx *Skinsystem) PostSkin(resp http.ResponseWriter, req *http.Request) { diff --git a/http/uuids_worker.go b/http/uuids_worker.go index 4765343..f983a5e 100644 --- a/http/uuids_worker.go +++ b/http/uuids_worker.go @@ -40,13 +40,7 @@ func (ctx *UUIDsWorker) Run() error { Handler: ctx.CreateHandler(), } - // noinspection GoUnhandledErrorResult - go server.Serve(listener) - - s := waitForSignal() - ctx.Logger.Info(fmt.Sprintf("Got signal: %v, exiting.", s)) - - return nil + return server.Serve(listener) } func (ctx *UUIDsWorker) CreateHandler() http.Handler { From a8e4f7ae56103d5f198247183adcfdb2534612c5 Mon Sep 17 00:00:00 2001 From: ErickSkrauch Date: Sun, 5 Jan 2020 23:25:17 +0300 Subject: [PATCH 10/12] Ugly and dirty solution to sync batch_uuids_provider_test --- mojangtextures/batch_uuids_provider_test.go | 32 ++++++++++----------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/mojangtextures/batch_uuids_provider_test.go b/mojangtextures/batch_uuids_provider_test.go index 24bfc82..f3f9f02 100644 --- a/mojangtextures/batch_uuids_provider_test.go +++ b/mojangtextures/batch_uuids_provider_test.go @@ -4,9 +4,7 @@ import ( "crypto/rand" "encoding/base64" "strings" - "sync" "testing" - "time" testify "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -117,15 +115,17 @@ func (suite *batchUuidsProviderTestSuite) SetupTest() { suite.iterateChan <- false } - var lock sync.Mutex suite.GetUuidAsync = func(username string) chan *batchUuidsProviderGetUuidResult { - lock.Lock() - defer lock.Unlock() + s := make(chan bool) + // This dirty hack ensures, that the username will be queued before we return control to the caller. + // It's needed to keep expected calls order and prevent cases when iteration happens before all usernames + // will be queued. + suite.Logger.On("IncCounter", "mojang_textures.usernames.queued", int64(1)).Once().Run(func(args mock.Arguments) { + s <- true + }) c := make(chan *batchUuidsProviderGetUuidResult) - s := make(chan int) go func() { - s <- 0 profile, err := suite.Provider.GetUuid(username) c <- &batchUuidsProviderGetUuidResult{ Result: profile, @@ -144,7 +144,6 @@ func (suite *batchUuidsProviderTestSuite) SetupTest() { func (suite *batchUuidsProviderTestSuite) TearDownTest() { suite.done() - time.Sleep(10 * time.Millisecond) // Add delay to let finish all goroutines before assert mocks calls suite.MojangApi.AssertExpectations(suite.T()) suite.Logger.AssertExpectations(suite.T()) } @@ -156,7 +155,6 @@ func TestBatchUuidsProvider(t *testing.T) { func (suite *batchUuidsProviderTestSuite) TestGetUuidForOneUsername() { expectedResult := &mojang.ProfileInfo{Id: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Name: "username"} - suite.Logger.On("IncCounter", "mojang_textures.usernames.queued", int64(1)).Once() suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(1)).Once() suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(0)).Once() suite.Logger.On("RecordTimer", "mojang_textures.usernames.round_time", mock.Anything).Once() @@ -176,7 +174,6 @@ func (suite *batchUuidsProviderTestSuite) TestGetUuidForTwoUsernames() { expectedResult1 := &mojang.ProfileInfo{Id: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Name: "username1"} expectedResult2 := &mojang.ProfileInfo{Id: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", Name: "username2"} - suite.Logger.On("IncCounter", "mojang_textures.usernames.queued", int64(1)).Twice() suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(2)).Once() suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(0)).Once() suite.Logger.On("RecordTimer", "mojang_textures.usernames.round_time", mock.Anything).Once() @@ -206,15 +203,18 @@ func (suite *batchUuidsProviderTestSuite) TestGetUuidForMoreThan10Usernames() { usernames[i] = randStr(8) } - suite.Logger.On("IncCounter", "mojang_textures.usernames.queued", int64(1)).Times(12) suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(10)).Once() suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(2)).Once() suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(2)).Once() suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(0)).Once() suite.Logger.On("RecordTimer", "mojang_textures.usernames.round_time", mock.Anything).Twice() - suite.MojangApi.On("UsernamesToUuids", usernames[0:10]).Once().Return([]*mojang.ProfileInfo{}, nil) - suite.MojangApi.On("UsernamesToUuids", usernames[10:12]).Once().Return([]*mojang.ProfileInfo{}, nil) + suite.MojangApi.On("UsernamesToUuids", mock.MatchedBy(func(usernames []string) bool { + return len(usernames) == 10 + })).Once().Return([]*mojang.ProfileInfo{}, nil) + suite.MojangApi.On("UsernamesToUuids", mock.MatchedBy(func(usernames []string) bool { + return len(usernames) == 2 + })).Once().Return([]*mojang.ProfileInfo{}, nil) channels := make([]chan *batchUuidsProviderGetUuidResult, len(usernames)) for i, username := range usernames { @@ -230,7 +230,6 @@ func (suite *batchUuidsProviderTestSuite) TestGetUuidForMoreThan10Usernames() { } func (suite *batchUuidsProviderTestSuite) TestDoNothingWhenNoTasks() { - suite.Logger.On("IncCounter", "mojang_textures.usernames.queued", int64(1)).Once() suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(1)).Once() suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(0)).Twice() suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(0)).Times(3) @@ -238,7 +237,7 @@ func (suite *batchUuidsProviderTestSuite) TestDoNothingWhenNoTasks() { suite.MojangApi.On("UsernamesToUuids", []string{"username"}).Once().Return([]*mojang.ProfileInfo{}, nil) - // Perform first iteration and await it finish + // Perform first iteration and await it finishes resultChan := suite.GetUuidAsync("username") suite.Iterate() @@ -247,7 +246,7 @@ func (suite *batchUuidsProviderTestSuite) TestDoNothingWhenNoTasks() { suite.Assert().Nil(result.Result) suite.Assert().Nil(result.Error) - // Let it to perform a few more iterations to ensure, that there is no calls to external APIs + // Let it to perform a few more iterations to ensure, that there are no calls to external APIs suite.Iterate() suite.Iterate() } @@ -255,7 +254,6 @@ func (suite *batchUuidsProviderTestSuite) TestDoNothingWhenNoTasks() { func (suite *batchUuidsProviderTestSuite) TestGetUuidForTwoUsernamesWithAnError() { expectedError := &mojang.TooManyRequestsError{} - suite.Logger.On("IncCounter", "mojang_textures.usernames.queued", int64(1)).Twice() suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(2)).Once() suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(0)).Once() suite.Logger.On("RecordTimer", "mojang_textures.usernames.round_time", mock.Anything).Once() From bc1427dd1f84284d74b830a726400884c05017f0 Mon Sep 17 00:00:00 2001 From: ErickSkrauch Date: Sun, 5 Jan 2020 23:35:10 +0300 Subject: [PATCH 11/12] Exclude deployment for pull requests, move docker and sudo requirements to deploy step --- .travis.yml | 49 ++++++++++++++++++++++++------------------------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/.travis.yml b/.travis.yml index 339b236..ffee940 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,16 +1,11 @@ -sudo: required - language: go go: - - 1.13 - -services: - - docker + - "1.13" stages: - test - name: deploy - if: branch = master OR tag IS present + if: env(TRAVIS_PULL_REQUEST) = false AND (branch = master OR tag IS present) install: - go get -u github.com/golang/dep/cmd/dep @@ -20,23 +15,27 @@ jobs: include: - stage: test script: - - go test -v -race -coverprofile=coverage.txt -covermode=atomic ./... - - bash <(curl -s https://codecov.io/bash) + - go test -v -race -coverprofile=coverage.txt -covermode=atomic ./... + - bash <(curl -s https://codecov.io/bash) + - stage: deploy + sudo: required + services: + - docker script: - - docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD" - - export DOCKER_TAG="${TRAVIS_TAG:-dev}" - - export APP_VERSION="${TRAVIS_TAG:-dev-${TRAVIS_COMMIT:0:7}}" - - > - env CGO_ENABLED=0 GOOS=linux GOARCH=amd64 - go build - -o release/chrly - -ldflags '-extldflags "-static" -X github.com/elyby/chrly/version.version=$APP_VERSION -X github.com/elyby/chrly/version.commit=$TRAVIS_COMMIT' - main.go - - docker build -t elyby/chrly:$DOCKER_TAG . - - docker push elyby/chrly:$DOCKER_TAG - - | - if [ ! -z ${TRAVIS_TAG+x} ] && [[ "$TRAVIS_TAG" != *"-"* ]]; then - docker tag elyby/chrly:$DOCKER_TAG elyby/chrly:latest - docker push elyby/chrly:latest - fi + - docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD" + - export DOCKER_TAG="${TRAVIS_TAG:-dev}" + - export APP_VERSION="${TRAVIS_TAG:-dev-${TRAVIS_COMMIT:0:7}}" + - > + env CGO_ENABLED=0 GOOS=linux GOARCH=amd64 + go build + -o release/chrly + -ldflags '-extldflags "-static" -X github.com/elyby/chrly/version.version=$APP_VERSION -X github.com/elyby/chrly/version.commit=$TRAVIS_COMMIT' + main.go + - docker build -t elyby/chrly:$DOCKER_TAG . + - docker push elyby/chrly:$DOCKER_TAG + - | + if [ ! -z ${TRAVIS_TAG+x} ] && [[ "$TRAVIS_TAG" != *"-"* ]]; then + docker tag elyby/chrly:$DOCKER_TAG elyby/chrly:latest + docker push elyby/chrly:latest + fi From 92473d15d6ea2945919e615f78b27f6c552ea6cd Mon Sep 17 00:00:00 2001 From: ErickSkrauch Date: Sun, 5 Jan 2020 23:45:11 +0300 Subject: [PATCH 12/12] Replace simple Mutex with RWMutex for in memory textures storage --- mojangtextures/in_memory_textures_storage.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mojangtextures/in_memory_textures_storage.go b/mojangtextures/in_memory_textures_storage.go index 37671df..e5b72b2 100644 --- a/mojangtextures/in_memory_textures_storage.go +++ b/mojangtextures/in_memory_textures_storage.go @@ -20,7 +20,7 @@ type InMemoryTexturesStorage struct { GCPeriod time.Duration Duration time.Duration - lock sync.Mutex + lock sync.RWMutex data map[string]*inMemoryItem working *abool.AtomicBool } @@ -60,8 +60,8 @@ func (s *InMemoryTexturesStorage) Stop() { } func (s *InMemoryTexturesStorage) GetTextures(uuid string) (*mojang.SignedTexturesResponse, error) { - s.lock.Lock() - defer s.lock.Unlock() + s.lock.RLock() + defer s.lock.RUnlock() item, exists := s.data[uuid] validRange := s.getMinimalNotExpiredTimestamp()