Prepare 4.3.0 release

.travis.yml

@@ -2,7 +2,7 @@ sudo: required
 
 language: go
 go:
-  - 1.12
+  - 1.13
 
 services:
   - docker

CHANGELOG.md

@@ -4,7 +4,25 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [Unreleased]
+## [Unreleased] - xxxx-xx-xx
+
+## [4.3.0] - 2019-11-08
+### Added
+- 403 Forbidden errors from the Mojang's API are now logged
+- `QUEUE_LOOP_DELAY` configuration param to adjust Mojang's textures queue performance
+
+### Changed
+- Mojang's textures queue loop is now has an iteration delay of 2.5 seconds (was 1) 
+- Bumped Go version to 1.13.
+
+## [4.2.3] - 2019-10-03
+### Changed
+- Mojang's textures queue batch size [reduced to 10](https://wiki.vg/index.php?title=Mojang_API&type=revision&diff=14964&oldid=14954).
+- 400 BadRequest errors from the Mojang's API are now logged.
+
+## [4.2.2] - 2019-06-19
+### Fixed
+- GC for in-memory textures cache has not been initialized.
 
 ## [4.2.1] - 2019-05-06
 ### Changed
@@ -57,6 +75,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   from the textures link instead.
 - `hash` field from `POST /api/skins` endpoint.
 
-[Unreleased]: https://github.com/elyby/chrly/compare/4.2.1...HEAD
+[Unreleased]: https://github.com/elyby/chrly/compare/4.3.0...HEAD
+[4.3.0]: https://github.com/elyby/chrly/compare/4.2.3...4.3.0
+[4.2.3]: https://github.com/elyby/chrly/compare/4.2.2...4.2.3
+[4.2.2]: https://github.com/elyby/chrly/compare/4.2.1...4.2.2
 [4.2.1]: https://github.com/elyby/chrly/compare/4.2.0...4.2.1
 [4.2.0]: https://github.com/elyby/chrly/compare/4.1.1...4.2.0

README.md

@@ -2,6 +2,7 @@
 
 [![Written in Go][ico-lang]][link-go]
 [![Build Status][ico-build]][link-build]
+[![Coverage][ico-coverage]][link-coverage]
 [![Keep a Changelog][ico-changelog]](CHANGELOG.md)
 [![Software License][ico-license]](LICENSE)
 
@@ -56,11 +57,12 @@ docker-compose up -d app
 
 **Variables to adjust:**
 
-| ENV                | Description                                                                        | Example                                   |
-|--------------------|------------------------------------------------------------------------------------|-------------------------------------------|
-| STORAGE_REDIS_POOL | By default, Chrly creates pool with 10 connection, but you may want to increase it | `20`                                      |
-| STATSD_ADDR        | StatsD can be used to collect metrics                                              | `localhost:8125`                          |
-| SENTRY_DSN         | Sentry can be used to collect app errors                                           | `https://public:private@your.sentry.io/1` |
+| ENV                | Description                                                                                     | Example                                   |
+|--------------------|-------------------------------------------------------------------------------------------------|-------------------------------------------|
+| STORAGE_REDIS_POOL | By default, Chrly creates pool with 10 connection, but you may want to increase it              | `20`                                      |
+| STATSD_ADDR        | StatsD can be used to collect metrics                                                           | `localhost:8125`                          |
+| SENTRY_DSN         | Sentry can be used to collect app errors                                                        | `https://public:private@your.sentry.io/1` |
+| QUEUE_LOOP_DELAY   | Parameter is sets the delay before each iteration of the Mojang's textures queue (milliseconds) | `3200`                                    |
 
 If something goes wrong, you can always access logs by executing `docker-compose logs -f app`.
 
@@ -260,8 +262,10 @@ To run tests execute `go test ./...`. If your Go version is older than 1.9, then
 
 [ico-lang]: https://img.shields.io/badge/lang-go%201.12-blue.svg?style=flat-square
 [ico-build]: https://img.shields.io/travis/elyby/chrly.svg?style=flat-square
+[ico-coverage]: https://img.shields.io/codecov/c/github/elyby/chrly.svg?style=flat-square
 [ico-changelog]: https://img.shields.io/badge/keep%20a-changelog-orange.svg?style=flat-square
 [ico-license]: https://img.shields.io/github/license/elyby/chrly.svg?style=flat-square
 
 [link-go]: https://golang.org
 [link-build]: https://travis-ci.org/elyby/chrly
+[link-coverage]: https://codecov.io/gh/elyby/chrly

api/mojang/mojang.go

@@ -125,6 +125,8 @@ func validateResponse(response *http.Response) error {
 		_ = json.Unmarshal(body, &decodedError)
 
 		return &BadRequestError{ErrorType: decodedError.Error, Message: decodedError.Message}
+	case response.StatusCode == 403:
+		return &ForbiddenError{}
 	case response.StatusCode == 429:
 		return &TooManyRequestsError{}
 	case response.StatusCode >= 500:
@@ -166,6 +168,15 @@ func (*BadRequestError) IsMojangError() bool {
 	return true
 }
 
+// When Mojang decides you're such a bad guy, this error appears (even if the request has no authorization)
+type ForbiddenError struct {
+	ResponseError
+}
+
+func (*ForbiddenError) Error() string {
+	return "Forbidden"
+}
+
 // When you exceed the set limit of requests, this error will be returned
 type TooManyRequestsError struct {
 	ResponseError

api/mojang/mojang_test.go

@@ -102,6 +102,27 @@ func TestUsernamesToUuids(t *testing.T) {
 		assert.Implements((*ResponseError)(nil), err)
 	})
 
+	t.Run("handle forbidden response", func(t *testing.T) {
+		assert := testify.New(t)
+
+		defer gock.Off()
+		gock.New("https://api.mojang.com").
+			Post("/profiles/minecraft").
+			Reply(403).
+			BodyString("just because")
+
+		client := &http.Client{}
+		gock.InterceptClient(client)
+
+		HttpClient = client
+
+		result, err := UsernamesToUuids([]string{"Thinkofdeath", "maksimkurb"})
+		assert.Nil(result)
+		assert.IsType(&ForbiddenError{}, err)
+		assert.EqualError(err, "Forbidden")
+		assert.Implements((*ResponseError)(nil), err)
+	})
+
 	t.Run("handle too many requests response", func(t *testing.T) {
 		assert := testify.New(t)
 

api/mojang/queue/in_memory_textures_storage.go

@@ -25,9 +25,11 @@ type inMemoryTexturesStorage struct {
 }
 
 func CreateInMemoryTexturesStorage() *inMemoryTexturesStorage {
-	return &inMemoryTexturesStorage{
+	storage := &inMemoryTexturesStorage{
 		data: make(map[string]*inMemoryItem),
 	}
+
+	return storage
 }
 
 func (s *inMemoryTexturesStorage) Start() {

api/mojang/queue/queue.go

@@ -14,9 +14,10 @@ import (
 	"github.com/elyby/chrly/api/mojang"
 )
 
+var UuidsQueueIterationDelay = 2*time.Second + 500*time.Millisecond
+
 var usernamesToUuids = mojang.UsernamesToUuids
 var uuidToTextures = mojang.UuidToTextures
-var uuidsQueueIterationDelay = time.Second
 var forever = func() bool {
 	return true
 }
@@ -97,13 +98,13 @@ func (ctx *JobsQueue) GetTexturesForUsername(username string) chan *mojang.Signe
 
 func (ctx *JobsQueue) startQueue() {
 	go func() {
-		time.Sleep(uuidsQueueIterationDelay)
+		time.Sleep(UuidsQueueIterationDelay)
 		for forever() {
 			start := time.Now()
 			ctx.queueRound()
 			elapsed := time.Since(start)
 			ctx.Logger.RecordTimer("mojang_textures.usernames.round_time", elapsed)
-			time.Sleep(uuidsQueueIterationDelay)
+			time.Sleep(UuidsQueueIterationDelay)
 		}
 	}()
 }
@@ -114,7 +115,7 @@ func (ctx *JobsQueue) queueRound() {
 	}
 
 	queueSize := ctx.queue.Size()
-	jobs := ctx.queue.Dequeue(100)
+	jobs := ctx.queue.Dequeue(10)
 	ctx.Logger.UpdateGauge("mojang_textures.usernames.iteration_size", int64(len(jobs)))
 	ctx.Logger.UpdateGauge("mojang_textures.usernames.queue_size", int64(queueSize-len(jobs)))
 	var usernames []string
@@ -182,8 +183,19 @@ func (ctx *JobsQueue) handleResponseError(err error, threadName string) {
 
 	switch err.(type) {
 	case mojang.ResponseError:
+		if _, ok := err.(*mojang.BadRequestError); ok {
+			ctx.Logger.Warning(":name: Got 400 Bad Request :err", wd.NameParam(threadName), wd.ErrParam(err))
+			return
+		}
+
+		if _, ok := err.(*mojang.ForbiddenError); ok {
+			ctx.Logger.Warning(":name: Got 403 Forbidden :err", wd.NameParam(threadName), wd.ErrParam(err))
+			return
+		}
+
 		if _, ok := err.(*mojang.TooManyRequestsError); ok {
 			ctx.Logger.Warning(":name: Got 429 Too Many Requests :err", wd.NameParam(threadName), wd.ErrParam(err))
+			return
 		}
 
 		return

api/mojang/queue/queue_test.go

@@ -85,7 +85,7 @@ type queueTestSuite struct {
 }
 
 func (suite *queueTestSuite) SetupSuite() {
-	uuidsQueueIterationDelay = 0
+	UuidsQueueIterationDelay = 0
 }
 
 func (suite *queueTestSuite) SetupTest() {
@@ -258,30 +258,30 @@ func (suite *queueTestSuite) TestReceiveTexturesForUsernameWithCachedUnknownUuid
 	suite.Assert().Nil(<-resultChan)
 }
 
-func (suite *queueTestSuite) TestReceiveTexturesForMoreThan100Usernames() {
-	usernames := make([]string, 120)
-	for i := 0; i < 120; i++ {
+func (suite *queueTestSuite) TestReceiveTexturesForMoreThan10Usernames() {
+	usernames := make([]string, 12)
+	for i := 0; i < cap(usernames); i++ {
 		usernames[i] = randStr(8)
 	}
 
-	suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Times(120)
-	suite.Logger.On("IncCounter", "mojang_textures.usernames.queued", int64(1)).Times(120)
-	suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(100)).Once()
-	suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(20)).Once()
-	suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(20)).Once()
+	suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Times(12)
+	suite.Logger.On("IncCounter", "mojang_textures.usernames.queued", int64(1)).Times(12)
+	suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(10)).Once()
+	suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(2)).Once()
+	suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(2)).Once()
 	suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(0)).Once()
 	suite.Logger.On("RecordTimer", "mojang_textures.usernames.round_time", mock.Anything).Twice()
-	suite.Logger.On("IncCounter", "mojang_textures.usernames.uuid_miss", int64(1)).Times(120)
-	suite.Logger.On("RecordTimer", "mojang_textures.result_time", mock.Anything).Times(120)
+	suite.Logger.On("IncCounter", "mojang_textures.usernames.uuid_miss", int64(1)).Times(12)
+	suite.Logger.On("RecordTimer", "mojang_textures.result_time", mock.Anything).Times(12)
 
-	suite.Storage.On("GetUuid", mock.Anything).Times(120).Return("", &ValueNotFound{})
-	suite.Storage.On("StoreUuid", mock.Anything, "").Times(120).Return(nil) // should be called with "" if username is not compared to uuid
+	suite.Storage.On("GetUuid", mock.Anything).Times(12).Return("", &ValueNotFound{})
+	suite.Storage.On("StoreUuid", mock.Anything, "").Times(12).Return(nil) // should be called with "" if username is not compared to uuid
 	// Storage.GetTextures and Storage.SetTextures shouldn't be called
 
-	suite.MojangApi.On("UsernamesToUuids", usernames[0:100]).Once().Return([]*mojang.ProfileInfo{}, nil)
-	suite.MojangApi.On("UsernamesToUuids", usernames[100:120]).Once().Return([]*mojang.ProfileInfo{}, nil)
+	suite.MojangApi.On("UsernamesToUuids", usernames[0:10]).Once().Return([]*mojang.ProfileInfo{}, nil)
+	suite.MojangApi.On("UsernamesToUuids", usernames[10:12]).Once().Return([]*mojang.ProfileInfo{}, nil)
 
-	channels := make([]chan *mojang.SignedTexturesResponse, 120)
+	channels := make([]chan *mojang.SignedTexturesResponse, 12)
 	for i, username := range usernames {
 		channels[i] = suite.Queue.GetTexturesForUsername(username)
 	}
@@ -403,6 +403,7 @@ func (*timeoutError) Temporary() bool { return false }
 
 var expectedErrors = []error{
 	&mojang.BadRequestError{},
+	&mojang.ForbiddenError{},
 	&mojang.TooManyRequestsError{},
 	&mojang.ServerError{},
 	&timeoutError{},
@@ -417,6 +418,8 @@ func (suite *queueTestSuite) TestShouldNotLogErrorWhenExpectedErrorReturnedFromU
 	suite.Logger.On("UpdateGauge", mock.Anything, mock.Anything)
 	suite.Logger.On("RecordTimer", mock.Anything, mock.Anything)
 	suite.Logger.On("Debug", ":name: Got response error :err", mock.Anything, mock.Anything).Times(len(expectedErrors))
+	suite.Logger.On("Warning", ":name: Got 400 Bad Request :err", mock.Anything, mock.Anything).Once()
+	suite.Logger.On("Warning", ":name: Got 403 Forbidden :err", mock.Anything, mock.Anything).Once()
 	suite.Logger.On("Warning", ":name: Got 429 Too Many Requests :err", mock.Anything, mock.Anything).Once()
 
 	suite.Storage.On("GetUuid", "maksimkurb").Return("", &ValueNotFound{})
@@ -453,6 +456,8 @@ func (suite *queueTestSuite) TestShouldNotLogErrorWhenExpectedErrorReturnedFromU
 	suite.Logger.On("UpdateGauge", mock.Anything, mock.Anything)
 	suite.Logger.On("RecordTimer", mock.Anything, mock.Anything)
 	suite.Logger.On("Debug", ":name: Got response error :err", mock.Anything, mock.Anything).Times(len(expectedErrors))
+	suite.Logger.On("Warning", ":name: Got 400 Bad Request :err", mock.Anything, mock.Anything).Once()
+	suite.Logger.On("Warning", ":name: Got 403 Forbidden :err", mock.Anything, mock.Anything).Once()
 	suite.Logger.On("Warning", ":name: Got 429 Too Many Requests :err", mock.Anything, mock.Anything).Once()
 
 	suite.Storage.On("GetUuid", "maksimkurb").Return("", &ValueNotFound{})

cmd/serve.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"fmt"
 	"log"
+	"time"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
@@ -51,11 +52,14 @@ var serveCmd = &cobra.Command{
 			return
 		}
 
+		queue.UuidsQueueIterationDelay = time.Duration(viper.GetInt("queue.loop_delay")) * time.Millisecond
+		texturesStorage := queue.CreateInMemoryTexturesStorage()
+		texturesStorage.Start()
 		mojangTexturesQueue := &queue.JobsQueue{
 			Logger: logger,
 			Storage: &queue.SplittedStorage{
 				UuidsStorage:    mojangUuidsRepository,
-				TexturesStorage: queue.CreateInMemoryTexturesStorage(),
+				TexturesStorage: texturesStorage,
 			},
 		}
 		logger.Info("Mojang's textures queue is successfully initialized")
@@ -84,4 +88,5 @@ func init() {
 	viper.SetDefault("storage.redis.poll", 10)
 	viper.SetDefault("storage.filesystem.basePath", "data")
 	viper.SetDefault("storage.filesystem.capesDirName", "capes")
+	viper.SetDefault("queue.loop_delay", 2_500)
 }