ElyBy-Mirror/chrly
1
0
Fork 0
mirror of https://github.com/elyby/chrly.git synced 2025-05-31 14:11:51 +05:30
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into ely

Browse Source
This commit is contained in:
ErickSkrauch
2019-11-08 02:13:05 +03:00
parent e573e6146d 6fd88e077e
commit ab847589ba
8 changed files with 70 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.travis.yml
View File

@@ -2,7 +2,7 @@ sudo: required
language: go
go:
- 1.12
- 1.13
services:
- docker

12
CHANGELOG.md
View File

@@ -6,6 +6,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased] - xxxx-xx-xx
## [4.3.0] - 2019-11-08
### Added
- 403 Forbidden errors from the Mojang's API are now logged
- `QUEUE_LOOP_DELAY` configuration param to adjust Mojang's textures queue performance
### Changed
- Mojang's textures queue loop is now has an iteration delay of 2.5 seconds (was 1)
- Bumped Go version to 1.13.
## [4.2.3] - 2019-10-03
### Changed
- Mojang's textures queue batch size [reduced to 10](https://wiki.vg/index.php?title=Mojang_API&type=revision&diff=14964&oldid=14954).
@@ -66,7 +75,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
from the textures link instead.
- `hash` field from `POST /api/skins` endpoint.
[Unreleased]: https://github.com/elyby/chrly/compare/4.2.3...HEAD
[Unreleased]: https://github.com/elyby/chrly/compare/4.3.0...HEAD
[4.3.0]: https://github.com/elyby/chrly/compare/4.2.3...4.3.0
[4.2.3]: https://github.com/elyby/chrly/compare/4.2.2...4.2.3
[4.2.2]: https://github.com/elyby/chrly/compare/4.2.1...4.2.2
[4.2.1]: https://github.com/elyby/chrly/compare/4.2.0...4.2.1

11
README.md
View File

@@ -57,11 +57,12 @@ docker-compose up -d app
**Variables to adjust:**
| ENV | Description | Example |
|--------------------|------------------------------------------------------------------------------------|-------------------------------------------|
| STORAGE_REDIS_POOL | By default, Chrly creates pool with 10 connection, but you may want to increase it | `20` |
| STATSD_ADDR | StatsD can be used to collect metrics | `localhost:8125` |
| SENTRY_DSN | Sentry can be used to collect app errors | `https://public:private@your.sentry.io/1` |
| ENV | Description | Example |
|--------------------|-------------------------------------------------------------------------------------------------|-------------------------------------------|
| STORAGE_REDIS_POOL | By default, Chrly creates pool with 10 connection, but you may want to increase it | `20` |
| STATSD_ADDR | StatsD can be used to collect metrics | `localhost:8125` |
| SENTRY_DSN | Sentry can be used to collect app errors | `https://public:private@your.sentry.io/1` |
| QUEUE_LOOP_DELAY | Parameter is sets the delay before each iteration of the Mojang's textures queue (milliseconds) | `3200` |
If something goes wrong, you can always access logs by executing `docker-compose logs -f app`.

11
api/mojang/mojang.go
View File

@@ -125,6 +125,8 @@ func validateResponse(response *http.Response) error {
_ = json.Unmarshal(body, &decodedError)
return &BadRequestError{ErrorType: decodedError.Error, Message: decodedError.Message}
case response.StatusCode == 403:
return &ForbiddenError{}
case response.StatusCode == 429:
return &TooManyRequestsError{}
case response.StatusCode >= 500:
@@ -166,6 +168,15 @@ func (*BadRequestError) IsMojangError() bool {
return true
}
// When Mojang decides you're such a bad guy, this error appears (even if the request has no authorization)
type ForbiddenError struct {
ResponseError
}
func (*ForbiddenError) Error() string {
return "Forbidden"
}
// When you exceed the set limit of requests, this error will be returned
type TooManyRequestsError struct {
ResponseError

21
api/mojang/mojang_test.go
View File

@@ -102,6 +102,27 @@ func TestUsernamesToUuids(t *testing.T) {
assert.Implements((*ResponseError)(nil), err)
})
t.Run("handle forbidden response", func(t *testing.T) {
assert := testify.New(t)
defer gock.Off()
gock.New("https://api.mojang.com").
Post("/profiles/minecraft").
Reply(403).
BodyString("just because")
client := &http.Client{}
gock.InterceptClient(client)
HttpClient = client
result, err := UsernamesToUuids([]string{"Thinkofdeath", "maksimkurb"})
assert.Nil(result)
assert.IsType(&ForbiddenError{}, err)
assert.EqualError(err, "Forbidden")
assert.Implements((*ResponseError)(nil), err)
})
t.Run("handle too many requests response", func(t *testing.T) {
assert := testify.New(t)

20
api/mojang/queue/queue.go
View File

@@ -14,9 +14,10 @@ import (
"github.com/elyby/chrly/api/mojang"
)
var UuidsQueueIterationDelay = 2*time.Second + 500*time.Millisecond
var usernamesToUuids = mojang.UsernamesToUuids
var uuidToTextures = mojang.UuidToTextures
var uuidsQueueIterationDelay = time.Second
var forever = func() bool {
return true
}
@@ -97,13 +98,13 @@ func (ctx *JobsQueue) GetTexturesForUsername(username string) chan *mojang.Signe
func (ctx *JobsQueue) startQueue() {
go func() {
time.Sleep(uuidsQueueIterationDelay)
time.Sleep(UuidsQueueIterationDelay)
for forever() {
start := time.Now()
ctx.queueRound()
elapsed := time.Since(start)
ctx.Logger.RecordTimer("mojang_textures.usernames.round_time", elapsed)
time.Sleep(uuidsQueueIterationDelay)
time.Sleep(UuidsQueueIterationDelay)
}
}()
}
@@ -182,13 +183,18 @@ func (ctx *JobsQueue) handleResponseError(err error, threadName string) {
switch err.(type) {
case mojang.ResponseError:
if _, ok := err.(*mojang.TooManyRequestsError); ok {
ctx.Logger.Warning(":name: Got 429 Too Many Requests :err", wd.NameParam(threadName), wd.ErrParam(err))
if _, ok := err.(*mojang.BadRequestError); ok {
ctx.Logger.Warning(":name: Got 400 Bad Request :err", wd.NameParam(threadName), wd.ErrParam(err))
return
}
if _, ok := err.(*mojang.BadRequestError); ok {
ctx.Logger.Warning(":name: Got 400 Bad Request :err", wd.NameParam(threadName), wd.ErrParam(err))
if _, ok := err.(*mojang.ForbiddenError); ok {
ctx.Logger.Warning(":name: Got 403 Forbidden :err", wd.NameParam(threadName), wd.ErrParam(err))
return
}
if _, ok := err.(*mojang.TooManyRequestsError); ok {
ctx.Logger.Warning(":name: Got 429 Too Many Requests :err", wd.NameParam(threadName), wd.ErrParam(err))
return
}

5
api/mojang/queue/queue_test.go
View File

@@ -85,7 +85,7 @@ type queueTestSuite struct {
}
func (suite *queueTestSuite) SetupSuite() {
uuidsQueueIterationDelay = 0
UuidsQueueIterationDelay = 0
}
func (suite *queueTestSuite) SetupTest() {
@@ -403,6 +403,7 @@ func (*timeoutError) Temporary() bool { return false }
var expectedErrors = []error{
&mojang.BadRequestError{},
&mojang.ForbiddenError{},
&mojang.TooManyRequestsError{},
&mojang.ServerError{},
&timeoutError{},
@@ -418,6 +419,7 @@ func (suite *queueTestSuite) TestShouldNotLogErrorWhenExpectedErrorReturnedFromU
suite.Logger.On("RecordTimer", mock.Anything, mock.Anything)
suite.Logger.On("Debug", ":name: Got response error :err", mock.Anything, mock.Anything).Times(len(expectedErrors))
suite.Logger.On("Warning", ":name: Got 400 Bad Request :err", mock.Anything, mock.Anything).Once()
suite.Logger.On("Warning", ":name: Got 403 Forbidden :err", mock.Anything, mock.Anything).Once()
suite.Logger.On("Warning", ":name: Got 429 Too Many Requests :err", mock.Anything, mock.Anything).Once()
suite.Storage.On("GetUuid", "maksimkurb").Return("", &ValueNotFound{})
@@ -455,6 +457,7 @@ func (suite *queueTestSuite) TestShouldNotLogErrorWhenExpectedErrorReturnedFromU
suite.Logger.On("RecordTimer", mock.Anything, mock.Anything)
suite.Logger.On("Debug", ":name: Got response error :err", mock.Anything, mock.Anything).Times(len(expectedErrors))
suite.Logger.On("Warning", ":name: Got 400 Bad Request :err", mock.Anything, mock.Anything).Once()
suite.Logger.On("Warning", ":name: Got 403 Forbidden :err", mock.Anything, mock.Anything).Once()
suite.Logger.On("Warning", ":name: Got 429 Too Many Requests :err", mock.Anything, mock.Anything).Once()
suite.Storage.On("GetUuid", "maksimkurb").Return("", &ValueNotFound{})

3
cmd/serve.go
View File

@@ -3,6 +3,7 @@ package cmd
import (
"fmt"
"log"
"time"
"github.com/spf13/cobra"
"github.com/spf13/viper"
@@ -51,6 +52,7 @@ var serveCmd = &cobra.Command{
return
}
queue.UuidsQueueIterationDelay = time.Duration(viper.GetInt("queue.loop_delay")) * time.Millisecond
texturesStorage := queue.CreateInMemoryTexturesStorage()
texturesStorage.Start()
mojangTexturesQueue := &queue.JobsQueue{
@@ -86,4 +88,5 @@ func init() {
viper.SetDefault("storage.redis.poll", 10)
viper.SetDefault("storage.filesystem.basePath", "data")
viper.SetDefault("storage.filesystem.capesDirName", "capes")
viper.SetDefault("queue.loop_delay", 2_500)
}