ElyBy-Mirror/chrly
1
0
Fork 0
mirror of https://github.com/elyby/chrly.git synced 2025-01-10 13:57:56 +05:30
Code Issues Packages Projects Releases Wiki Activity

Remove worker mode

Browse Source
This commit is contained in:
ErickSkrauch 2023-12-15 03:42:38 +01:00
parent 20ba78953b
commit e568d4cf91
No known key found for this signature in database
GPG Key ID: 669339FCBB30EE0E
11 changed files with 2 additions and 569 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- StatsD metrics: - StatsD metrics:
- Gauges: - Gauges:
- `ely.skinsystem.{hostname}.app.redis.pool.available` - `ely.skinsystem.{hostname}.app.redis.pool.available`
- Worker mode. Use URL spoofing to load balance outgoing requests.
## [4.6.0] - 2021-03-04 ## [4.6.0] - 2021-03-04
### Added ### Added

41
README.md
View File

@ -134,22 +134,6 @@ docker-compose up -d app
</td> </td>
<td><code>true</code></td> <td><code>true</code></td>
</tr> </tr>
<tr>
<td id="remote-mojang-uuids-provider">MOJANG_TEXTURES_UUIDS_PROVIDER_DRIVER</td>
<td>
Specifies the preferred provider of the Mojang's UUIDs. Takes <code>remote</code> value.
In any other case, the local queue will be used.
</td>
<td><code>remote</code></td>
</tr>
<tr>
<td>MOJANG_TEXTURES_UUIDS_PROVIDER_URL</td>
<td>
When the UUIDs driver set to <code>remote</code>, sets the remote URL.
The trailing slash won't cause any problems.
</td>
<td><code>http://remote-provider.com/api/worker/mojang-uuid</code></td>
</tr>
<tr> <tr>
<td>MOJANG_API_BASE_URL</td> <td>MOJANG_API_BASE_URL</td>
<td> <td>
@ -399,31 +383,6 @@ response will be:
} }
``` ```
### Worker mode
The worker mode can be used in cooperation with the [remote server mode](#remote-mojang-uuids-provider)
to exchange Mojang usernames to UUIDs. This mode by itself doesn't solve the problem of
[extremely strict limits](https://github.com/elyby/chrly/issues/10) on the number of requests to the Mojang's API.
But with a proxying load balancer (e.g. HAProxy, Nginx, etc.) it's easy to build a cluster of workers,
which will multiply the bandwidth of the exchanging usernames to its UUIDs.
The instructions for setting up a proxy load balancer are outside the context of this documentation,
but you get the idea ;)
#### `GET /api/worker/mojang-uuid/{username}`
Performs [batch usernames exchange to UUIDs](https://github.com/elyby/chrly/issues/1) and returns the result in the
[same format as it returns from the Mojang's API](https://wiki.vg/Mojang_API#Username_-.3E_UUID_at_time):
```json
{
"id": "3e3ee6c35afa48abb61e8cd8c42fc0d9",
"name": "ErickSkrauch"
}
```
> **Note**: the results aren't cached.
### Health check ### Health check
#### `GET /healthcheck` #### `GET /healthcheck`

17
cmd/worker.go
View File

@ -1,17 +0,0 @@
package cmd
import (
"github.com/spf13/cobra"
)
var workerCmd = &cobra.Command{
Use: "worker",
Short: "Starts HTTP handler for the Mojang usernames to UUIDs worker",
Run: func(cmd *cobra.Command, args []string) {
startServer([]string{"worker"})
},
}
func init() {
RootCmd.AddCommand(workerCmd)
}

19
di/handlers.go
View File

@ -11,14 +11,12 @@ import (
"github.com/spf13/viper" "github.com/spf13/viper"
. "github.com/elyby/chrly/http" . "github.com/elyby/chrly/http"
"github.com/elyby/chrly/mojangtextures"
) )
var handlers = di.Options( var handlers = di.Options(
di.Provide(newHandlerFactory, di.As(new(http.Handler))), di.Provide(newHandlerFactory, di.As(new(http.Handler))),
di.Provide(newSkinsystemHandler, di.WithName("skinsystem")), di.Provide(newSkinsystemHandler, di.WithName("skinsystem")),
di.Provide(newApiHandler, di.WithName("api")), di.Provide(newApiHandler, di.WithName("api")),
di.Provide(newUUIDsWorkerHandler, di.WithName("worker")),
) )
func newHandlerFactory( func newHandlerFactory(
@ -48,17 +46,6 @@ func newHandlerFactory(
// See https://github.com/gorilla/mux/issues/416#issuecomment-600079279 // See https://github.com/gorilla/mux/issues/416#issuecomment-600079279
router.NotFoundHandler = requestEventsMiddleware(http.HandlerFunc(NotFoundHandler)) router.NotFoundHandler = requestEventsMiddleware(http.HandlerFunc(NotFoundHandler))
// Enable the worker module before api to allow gorilla.mux to correctly find the target router
// as it uses the first matching and /api overrides the more accurate /api/worker
if hasValue(enabledModules, "worker") {
var workerRouter *mux.Router
if err := container.Resolve(&workerRouter, di.Name("worker")); err != nil {
return nil, err
}
mount(router, "/api/worker", workerRouter)
}
if hasValue(enabledModules, "api") { if hasValue(enabledModules, "api") {
var apiRouter *mux.Router var apiRouter *mux.Router
if err := container.Resolve(&apiRouter, di.Name("api")); err != nil { if err := container.Resolve(&apiRouter, di.Name("api")); err != nil {
@ -127,12 +114,6 @@ func newApiHandler(skinsRepository SkinsRepository) *mux.Router {
}).Handler() }).Handler()
} }
func newUUIDsWorkerHandler(mojangUUIDsProvider *mojangtextures.BatchUuidsProvider) *mux.Router {
return (&UUIDsWorker{
MojangUuidsProvider: mojangUUIDsProvider,
}).Handler()
}
func hasValue(slice []string, needle string) bool { func hasValue(slice []string, needle string) bool {
for _, value := range slice { for _, value := range slice {
if value == needle { if value == needle {

40
di/mojang_textures.go
View File

@ -24,7 +24,6 @@ var mojangTextures = di.Options(
di.Provide(newMojangTexturesBatchUUIDsProviderStrategyFactory), di.Provide(newMojangTexturesBatchUUIDsProviderStrategyFactory),
di.Provide(newMojangTexturesBatchUUIDsProviderDelayedStrategy), di.Provide(newMojangTexturesBatchUUIDsProviderDelayedStrategy),
di.Provide(newMojangTexturesBatchUUIDsProviderFullBusStrategy), di.Provide(newMojangTexturesBatchUUIDsProviderFullBusStrategy),
di.Provide(newMojangTexturesRemoteUUIDsProvider),
di.Provide(newMojangSignedTexturesProvider), di.Provide(newMojangSignedTexturesProvider),
di.Provide(newMojangTexturesStorageFactory), di.Provide(newMojangTexturesStorageFactory),
) )
@ -86,17 +85,8 @@ func newMojangTexturesProvider(
} }
func newMojangTexturesUuidsProviderFactory( func newMojangTexturesUuidsProviderFactory(
config *viper.Viper,
container *di.Container, container *di.Container,
) (mojangtextures.UUIDsProvider, error) { ) (mojangtextures.UUIDsProvider, error) {
preferredUuidsProvider := config.GetString("mojang_textures.uuids_provider.driver")
if preferredUuidsProvider == "remote" {
var provider *mojangtextures.RemoteApiUuidsProvider
err := container.Resolve(&provider)
return provider, err
}
var provider *mojangtextures.BatchUuidsProvider var provider *mojangtextures.BatchUuidsProvider
err := container.Resolve(&provider) err := container.Resolve(&provider)
@ -188,36 +178,6 @@ func newMojangTexturesBatchUUIDsProviderFullBusStrategy(config *viper.Viper) *mo
) )
} }
func newMojangTexturesRemoteUUIDsProvider(
container *di.Container,
config *viper.Viper,
emitter mojangtextures.Emitter,
) (*mojangtextures.RemoteApiUuidsProvider, error) {
remoteUrl, err := url.Parse(config.GetString("mojang_textures.uuids_provider.url"))
if err != nil {
return nil, fmt.Errorf("unable to parse remote url: %w", err)
}
if err := container.Provide(func(emitter es.Subscriber, config *viper.Viper) *namedHealthChecker {
config.SetDefault("healthcheck.mojang_api_textures_provider_cool_down_duration", time.Minute+10*time.Second)
return &namedHealthChecker{
Name: "mojang-api-textures-provider-response-checker",
Checker: es.MojangApiTexturesProviderResponseChecker(
emitter,
config.GetDuration("healthcheck.mojang_api_textures_provider_cool_down_duration"),
),
}
}); err != nil {
return nil, err
}
return &mojangtextures.RemoteApiUuidsProvider{
Emitter: emitter,
Url: *remoteUrl,
}, nil
}
func newMojangSignedTexturesProvider(emitter mojangtextures.Emitter) mojangtextures.TexturesProvider { func newMojangSignedTexturesProvider(emitter mojangtextures.Emitter) mojangtextures.TexturesProvider {
return &mojangtextures.MojangApiTexturesProvider{ return &mojangtextures.MojangApiTexturesProvider{
Emitter: emitter, Emitter: emitter,

9
docker-compose.prod.yml
View File

@ -20,15 +20,6 @@ services:
environment: environment:
CHRLY_SECRET: replace_this_value_in_production CHRLY_SECRET: replace_this_value_in_production
# Use this configuration in case when you need a remote Mojang UUIDs provider
# worker:
# image: elyby/chrly
# hostname: chrly0
# restart: always
# ports:
# - "8080:80"
# command: ["worker"]
redis: redis:
image: redis:4.0-32bit # 32-bit version is recommended to spare some memory image: redis:4.0-32bit # 32-bit version is recommended to spare some memory
restart: always restart: always

2
docker-entrypoint.sh
View File

@ -5,7 +5,7 @@ if [ ! -d /data/capes ]; then
mkdir -p /data/capes mkdir -p /data/capes
fi fi
if [ "$1" = "serve" ] || [ "$1" = "worker" ] || [ "$1" = "token" ] || [ "$1" = "version" ]; then if [ "$1" = "serve" ] || [ "$1" = "token" ] || [ "$1" = "version" ]; then
set -- /usr/local/bin/chrly "$@" set -- /usr/local/bin/chrly "$@"
fi fi

53
http/uuids_worker.go
View File

@ -1,53 +0,0 @@
package http
import (
"encoding/json"
"net/http"
"github.com/gorilla/mux"
"github.com/elyby/chrly/api/mojang"
)
type MojangUuidsProvider interface {
GetUuid(username string) (*mojang.ProfileInfo, error)
}
type UUIDsWorker struct {
MojangUuidsProvider
}
func (ctx *UUIDsWorker) Handler() *mux.Router {
router := mux.NewRouter().StrictSlash(true)
router.Handle("/mojang-uuid/{username}", http.HandlerFunc(ctx.getUUIDHandler)).Methods("GET")
return router
}
func (ctx *UUIDsWorker) getUUIDHandler(response http.ResponseWriter, request *http.Request) {
username := mux.Vars(request)["username"]
profile, err := ctx.GetUuid(username)
if err != nil {
if _, ok := err.(*mojang.TooManyRequestsError); ok {
response.WriteHeader(http.StatusTooManyRequests)
return
}
response.Header().Set("Content-Type", "application/json")
response.WriteHeader(http.StatusInternalServerError)
result, _ := json.Marshal(map[string]interface{}{
"provider": err.Error(),
})
_, _ = response.Write(result)
return
}
if profile == nil {
response.WriteHeader(http.StatusNoContent)
return
}
response.Header().Set("Content-Type", "application/json")
responseData, _ := json.Marshal(profile)
_, _ = response.Write(responseData)
}

154
http/uuids_worker_test.go
View File

@ -1,154 +0,0 @@
package http
import (
"errors"
"io/ioutil"
"net/http"
"net/http/httptest"
"testing"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/suite"
"github.com/elyby/chrly/api/mojang"
)
/***************
* Setup mocks *
***************/
type uuidsProviderMock struct {
mock.Mock
}
func (m *uuidsProviderMock) GetUuid(username string) (*mojang.ProfileInfo, error) {
args := m.Called(username)
var result *mojang.ProfileInfo
if casted, ok := args.Get(0).(*mojang.ProfileInfo); ok {
result = casted
}
return result, args.Error(1)
}
type uuidsWorkerTestSuite struct {
suite.Suite
App *UUIDsWorker
UuidsProvider *uuidsProviderMock
}
/********************
* Setup test suite *
********************/
func (suite *uuidsWorkerTestSuite) SetupTest() {
suite.UuidsProvider = &uuidsProviderMock{}
suite.App = &UUIDsWorker{
MojangUuidsProvider: suite.UuidsProvider,
}
}
func (suite *uuidsWorkerTestSuite) TearDownTest() {
suite.UuidsProvider.AssertExpectations(suite.T())
}
func (suite *uuidsWorkerTestSuite) RunSubTest(name string, subTest func()) {
suite.SetupTest()
suite.Run(name, subTest)
suite.TearDownTest()
}
/*************
* Run tests *
*************/
func TestUUIDsWorker(t *testing.T) {
suite.Run(t, new(uuidsWorkerTestSuite))
}
type uuidsWorkerTestCase struct {
Name string
BeforeTest func(suite *uuidsWorkerTestSuite)
AfterTest func(suite *uuidsWorkerTestSuite, response *http.Response)
}
/************************
* Get UUID tests cases *
************************/
var getUuidTestsCases = []*uuidsWorkerTestCase{
{
Name: "Success provider response",
BeforeTest: func(suite *uuidsWorkerTestSuite) {
suite.UuidsProvider.On("GetUuid", "mock_username").Return(&mojang.ProfileInfo{
Id: "0fcc38620f1845f3a54e1b523c1bd1c7",
Name: "mock_username",
}, nil)
},
AfterTest: func(suite *uuidsWorkerTestSuite, response *http.Response) {
suite.Equal(200, response.StatusCode)
suite.Equal("application/json", response.Header.Get("Content-Type"))
body, _ := ioutil.ReadAll(response.Body)
suite.JSONEq(`{
"id": "0fcc38620f1845f3a54e1b523c1bd1c7",
"name": "mock_username"
}`, string(body))
},
},
{
Name: "Receive empty response from UUIDs provider",
BeforeTest: func(suite *uuidsWorkerTestSuite) {
suite.UuidsProvider.On("GetUuid", "mock_username").Return(nil, nil)
},
AfterTest: func(suite *uuidsWorkerTestSuite, response *http.Response) {
suite.Equal(204, response.StatusCode)
body, _ := ioutil.ReadAll(response.Body)
suite.Assert().Empty(body)
},
},
{
Name: "Receive error from UUIDs provider",
BeforeTest: func(suite *uuidsWorkerTestSuite) {
err := errors.New("this is an error")
suite.UuidsProvider.On("GetUuid", "mock_username").Return(nil, err)
},
AfterTest: func(suite *uuidsWorkerTestSuite, response *http.Response) {
suite.Equal(500, response.StatusCode)
suite.Equal("application/json", response.Header.Get("Content-Type"))
body, _ := ioutil.ReadAll(response.Body)
suite.JSONEq(`{
"provider": "this is an error"
}`, string(body))
},
},
{
Name: "Receive Too Many Requests from UUIDs provider",
BeforeTest: func(suite *uuidsWorkerTestSuite) {
err := &mojang.TooManyRequestsError{}
suite.UuidsProvider.On("GetUuid", "mock_username").Return(nil, err)
},
AfterTest: func(suite *uuidsWorkerTestSuite, response *http.Response) {
suite.Equal(429, response.StatusCode)
body, _ := ioutil.ReadAll(response.Body)
suite.Empty(body)
},
},
}
func (suite *uuidsWorkerTestSuite) TestGetUUID() {
for _, testCase := range getUuidTestsCases {
suite.RunSubTest(testCase.Name, func() {
testCase.BeforeTest(suite)
req := httptest.NewRequest("GET", "http://chrly/mojang-uuid/mock_username", nil)
w := httptest.NewRecorder()
suite.App.Handler().ServeHTTP(w, req)
testCase.AfterTest(suite, w.Result())
})
}
}

67
mojangtextures/remote_api_uuids_provider.go
View File

@ -1,67 +0,0 @@
package mojangtextures
import (
"encoding/json"
"io/ioutil"
"net/http"
. "net/url"
"path"
"github.com/elyby/chrly/api/mojang"
"github.com/elyby/chrly/version"
)
var HttpClient = &http.Client{
Transport: &http.Transport{
MaxIdleConnsPerHost: 1024,
},
}
type RemoteApiUuidsProvider struct {
Emitter
Url URL
}
func (ctx *RemoteApiUuidsProvider) GetUuid(username string) (*mojang.ProfileInfo, error) {
url := ctx.Url
url.Path = path.Join(url.Path, username)
urlStr := url.String()
request, _ := http.NewRequest("GET", urlStr, nil)
request.Header.Add("Accept", "application/json")
// Change default User-Agent to allow specify "Username -> UUID at time" Mojang's api endpoint
request.Header.Add("User-Agent", "Chrly/"+version.Version())
ctx.Emit("mojang_textures:remote_api_uuids_provider:before_request", urlStr)
response, err := HttpClient.Do(request)
ctx.Emit("mojang_textures:remote_api_uuids_provider:after_request", response, err)
if err != nil {
return nil, err
}
defer response.Body.Close()
if response.StatusCode == 204 {
return nil, nil
}
if response.StatusCode != 200 {
return nil, &UnexpectedRemoteApiResponse{response}
}
var result *mojang.ProfileInfo
body, _ := ioutil.ReadAll(response.Body)
err = json.Unmarshal(body, &result)
if err != nil {
return nil, err
}
return result, nil
}
type UnexpectedRemoteApiResponse struct {
Response *http.Response
}
func (*UnexpectedRemoteApiResponse) Error() string {
return "Unexpected remote api response"
}

168
mojangtextures/remote_api_uuids_provider_test.go
View File

@ -1,168 +0,0 @@
package mojangtextures
import (
"net"
"net/http"
. "net/url"
"testing"
"github.com/h2non/gock"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/suite"
)
type remoteApiUuidsProviderTestSuite struct {
suite.Suite
Provider *RemoteApiUuidsProvider
Emitter *mockEmitter
}
func (suite *remoteApiUuidsProviderTestSuite) SetupSuite() {
client := &http.Client{}
gock.InterceptClient(client)
HttpClient = client
}
func (suite *remoteApiUuidsProviderTestSuite) SetupTest() {
suite.Emitter = &mockEmitter{}
suite.Provider = &RemoteApiUuidsProvider{
Emitter: suite.Emitter,
}
}
func (suite *remoteApiUuidsProviderTestSuite) TearDownTest() {
suite.Emitter.AssertExpectations(suite.T())
gock.Off()
}
func TestRemoteApiUuidsProvider(t *testing.T) {
suite.Run(t, new(remoteApiUuidsProviderTestSuite))
}
func (suite *remoteApiUuidsProviderTestSuite) TestGetUuidForValidUsername() {
suite.Emitter.On("Emit", "mojang_textures:remote_api_uuids_provider:before_request", "http://example.com/subpath/username").Once()
suite.Emitter.On("Emit",
"mojang_textures:remote_api_uuids_provider:after_request",
mock.AnythingOfType("*http.Response"),
nil,
).Once()
gock.New("http://example.com").
Get("/subpath/username").
Reply(200).
JSON(map[string]interface{}{
"id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"name": "username",
})
suite.Provider.Url = shouldParseUrl("http://example.com/subpath")
result, err := suite.Provider.GetUuid("username")
assert := suite.Assert()
if assert.NoError(err) {
assert.Equal("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", result.Id)
assert.Equal("username", result.Name)
assert.False(result.IsLegacy)
assert.False(result.IsDemo)
}
}
func (suite *remoteApiUuidsProviderTestSuite) TestGetUuidForNotExistsUsername() {
suite.Emitter.On("Emit", "mojang_textures:remote_api_uuids_provider:before_request", "http://example.com/subpath/username").Once()
suite.Emitter.On("Emit",
"mojang_textures:remote_api_uuids_provider:after_request",
mock.AnythingOfType("*http.Response"),
nil,
).Once()
gock.New("http://example.com").
Get("/subpath/username").
Reply(204)
suite.Provider.Url = shouldParseUrl("http://example.com/subpath")
result, err := suite.Provider.GetUuid("username")
assert := suite.Assert()
assert.Nil(result)
assert.Nil(err)
}
func (suite *remoteApiUuidsProviderTestSuite) TestGetUuidForNon20xResponse() {
suite.Emitter.On("Emit", "mojang_textures:remote_api_uuids_provider:before_request", "http://example.com/subpath/username").Once()
suite.Emitter.On("Emit",
"mojang_textures:remote_api_uuids_provider:after_request",
mock.AnythingOfType("*http.Response"),
nil,
).Once()
gock.New("http://example.com").
Get("/subpath/username").
Reply(504).
BodyString("504 Gateway Timeout")
suite.Provider.Url = shouldParseUrl("http://example.com/subpath")
result, err := suite.Provider.GetUuid("username")
assert := suite.Assert()
assert.Nil(result)
assert.EqualError(err, "Unexpected remote api response")
}
func (suite *remoteApiUuidsProviderTestSuite) TestGetUuidForNotSuccessRequest() {
suite.Emitter.On("Emit", "mojang_textures:remote_api_uuids_provider:before_request", "http://example.com/subpath/username").Once()
suite.Emitter.On("Emit",
"mojang_textures:remote_api_uuids_provider:after_request",
mock.AnythingOfType("*http.Response"),
mock.AnythingOfType("*url.Error"),
).Once()
expectedError := &net.OpError{Op: "dial"}
gock.New("http://example.com").
Get("/subpath/username").
ReplyError(expectedError)
suite.Provider.Url = shouldParseUrl("http://example.com/subpath")
result, err := suite.Provider.GetUuid("username")
assert := suite.Assert()
assert.Nil(result)
if assert.Error(err) {
assert.IsType(&Error{}, err)
casterErr, _ := err.(*Error)
assert.Equal(expectedError, casterErr.Err)
}
}
func (suite *remoteApiUuidsProviderTestSuite) TestGetUuidForInvalidSuccessResponse() {
suite.Emitter.On("Emit", "mojang_textures:remote_api_uuids_provider:before_request", "http://example.com/subpath/username").Once()
suite.Emitter.On("Emit",
"mojang_textures:remote_api_uuids_provider:after_request",
mock.AnythingOfType("*http.Response"),
nil,
).Once()
gock.New("http://example.com").
Get("/subpath/username").
Reply(200).
BodyString("completely not json")
suite.Provider.Url = shouldParseUrl("http://example.com/subpath")
result, err := suite.Provider.GetUuid("username")
assert := suite.Assert()
assert.Nil(result)
assert.Error(err)
}
func shouldParseUrl(rawUrl string) URL {
url, err := Parse(rawUrl)
if err != nil {
panic(err)
}
return *url
}