From e568d4cf91faf81678f2ef43fe14f0a3e69ef097 Mon Sep 17 00:00:00 2001 From: ErickSkrauch Date: Fri, 15 Dec 2023 03:42:38 +0100 Subject: [PATCH] Remove worker mode --- CHANGELOG.md | 1 + README.md | 41 ----- cmd/worker.go | 17 -- di/handlers.go | 19 -- di/mojang_textures.go | 40 ----- docker-compose.prod.yml | 9 - docker-entrypoint.sh | 2 +- http/uuids_worker.go | 53 ------ http/uuids_worker_test.go | 154 ---------------- mojangtextures/remote_api_uuids_provider.go | 67 ------- .../remote_api_uuids_provider_test.go | 168 ------------------ 11 files changed, 2 insertions(+), 569 deletions(-) delete mode 100644 cmd/worker.go delete mode 100644 http/uuids_worker.go delete mode 100644 http/uuids_worker_test.go delete mode 100644 mojangtextures/remote_api_uuids_provider.go delete mode 100644 mojangtextures/remote_api_uuids_provider_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 66ae8da..4c4a40e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - StatsD metrics: - Gauges: - `ely.skinsystem.{hostname}.app.redis.pool.available` +- Worker mode. Use URL spoofing to load balance outgoing requests. ## [4.6.0] - 2021-03-04 ### Added diff --git a/README.md b/README.md index 18f8090..50081a1 100644 --- a/README.md +++ b/README.md @@ -134,22 +134,6 @@ docker-compose up -d app true - - MOJANG_TEXTURES_UUIDS_PROVIDER_DRIVER - - Specifies the preferred provider of the Mojang's UUIDs. Takes remote value. - In any other case, the local queue will be used. - - remote - - - MOJANG_TEXTURES_UUIDS_PROVIDER_URL - - When the UUIDs driver set to remote, sets the remote URL. - The trailing slash won't cause any problems. - - http://remote-provider.com/api/worker/mojang-uuid - MOJANG_API_BASE_URL @@ -399,31 +383,6 @@ response will be: } ``` -### Worker mode - -The worker mode can be used in cooperation with the [remote server mode](#remote-mojang-uuids-provider) -to exchange Mojang usernames to UUIDs. This mode by itself doesn't solve the problem of -[extremely strict limits](https://github.com/elyby/chrly/issues/10) on the number of requests to the Mojang's API. -But with a proxying load balancer (e.g. HAProxy, Nginx, etc.) it's easy to build a cluster of workers, -which will multiply the bandwidth of the exchanging usernames to its UUIDs. - -The instructions for setting up a proxy load balancer are outside the context of this documentation, -but you get the idea ;) - -#### `GET /api/worker/mojang-uuid/{username}` - -Performs [batch usernames exchange to UUIDs](https://github.com/elyby/chrly/issues/1) and returns the result in the -[same format as it returns from the Mojang's API](https://wiki.vg/Mojang_API#Username_-.3E_UUID_at_time): - -```json -{ - "id": "3e3ee6c35afa48abb61e8cd8c42fc0d9", - "name": "ErickSkrauch" -} -``` - -> **Note**: the results aren't cached. - ### Health check #### `GET /healthcheck` diff --git a/cmd/worker.go b/cmd/worker.go deleted file mode 100644 index b3bd4aa..0000000 --- a/cmd/worker.go +++ /dev/null @@ -1,17 +0,0 @@ -package cmd - -import ( - "github.com/spf13/cobra" -) - -var workerCmd = &cobra.Command{ - Use: "worker", - Short: "Starts HTTP handler for the Mojang usernames to UUIDs worker", - Run: func(cmd *cobra.Command, args []string) { - startServer([]string{"worker"}) - }, -} - -func init() { - RootCmd.AddCommand(workerCmd) -} diff --git a/di/handlers.go b/di/handlers.go index a0ed9dc..08527b2 100644 --- a/di/handlers.go +++ b/di/handlers.go @@ -11,14 +11,12 @@ import ( "github.com/spf13/viper" . "github.com/elyby/chrly/http" - "github.com/elyby/chrly/mojangtextures" ) var handlers = di.Options( di.Provide(newHandlerFactory, di.As(new(http.Handler))), di.Provide(newSkinsystemHandler, di.WithName("skinsystem")), di.Provide(newApiHandler, di.WithName("api")), - di.Provide(newUUIDsWorkerHandler, di.WithName("worker")), ) func newHandlerFactory( @@ -48,17 +46,6 @@ func newHandlerFactory( // See https://github.com/gorilla/mux/issues/416#issuecomment-600079279 router.NotFoundHandler = requestEventsMiddleware(http.HandlerFunc(NotFoundHandler)) - // Enable the worker module before api to allow gorilla.mux to correctly find the target router - // as it uses the first matching and /api overrides the more accurate /api/worker - if hasValue(enabledModules, "worker") { - var workerRouter *mux.Router - if err := container.Resolve(&workerRouter, di.Name("worker")); err != nil { - return nil, err - } - - mount(router, "/api/worker", workerRouter) - } - if hasValue(enabledModules, "api") { var apiRouter *mux.Router if err := container.Resolve(&apiRouter, di.Name("api")); err != nil { @@ -127,12 +114,6 @@ func newApiHandler(skinsRepository SkinsRepository) *mux.Router { }).Handler() } -func newUUIDsWorkerHandler(mojangUUIDsProvider *mojangtextures.BatchUuidsProvider) *mux.Router { - return (&UUIDsWorker{ - MojangUuidsProvider: mojangUUIDsProvider, - }).Handler() -} - func hasValue(slice []string, needle string) bool { for _, value := range slice { if value == needle { diff --git a/di/mojang_textures.go b/di/mojang_textures.go index 47da614..9f1aa64 100644 --- a/di/mojang_textures.go +++ b/di/mojang_textures.go @@ -24,7 +24,6 @@ var mojangTextures = di.Options( di.Provide(newMojangTexturesBatchUUIDsProviderStrategyFactory), di.Provide(newMojangTexturesBatchUUIDsProviderDelayedStrategy), di.Provide(newMojangTexturesBatchUUIDsProviderFullBusStrategy), - di.Provide(newMojangTexturesRemoteUUIDsProvider), di.Provide(newMojangSignedTexturesProvider), di.Provide(newMojangTexturesStorageFactory), ) @@ -86,17 +85,8 @@ func newMojangTexturesProvider( } func newMojangTexturesUuidsProviderFactory( - config *viper.Viper, container *di.Container, ) (mojangtextures.UUIDsProvider, error) { - preferredUuidsProvider := config.GetString("mojang_textures.uuids_provider.driver") - if preferredUuidsProvider == "remote" { - var provider *mojangtextures.RemoteApiUuidsProvider - err := container.Resolve(&provider) - - return provider, err - } - var provider *mojangtextures.BatchUuidsProvider err := container.Resolve(&provider) @@ -188,36 +178,6 @@ func newMojangTexturesBatchUUIDsProviderFullBusStrategy(config *viper.Viper) *mo ) } -func newMojangTexturesRemoteUUIDsProvider( - container *di.Container, - config *viper.Viper, - emitter mojangtextures.Emitter, -) (*mojangtextures.RemoteApiUuidsProvider, error) { - remoteUrl, err := url.Parse(config.GetString("mojang_textures.uuids_provider.url")) - if err != nil { - return nil, fmt.Errorf("unable to parse remote url: %w", err) - } - - if err := container.Provide(func(emitter es.Subscriber, config *viper.Viper) *namedHealthChecker { - config.SetDefault("healthcheck.mojang_api_textures_provider_cool_down_duration", time.Minute+10*time.Second) - - return &namedHealthChecker{ - Name: "mojang-api-textures-provider-response-checker", - Checker: es.MojangApiTexturesProviderResponseChecker( - emitter, - config.GetDuration("healthcheck.mojang_api_textures_provider_cool_down_duration"), - ), - } - }); err != nil { - return nil, err - } - - return &mojangtextures.RemoteApiUuidsProvider{ - Emitter: emitter, - Url: *remoteUrl, - }, nil -} - func newMojangSignedTexturesProvider(emitter mojangtextures.Emitter) mojangtextures.TexturesProvider { return &mojangtextures.MojangApiTexturesProvider{ Emitter: emitter, diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index fa306ac..ca4e940 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -20,15 +20,6 @@ services: environment: CHRLY_SECRET: replace_this_value_in_production - # Use this configuration in case when you need a remote Mojang UUIDs provider - # worker: - # image: elyby/chrly - # hostname: chrly0 - # restart: always - # ports: - # - "8080:80" - # command: ["worker"] - redis: image: redis:4.0-32bit # 32-bit version is recommended to spare some memory restart: always diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index cefed94..de3fa34 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -5,7 +5,7 @@ if [ ! -d /data/capes ]; then mkdir -p /data/capes fi -if [ "$1" = "serve" ] || [ "$1" = "worker" ] || [ "$1" = "token" ] || [ "$1" = "version" ]; then +if [ "$1" = "serve" ] || [ "$1" = "token" ] || [ "$1" = "version" ]; then set -- /usr/local/bin/chrly "$@" fi diff --git a/http/uuids_worker.go b/http/uuids_worker.go deleted file mode 100644 index a5224de..0000000 --- a/http/uuids_worker.go +++ /dev/null @@ -1,53 +0,0 @@ -package http - -import ( - "encoding/json" - "net/http" - - "github.com/gorilla/mux" - - "github.com/elyby/chrly/api/mojang" -) - -type MojangUuidsProvider interface { - GetUuid(username string) (*mojang.ProfileInfo, error) -} - -type UUIDsWorker struct { - MojangUuidsProvider -} - -func (ctx *UUIDsWorker) Handler() *mux.Router { - router := mux.NewRouter().StrictSlash(true) - router.Handle("/mojang-uuid/{username}", http.HandlerFunc(ctx.getUUIDHandler)).Methods("GET") - - return router -} - -func (ctx *UUIDsWorker) getUUIDHandler(response http.ResponseWriter, request *http.Request) { - username := mux.Vars(request)["username"] - profile, err := ctx.GetUuid(username) - if err != nil { - if _, ok := err.(*mojang.TooManyRequestsError); ok { - response.WriteHeader(http.StatusTooManyRequests) - return - } - - response.Header().Set("Content-Type", "application/json") - response.WriteHeader(http.StatusInternalServerError) - result, _ := json.Marshal(map[string]interface{}{ - "provider": err.Error(), - }) - _, _ = response.Write(result) - return - } - - if profile == nil { - response.WriteHeader(http.StatusNoContent) - return - } - - response.Header().Set("Content-Type", "application/json") - responseData, _ := json.Marshal(profile) - _, _ = response.Write(responseData) -} diff --git a/http/uuids_worker_test.go b/http/uuids_worker_test.go deleted file mode 100644 index 4c32ee1..0000000 --- a/http/uuids_worker_test.go +++ /dev/null @@ -1,154 +0,0 @@ -package http - -import ( - "errors" - "io/ioutil" - "net/http" - "net/http/httptest" - "testing" - - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/suite" - - "github.com/elyby/chrly/api/mojang" -) - -/*************** - * Setup mocks * - ***************/ - -type uuidsProviderMock struct { - mock.Mock -} - -func (m *uuidsProviderMock) GetUuid(username string) (*mojang.ProfileInfo, error) { - args := m.Called(username) - var result *mojang.ProfileInfo - if casted, ok := args.Get(0).(*mojang.ProfileInfo); ok { - result = casted - } - - return result, args.Error(1) -} - -type uuidsWorkerTestSuite struct { - suite.Suite - - App *UUIDsWorker - - UuidsProvider *uuidsProviderMock -} - -/******************** - * Setup test suite * - ********************/ - -func (suite *uuidsWorkerTestSuite) SetupTest() { - suite.UuidsProvider = &uuidsProviderMock{} - - suite.App = &UUIDsWorker{ - MojangUuidsProvider: suite.UuidsProvider, - } -} - -func (suite *uuidsWorkerTestSuite) TearDownTest() { - suite.UuidsProvider.AssertExpectations(suite.T()) -} - -func (suite *uuidsWorkerTestSuite) RunSubTest(name string, subTest func()) { - suite.SetupTest() - suite.Run(name, subTest) - suite.TearDownTest() -} - -/************* - * Run tests * - *************/ - -func TestUUIDsWorker(t *testing.T) { - suite.Run(t, new(uuidsWorkerTestSuite)) -} - -type uuidsWorkerTestCase struct { - Name string - BeforeTest func(suite *uuidsWorkerTestSuite) - AfterTest func(suite *uuidsWorkerTestSuite, response *http.Response) -} - -/************************ - * Get UUID tests cases * - ************************/ - -var getUuidTestsCases = []*uuidsWorkerTestCase{ - { - Name: "Success provider response", - BeforeTest: func(suite *uuidsWorkerTestSuite) { - suite.UuidsProvider.On("GetUuid", "mock_username").Return(&mojang.ProfileInfo{ - Id: "0fcc38620f1845f3a54e1b523c1bd1c7", - Name: "mock_username", - }, nil) - }, - AfterTest: func(suite *uuidsWorkerTestSuite, response *http.Response) { - suite.Equal(200, response.StatusCode) - suite.Equal("application/json", response.Header.Get("Content-Type")) - body, _ := ioutil.ReadAll(response.Body) - suite.JSONEq(`{ - "id": "0fcc38620f1845f3a54e1b523c1bd1c7", - "name": "mock_username" - }`, string(body)) - }, - }, - { - Name: "Receive empty response from UUIDs provider", - BeforeTest: func(suite *uuidsWorkerTestSuite) { - suite.UuidsProvider.On("GetUuid", "mock_username").Return(nil, nil) - }, - AfterTest: func(suite *uuidsWorkerTestSuite, response *http.Response) { - suite.Equal(204, response.StatusCode) - body, _ := ioutil.ReadAll(response.Body) - suite.Assert().Empty(body) - }, - }, - { - Name: "Receive error from UUIDs provider", - BeforeTest: func(suite *uuidsWorkerTestSuite) { - err := errors.New("this is an error") - suite.UuidsProvider.On("GetUuid", "mock_username").Return(nil, err) - }, - AfterTest: func(suite *uuidsWorkerTestSuite, response *http.Response) { - suite.Equal(500, response.StatusCode) - suite.Equal("application/json", response.Header.Get("Content-Type")) - body, _ := ioutil.ReadAll(response.Body) - suite.JSONEq(`{ - "provider": "this is an error" - }`, string(body)) - }, - }, - { - Name: "Receive Too Many Requests from UUIDs provider", - BeforeTest: func(suite *uuidsWorkerTestSuite) { - err := &mojang.TooManyRequestsError{} - suite.UuidsProvider.On("GetUuid", "mock_username").Return(nil, err) - }, - AfterTest: func(suite *uuidsWorkerTestSuite, response *http.Response) { - suite.Equal(429, response.StatusCode) - body, _ := ioutil.ReadAll(response.Body) - suite.Empty(body) - }, - }, -} - -func (suite *uuidsWorkerTestSuite) TestGetUUID() { - for _, testCase := range getUuidTestsCases { - suite.RunSubTest(testCase.Name, func() { - testCase.BeforeTest(suite) - - req := httptest.NewRequest("GET", "http://chrly/mojang-uuid/mock_username", nil) - w := httptest.NewRecorder() - - suite.App.Handler().ServeHTTP(w, req) - - testCase.AfterTest(suite, w.Result()) - }) - } -} diff --git a/mojangtextures/remote_api_uuids_provider.go b/mojangtextures/remote_api_uuids_provider.go deleted file mode 100644 index 235bab8..0000000 --- a/mojangtextures/remote_api_uuids_provider.go +++ /dev/null @@ -1,67 +0,0 @@ -package mojangtextures - -import ( - "encoding/json" - "io/ioutil" - "net/http" - . "net/url" - "path" - - "github.com/elyby/chrly/api/mojang" - "github.com/elyby/chrly/version" -) - -var HttpClient = &http.Client{ - Transport: &http.Transport{ - MaxIdleConnsPerHost: 1024, - }, -} - -type RemoteApiUuidsProvider struct { - Emitter - Url URL -} - -func (ctx *RemoteApiUuidsProvider) GetUuid(username string) (*mojang.ProfileInfo, error) { - url := ctx.Url - url.Path = path.Join(url.Path, username) - urlStr := url.String() - - request, _ := http.NewRequest("GET", urlStr, nil) - request.Header.Add("Accept", "application/json") - // Change default User-Agent to allow specify "Username -> UUID at time" Mojang's api endpoint - request.Header.Add("User-Agent", "Chrly/"+version.Version()) - - ctx.Emit("mojang_textures:remote_api_uuids_provider:before_request", urlStr) - response, err := HttpClient.Do(request) - ctx.Emit("mojang_textures:remote_api_uuids_provider:after_request", response, err) - if err != nil { - return nil, err - } - defer response.Body.Close() - - if response.StatusCode == 204 { - return nil, nil - } - - if response.StatusCode != 200 { - return nil, &UnexpectedRemoteApiResponse{response} - } - - var result *mojang.ProfileInfo - body, _ := ioutil.ReadAll(response.Body) - err = json.Unmarshal(body, &result) - if err != nil { - return nil, err - } - - return result, nil -} - -type UnexpectedRemoteApiResponse struct { - Response *http.Response -} - -func (*UnexpectedRemoteApiResponse) Error() string { - return "Unexpected remote api response" -} diff --git a/mojangtextures/remote_api_uuids_provider_test.go b/mojangtextures/remote_api_uuids_provider_test.go deleted file mode 100644 index 9979a95..0000000 --- a/mojangtextures/remote_api_uuids_provider_test.go +++ /dev/null @@ -1,168 +0,0 @@ -package mojangtextures - -import ( - "net" - "net/http" - . "net/url" - "testing" - - "github.com/h2non/gock" - - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/suite" -) - -type remoteApiUuidsProviderTestSuite struct { - suite.Suite - - Provider *RemoteApiUuidsProvider - Emitter *mockEmitter -} - -func (suite *remoteApiUuidsProviderTestSuite) SetupSuite() { - client := &http.Client{} - gock.InterceptClient(client) - - HttpClient = client -} - -func (suite *remoteApiUuidsProviderTestSuite) SetupTest() { - suite.Emitter = &mockEmitter{} - suite.Provider = &RemoteApiUuidsProvider{ - Emitter: suite.Emitter, - } -} - -func (suite *remoteApiUuidsProviderTestSuite) TearDownTest() { - suite.Emitter.AssertExpectations(suite.T()) - gock.Off() -} - -func TestRemoteApiUuidsProvider(t *testing.T) { - suite.Run(t, new(remoteApiUuidsProviderTestSuite)) -} - -func (suite *remoteApiUuidsProviderTestSuite) TestGetUuidForValidUsername() { - suite.Emitter.On("Emit", "mojang_textures:remote_api_uuids_provider:before_request", "http://example.com/subpath/username").Once() - suite.Emitter.On("Emit", - "mojang_textures:remote_api_uuids_provider:after_request", - mock.AnythingOfType("*http.Response"), - nil, - ).Once() - - gock.New("http://example.com"). - Get("/subpath/username"). - Reply(200). - JSON(map[string]interface{}{ - "id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "name": "username", - }) - - suite.Provider.Url = shouldParseUrl("http://example.com/subpath") - result, err := suite.Provider.GetUuid("username") - - assert := suite.Assert() - if assert.NoError(err) { - assert.Equal("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", result.Id) - assert.Equal("username", result.Name) - assert.False(result.IsLegacy) - assert.False(result.IsDemo) - } -} - -func (suite *remoteApiUuidsProviderTestSuite) TestGetUuidForNotExistsUsername() { - suite.Emitter.On("Emit", "mojang_textures:remote_api_uuids_provider:before_request", "http://example.com/subpath/username").Once() - suite.Emitter.On("Emit", - "mojang_textures:remote_api_uuids_provider:after_request", - mock.AnythingOfType("*http.Response"), - nil, - ).Once() - - gock.New("http://example.com"). - Get("/subpath/username"). - Reply(204) - - suite.Provider.Url = shouldParseUrl("http://example.com/subpath") - result, err := suite.Provider.GetUuid("username") - - assert := suite.Assert() - assert.Nil(result) - assert.Nil(err) -} - -func (suite *remoteApiUuidsProviderTestSuite) TestGetUuidForNon20xResponse() { - suite.Emitter.On("Emit", "mojang_textures:remote_api_uuids_provider:before_request", "http://example.com/subpath/username").Once() - suite.Emitter.On("Emit", - "mojang_textures:remote_api_uuids_provider:after_request", - mock.AnythingOfType("*http.Response"), - nil, - ).Once() - - gock.New("http://example.com"). - Get("/subpath/username"). - Reply(504). - BodyString("504 Gateway Timeout") - - suite.Provider.Url = shouldParseUrl("http://example.com/subpath") - result, err := suite.Provider.GetUuid("username") - - assert := suite.Assert() - assert.Nil(result) - assert.EqualError(err, "Unexpected remote api response") -} - -func (suite *remoteApiUuidsProviderTestSuite) TestGetUuidForNotSuccessRequest() { - suite.Emitter.On("Emit", "mojang_textures:remote_api_uuids_provider:before_request", "http://example.com/subpath/username").Once() - suite.Emitter.On("Emit", - "mojang_textures:remote_api_uuids_provider:after_request", - mock.AnythingOfType("*http.Response"), - mock.AnythingOfType("*url.Error"), - ).Once() - - expectedError := &net.OpError{Op: "dial"} - - gock.New("http://example.com"). - Get("/subpath/username"). - ReplyError(expectedError) - - suite.Provider.Url = shouldParseUrl("http://example.com/subpath") - result, err := suite.Provider.GetUuid("username") - - assert := suite.Assert() - assert.Nil(result) - if assert.Error(err) { - assert.IsType(&Error{}, err) - casterErr, _ := err.(*Error) - assert.Equal(expectedError, casterErr.Err) - } -} - -func (suite *remoteApiUuidsProviderTestSuite) TestGetUuidForInvalidSuccessResponse() { - suite.Emitter.On("Emit", "mojang_textures:remote_api_uuids_provider:before_request", "http://example.com/subpath/username").Once() - suite.Emitter.On("Emit", - "mojang_textures:remote_api_uuids_provider:after_request", - mock.AnythingOfType("*http.Response"), - nil, - ).Once() - - gock.New("http://example.com"). - Get("/subpath/username"). - Reply(200). - BodyString("completely not json") - - suite.Provider.Url = shouldParseUrl("http://example.com/subpath") - result, err := suite.Provider.GetUuid("username") - - assert := suite.Assert() - assert.Nil(result) - assert.Error(err) -} - -func shouldParseUrl(rawUrl string) URL { - url, err := Parse(rawUrl) - if err != nil { - panic(err) - } - - return *url -}