Implemented worker command

This commit is contained in:
ErickSkrauch 2020-01-03 00:51:57 +03:00
parent 1e91aef0a6
commit 5a0c10c1a1
13 changed files with 367 additions and 53 deletions

View File

@ -31,7 +31,7 @@ jobs:
env CGO_ENABLED=0 GOOS=linux GOARCH=amd64 env CGO_ENABLED=0 GOOS=linux GOARCH=amd64
go build go build
-o release/chrly -o release/chrly
-ldflags '-extldflags "-static" -X github.com/elyby/chrly/bootstrap.version=$APP_VERSION' -ldflags '-extldflags "-static" -X github.com/elyby/chrly/version.version=$APP_VERSION -X github.com/elyby/chrly/version.commit=$TRAVIS_COMMIT'
main.go main.go
- docker build -t elyby/chrly:$DOCKER_TAG . - docker build -t elyby/chrly:$DOCKER_TAG .
- docker push elyby/chrly:$DOCKER_TAG - docker push elyby/chrly:$DOCKER_TAG

View File

@ -1,7 +1,9 @@
package bootstrap package bootstrap
import ( import (
"net/url"
"os" "os"
"time"
"github.com/getsentry/raven-go" "github.com/getsentry/raven-go"
"github.com/mono83/slf/rays" "github.com/mono83/slf/rays"
@ -9,19 +11,18 @@ import (
"github.com/mono83/slf/recievers/statsd" "github.com/mono83/slf/recievers/statsd"
"github.com/mono83/slf/recievers/writer" "github.com/mono83/slf/recievers/writer"
"github.com/mono83/slf/wd" "github.com/mono83/slf/wd"
"github.com/spf13/viper"
"github.com/elyby/chrly/mojangtextures"
"github.com/elyby/chrly/version"
) )
var version = ""
func GetVersion() string {
return version
}
func CreateLogger(statsdAddr string, sentryAddr string) (wd.Watchdog, error) { func CreateLogger(statsdAddr string, sentryAddr string) (wd.Watchdog, error) {
wd.AddReceiver(writer.New(writer.Options{ wd.AddReceiver(writer.New(writer.Options{
Marker: false, Marker: false,
TimeFormat: "15:04:05.000", TimeFormat: "15:04:05.000",
})) }))
if statsdAddr != "" { if statsdAddr != "" {
hostname, _ := os.Hostname() hostname, _ := os.Hostname()
statsdReceiver, err := statsd.NewReceiver(statsd.Config{ statsdReceiver, err := statsd.NewReceiver(statsd.Config{
@ -45,7 +46,7 @@ func CreateLogger(statsdAddr string, sentryAddr string) (wd.Watchdog, error) {
ravenClient.SetEnvironment("production") ravenClient.SetEnvironment("production")
ravenClient.SetDefaultLoggerName("sentry-watchdog-receiver") ravenClient.SetDefaultLoggerName("sentry-watchdog-receiver")
programVersion := GetVersion() programVersion := version.Version()
if programVersion != "" { if programVersion != "" {
raven.SetRelease(programVersion) raven.SetRelease(programVersion)
} }
@ -62,3 +63,32 @@ func CreateLogger(statsdAddr string, sentryAddr string) (wd.Watchdog, error) {
return wd.New("", "").WithParams(rays.Host), nil return wd.New("", "").WithParams(rays.Host), nil
} }
func init() {
viper.SetDefault("queue.loop_delay", 2*time.Second+500*time.Millisecond)
viper.SetDefault("queue.batch_size", 10)
}
func CreateMojangUUIDsProvider(logger wd.Watchdog) (mojangtextures.UUIDsProvider, error) {
var uuidsProvider mojangtextures.UUIDsProvider
preferredUuidsProvider := viper.GetString("mojang_textures.uuids_provider.driver")
if preferredUuidsProvider == "remote" {
remoteUrl, err := url.Parse(viper.GetString("mojang_textures.uuids_provider.url"))
if err != nil {
return nil, err
}
uuidsProvider = &mojangtextures.RemoteApiUuidsProvider{
Url: *remoteUrl,
Logger: logger,
}
} else {
uuidsProvider = &mojangtextures.BatchUuidsProvider{
IterationDelay: viper.GetDuration("queue.loop_delay"),
IterationSize: viper.GetInt("queue.batch_size"),
Logger: logger,
}
}
return uuidsProvider, nil
}

View File

@ -5,16 +5,16 @@ import (
"os" "os"
"strings" "strings"
"github.com/elyby/chrly/bootstrap"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/spf13/viper" "github.com/spf13/viper"
"github.com/elyby/chrly/version"
) )
var RootCmd = &cobra.Command{ var RootCmd = &cobra.Command{
Use: "chrly", Use: "chrly",
Short: "Implementation of Minecraft skins system server", Short: "Implementation of Minecraft skins system server",
Version: bootstrap.GetVersion(), Version: version.Version(),
} }
// Execute adds all child commands to the root command and sets flags appropriately. // Execute adds all child commands to the root command and sets flags appropriately.

View File

@ -3,8 +3,6 @@ package cmd
import ( import (
"fmt" "fmt"
"log" "log"
"net/url"
"time"
"github.com/mono83/slf/wd" "github.com/mono83/slf/wd"
"github.com/spf13/cobra" "github.com/spf13/cobra"
@ -19,7 +17,7 @@ import (
var serveCmd = &cobra.Command{ var serveCmd = &cobra.Command{
Use: "serve", Use: "serve",
Short: "Starts http handler for the skins system", Short: "Starts HTTP handler for the skins system",
Run: func(cmd *cobra.Command, args []string) { Run: func(cmd *cobra.Command, args []string) {
// TODO: this is a mess, need to organize this code somehow to make services initialization more compact // TODO: this is a mess, need to organize this code somehow to make services initialization more compact
logger, err := bootstrap.CreateLogger(viper.GetString("statsd.addr"), viper.GetString("sentry.dsn")) logger, err := bootstrap.CreateLogger(viper.GetString("statsd.addr"), viper.GetString("sentry.dsn"))
@ -55,32 +53,17 @@ var serveCmd = &cobra.Command{
return return
} }
var uuidsProvider mojangtextures.UuidsProvider uuidsProvider, err := bootstrap.CreateMojangUUIDsProvider(logger)
preferredUuidsProvider := viper.GetString("mojang_textures.uuids_provider.driver")
if preferredUuidsProvider == "remote" {
remoteUrl, err := url.Parse(viper.GetString("mojang_textures.uuids_provider.url"))
if err != nil { if err != nil {
logger.Emergency("Unable to parse remote url :err", wd.ErrParam(err)) logger.Emergency("Unable to parse remote url :err", wd.ErrParam(err))
return return
} }
uuidsProvider = &mojangtextures.RemoteApiUuidsProvider{
Url: *remoteUrl,
Logger: logger,
}
} else {
uuidsProvider = &mojangtextures.BatchUuidsProvider{
IterationDelay: time.Duration(viper.GetInt("queue.loop_delay")) * time.Millisecond,
IterationSize: viper.GetInt("queue.batch_size"),
Logger: logger,
}
}
texturesStorage := mojangtextures.NewInMemoryTexturesStorage() texturesStorage := mojangtextures.NewInMemoryTexturesStorage()
texturesStorage.Start() texturesStorage.Start()
mojangTexturesProvider := &mojangtextures.Provider{ mojangTexturesProvider := &mojangtextures.Provider{
Logger: logger, Logger: logger,
UuidsProvider: uuidsProvider, UUIDsProvider: uuidsProvider,
TexturesProvider: &mojangtextures.MojangApiTexturesProvider{ TexturesProvider: &mojangtextures.MojangApiTexturesProvider{
Logger: logger, Logger: logger,
}, },
@ -115,6 +98,4 @@ func init() {
viper.SetDefault("storage.redis.poll", 10) viper.SetDefault("storage.redis.poll", 10)
viper.SetDefault("storage.filesystem.basePath", "data") viper.SetDefault("storage.filesystem.basePath", "data")
viper.SetDefault("storage.filesystem.capesDirName", "capes") viper.SetDefault("storage.filesystem.capesDirName", "capes")
viper.SetDefault("queue.loop_delay", 2_500)
viper.SetDefault("queue.batch_size", 10)
} }

View File

@ -2,17 +2,19 @@ package cmd
import ( import (
"fmt" "fmt"
"runtime"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/elyby/chrly/bootstrap"
"runtime" "github.com/elyby/chrly/version"
) )
var versionCmd = &cobra.Command{ var versionCmd = &cobra.Command{
Use: "version", Use: "version",
Short: "Show the Chrly version information", Short: "Show the Chrly version information",
Run: func(cmd *cobra.Command, args []string) { Run: func(cmd *cobra.Command, args []string) {
fmt.Printf("Version: %s\n", bootstrap.GetVersion()) fmt.Printf("Version: %s\n", version.Version())
fmt.Printf("Commit: %s\n", version.Commit())
fmt.Printf("Go version: %s\n", runtime.Version()) fmt.Printf("Go version: %s\n", runtime.Version())
fmt.Printf("OS/Arch: %s/%s\n", runtime.GOOS, runtime.GOARCH) fmt.Printf("OS/Arch: %s/%s\n", runtime.GOOS, runtime.GOARCH)
}, },

45
cmd/worker.go Normal file
View File

@ -0,0 +1,45 @@
package cmd
import (
"fmt"
"log"
"github.com/mono83/slf/wd"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"github.com/elyby/chrly/bootstrap"
"github.com/elyby/chrly/http"
)
var workerCmd = &cobra.Command{
Use: "worker",
Short: "Starts HTTP handler for the Mojang usernames to UUIDs worker",
Run: func(cmd *cobra.Command, args []string) {
logger, err := bootstrap.CreateLogger(viper.GetString("statsd.addr"), viper.GetString("sentry.dsn"))
if err != nil {
log.Fatal(fmt.Printf("Cannot initialize logger: %v", err))
}
logger.Info("Logger successfully initialized")
uuidsProvider, err := bootstrap.CreateMojangUUIDsProvider(logger)
if err != nil {
logger.Emergency("Unable to parse remote url :err", wd.ErrParam(err))
return
}
cfg := &http.UUIDsWorker{
ListenSpec: fmt.Sprintf("%s:%d", viper.GetString("server.host"), viper.GetInt("server.port")),
UUIDsProvider: uuidsProvider,
Logger: logger,
}
if err := cfg.Run(); err != nil {
logger.Error(fmt.Sprintf("Error in main(): %v", err))
}
},
}
func init() {
RootCmd.AddCommand(workerCmd)
}

89
http/uuids_worker.go Normal file
View File

@ -0,0 +1,89 @@
package http
import (
"encoding/json"
"fmt"
"net"
"net/http"
"time"
"github.com/gorilla/mux"
"github.com/mono83/slf/wd"
"github.com/elyby/chrly/api/mojang"
"github.com/elyby/chrly/mojangtextures"
)
type UuidsProvider interface {
GetUuid(username string) (*mojang.ProfileInfo, error)
}
type UUIDsWorker struct {
ListenSpec string
UUIDsProvider mojangtextures.UUIDsProvider
Logger wd.Watchdog
}
func (ctx *UUIDsWorker) Run() error {
ctx.Logger.Info(fmt.Sprintf("Starting, HTTP on: %s\n", ctx.ListenSpec))
listener, err := net.Listen("tcp", ctx.ListenSpec)
if err != nil {
return err
}
server := &http.Server{
ReadTimeout: 60 * time.Second,
WriteTimeout: 60 * time.Second, // TODO: should I adjust this values?
MaxHeaderBytes: 1 << 16,
Handler: ctx.CreateHandler(),
}
// noinspection GoUnhandledErrorResult
go server.Serve(listener)
s := waitForSignal()
ctx.Logger.Info(fmt.Sprintf("Got signal: %v, exiting.", s))
return nil
}
func (ctx *UUIDsWorker) CreateHandler() http.Handler {
router := mux.NewRouter().StrictSlash(true)
router.NotFoundHandler = http.HandlerFunc(NotFound)
router.Handle("/api/worker/mojang-uuid/{username}", http.HandlerFunc(ctx.GetUUID)).Methods("GET")
return router
}
func (ctx *UUIDsWorker) GetUUID(response http.ResponseWriter, request *http.Request) {
username := parseUsername(mux.Vars(request)["username"])
profile, err := ctx.UUIDsProvider.GetUuid(username)
if err != nil {
if _, ok := err.(*mojang.TooManyRequestsError); ok {
ctx.Logger.Warning("Got 429 Too Many Requests")
response.WriteHeader(http.StatusTooManyRequests)
return
}
ctx.Logger.Warning("Got non success response: :err", wd.ErrParam(err))
response.Header().Set("Content-Type", "application/json")
response.WriteHeader(http.StatusInternalServerError)
result, _ := json.Marshal(map[string]interface{}{
"provider": err.Error(),
})
_, _ = response.Write(result)
return
}
if profile == nil {
response.WriteHeader(http.StatusNoContent)
return
}
response.Header().Set("Content-Type", "application/json")
responseData, _ := json.Marshal(profile)
_, _ = response.Write(responseData)
}

157
http/uuids_worker_test.go Normal file
View File

@ -0,0 +1,157 @@
package http
import (
"errors"
"github.com/elyby/chrly/api/mojang"
"github.com/elyby/chrly/tests"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/suite"
"io/ioutil"
"net/http"
"net/http/httptest"
"testing"
)
/***************
* Setup mocks *
***************/
type uuidsProviderMock struct {
mock.Mock
}
func (m *uuidsProviderMock) GetUuid(username string) (*mojang.ProfileInfo, error) {
args := m.Called(username)
var result *mojang.ProfileInfo
if casted, ok := args.Get(0).(*mojang.ProfileInfo); ok {
result = casted
}
return result, args.Error(1)
}
type uuidsWorkerTestSuite struct {
suite.Suite
App *UUIDsWorker
UuidsProvider *uuidsProviderMock
Logger *tests.WdMock
}
/********************
* Setup test suite *
********************/
func (suite *uuidsWorkerTestSuite) SetupTest() {
suite.UuidsProvider = &uuidsProviderMock{}
suite.Logger = &tests.WdMock{}
suite.App = &UUIDsWorker{
UUIDsProvider: suite.UuidsProvider,
Logger: suite.Logger,
}
}
func (suite *uuidsWorkerTestSuite) TearDownTest() {
suite.UuidsProvider.AssertExpectations(suite.T())
suite.Logger.AssertExpectations(suite.T())
}
func (suite *uuidsWorkerTestSuite) RunSubTest(name string, subTest func()) {
suite.SetupTest()
suite.Run(name, subTest)
suite.TearDownTest()
}
/*************
* Run tests *
*************/
func TestUUIDsWorker(t *testing.T) {
suite.Run(t, new(uuidsWorkerTestSuite))
}
type uuidsWorkerTestCase struct {
Name string
BeforeTest func(suite *uuidsWorkerTestSuite)
AfterTest func(suite *uuidsWorkerTestSuite, response *http.Response)
}
/************************
* Get UUID tests cases *
************************/
var getUuidTestsCases = []*uuidsWorkerTestCase{
{
Name: "Success provider response",
BeforeTest: func(suite *uuidsWorkerTestSuite) {
suite.UuidsProvider.On("GetUuid", "mock_username").Return(&mojang.ProfileInfo{
Id: "0fcc38620f1845f3a54e1b523c1bd1c7",
Name: "mock_username",
}, nil)
},
AfterTest: func(suite *uuidsWorkerTestSuite, response *http.Response) {
suite.Equal(200, response.StatusCode)
suite.Equal("application/json", response.Header.Get("Content-Type"))
body, _ := ioutil.ReadAll(response.Body)
suite.JSONEq(`{
"id": "0fcc38620f1845f3a54e1b523c1bd1c7",
"name": "mock_username"
}`, string(body))
},
},
{
Name: "Receive empty response from UUIDs provider",
BeforeTest: func(suite *uuidsWorkerTestSuite) {
suite.UuidsProvider.On("GetUuid", "mock_username").Return(nil, nil)
},
AfterTest: func(suite *uuidsWorkerTestSuite, response *http.Response) {
suite.Equal(204, response.StatusCode)
body, _ := ioutil.ReadAll(response.Body)
suite.Assert().Empty(body)
},
},
{
Name: "Receive error from UUIDs provider",
BeforeTest: func(suite *uuidsWorkerTestSuite) {
suite.UuidsProvider.On("GetUuid", "mock_username").Return(nil, errors.New("this is an error"))
suite.Logger.On("Warning", "Got non success response: :err", mock.Anything).Times(1)
},
AfterTest: func(suite *uuidsWorkerTestSuite, response *http.Response) {
suite.Equal(500, response.StatusCode)
suite.Equal("application/json", response.Header.Get("Content-Type"))
body, _ := ioutil.ReadAll(response.Body)
suite.JSONEq(`{
"provider": "this is an error"
}`, string(body))
},
},
{
Name: "Receive Too Many Requests from UUIDs provider",
BeforeTest: func(suite *uuidsWorkerTestSuite) {
suite.UuidsProvider.On("GetUuid", "mock_username").Return(nil, &mojang.TooManyRequestsError{})
suite.Logger.On("Warning", "Got 429 Too Many Requests").Times(1)
},
AfterTest: func(suite *uuidsWorkerTestSuite, response *http.Response) {
suite.Equal(429, response.StatusCode)
body, _ := ioutil.ReadAll(response.Body)
suite.Empty(body)
},
},
}
func (suite *uuidsWorkerTestSuite) TestGetUUID() {
for _, testCase := range getUuidTestsCases {
suite.RunSubTest(testCase.Name, func() {
testCase.BeforeTest(suite)
req := httptest.NewRequest("GET", "http://chrly/api/worker/mojang-uuid/mock_username", nil)
w := httptest.NewRecorder()
suite.App.CreateHandler().ServeHTTP(w, req)
testCase.AfterTest(suite, w.Result())
})
}
}

View File

@ -69,7 +69,7 @@ func (c *broadcaster) BroadcastAndRemove(username string, result *broadcastResul
// https://help.mojang.com/customer/portal/articles/928638 // https://help.mojang.com/customer/portal/articles/928638
var allowedUsernamesRegex = regexp.MustCompile(`^[\w_]{3,16}$`) var allowedUsernamesRegex = regexp.MustCompile(`^[\w_]{3,16}$`)
type UuidsProvider interface { type UUIDsProvider interface {
GetUuid(username string) (*mojang.ProfileInfo, error) GetUuid(username string) (*mojang.ProfileInfo, error)
} }
@ -78,7 +78,7 @@ type TexturesProvider interface {
} }
type Provider struct { type Provider struct {
UuidsProvider UUIDsProvider
TexturesProvider TexturesProvider
Storage Storage
Logger wd.Watchdog Logger wd.Watchdog
@ -139,7 +139,7 @@ func (ctx *Provider) getResultAndBroadcast(username string, uuid string) {
func (ctx *Provider) getResult(username string, uuid string) *broadcastResult { func (ctx *Provider) getResult(username string, uuid string) *broadcastResult {
if uuid == "" { if uuid == "" {
profile, err := ctx.UuidsProvider.GetUuid(username) profile, err := ctx.UUIDsProvider.GetUuid(username)
if err != nil { if err != nil {
ctx.handleMojangApiResponseError(err, "usernames") ctx.handleMojangApiResponseError(err, "usernames")
return &broadcastResult{nil, err} return &broadcastResult{nil, err}

View File

@ -158,7 +158,7 @@ func (suite *providerTestSuite) SetupTest() {
suite.Logger = &mocks.WdMock{} suite.Logger = &mocks.WdMock{}
suite.Provider = &Provider{ suite.Provider = &Provider{
UuidsProvider: suite.UuidsProvider, UUIDsProvider: suite.UuidsProvider,
TexturesProvider: suite.TexturesProvider, TexturesProvider: suite.TexturesProvider,
Storage: suite.Storage, Storage: suite.Storage,
Logger: suite.Logger, Logger: suite.Logger,

View File

@ -2,6 +2,7 @@ package mojangtextures
import ( import (
"encoding/json" "encoding/json"
"github.com/elyby/chrly/version"
"io/ioutil" "io/ioutil"
"net/http" "net/http"
. "net/url" . "net/url"
@ -11,7 +12,6 @@ import (
"github.com/mono83/slf/wd" "github.com/mono83/slf/wd"
"github.com/elyby/chrly/api/mojang" "github.com/elyby/chrly/api/mojang"
"github.com/elyby/chrly/bootstrap"
) )
var HttpClient = &http.Client{ var HttpClient = &http.Client{
@ -34,7 +34,7 @@ func (ctx *RemoteApiUuidsProvider) GetUuid(username string) (*mojang.ProfileInfo
request, _ := http.NewRequest("GET", url.String(), nil) request, _ := http.NewRequest("GET", url.String(), nil)
request.Header.Add("Accept", "application/json") request.Header.Add("Accept", "application/json")
// Change default User-Agent to allow specify "Username -> UUID at time" Mojang's api endpoint // Change default User-Agent to allow specify "Username -> UUID at time" Mojang's api endpoint
request.Header.Add("User-Agent", "Chrly/"+bootstrap.GetVersion()) request.Header.Add("User-Agent", "Chrly/"+version.Version())
start := time.Now() start := time.Now()
response, err := HttpClient.Do(request) response, err := HttpClient.Do(request)

View File

@ -1,4 +0,0 @@
#!/bin/sh
mockgen -source=interfaces/repositories.go -destination=interfaces/mock_interfaces/mock_interfaces.go
mockgen -source=interfaces/auth.go -destination=interfaces/mock_interfaces/mock_auth.go

14
version/version.go Normal file
View File

@ -0,0 +1,14 @@
package version
var (
version = ""
commit = ""
)
func Version() string {
return version
}
func Commit() string {
return commit
}