progress [skip ci]

This commit is contained in:
ErickSkrauch 2024-02-19 13:54:12 +01:00
parent feb8e32069
commit f037fb11e1
No known key found for this signature in database
GPG Key ID: 669339FCBB30EE0E
19 changed files with 366 additions and 102 deletions

4
go.mod
View File

@ -4,7 +4,6 @@ go 1.21
// Main dependencies
require (
github.com/SentimensRG/ctx v0.0.0-20180729130232-0bfd988c655d
github.com/agoda-com/opentelemetry-go/otelslog v0.1.1
github.com/agoda-com/opentelemetry-logs-go v0.4.3
github.com/brunomvsouza/singleflight v0.4.0
@ -21,6 +20,7 @@ require (
github.com/spf13/viper v1.18.1
github.com/valyala/fastjson v1.6.4
go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.48.0
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.48.0
go.opentelemetry.io/contrib/instrumentation/runtime v0.48.0
go.opentelemetry.io/otel v1.23.1
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.23.1
@ -29,6 +29,7 @@ require (
go.opentelemetry.io/otel/sdk v1.23.1
go.opentelemetry.io/otel/sdk/metric v1.23.1
go.opentelemetry.io/otel/trace v1.23.1
go.uber.org/multierr v1.11.0
)
// Dev dependencies
@ -70,7 +71,6 @@ require (
github.com/tilinna/clock v1.0.2 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.23.1 // indirect
go.opentelemetry.io/proto/otlp v1.1.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/crypto v0.19.0 // indirect
golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb // indirect
golang.org/x/net v0.21.0 // indirect

37
go.sum
View File

@ -1,5 +1,3 @@
github.com/SentimensRG/ctx v0.0.0-20180729130232-0bfd988c655d h1:CbB/Ef3TyBvSSJx2HDSUiw49ONTpaX6BGiI0jJEX6b8=
github.com/SentimensRG/ctx v0.0.0-20180729130232-0bfd988c655d/go.mod h1:cfn0Ycx1ASzCkl8+04zI4hrclf9YQ1QfncxzFiNtQLo=
github.com/agoda-com/opentelemetry-go/otelslog v0.1.1 h1:6nV8PZCzySHuh9kP/HZ2OJqGucwQiM+yZRugKDvtzj4=
github.com/agoda-com/opentelemetry-go/otelslog v0.1.1/go.mod h1:CSc0veIcY/HsIfH7l5PGtIpRvBttk09QUQlweVkD2PI=
github.com/agoda-com/opentelemetry-logs-go v0.4.3 h1:dYAx/q9di+/Pv6HuGq59DFIOjqKT0LTy3PYTIz8ccq8=
@ -34,9 +32,6 @@ github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ=
github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
@ -51,8 +46,6 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS
github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
@ -75,9 +68,6 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4=
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed h1:036IscGBfJsFIgJQzlui7nK1Ncm0tp2ktmPj8xO4N/0=
github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k=
github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
github.com/mediocregopher/radix/v4 v4.1.4 h1:Uze6DEbEAvL+VHXUEu/EDBTkUk5CLct5h3nVSGpc6Ts=
@ -93,9 +83,6 @@ github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdU
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b h1:0LFwY6Q3gMACTjAbMZBjXAqTOzOwFaj2Ld6cjeQ7Rig=
github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
@ -103,11 +90,6 @@ github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6ke
github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4=
github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE=
github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ=
github.com/shirou/gopsutil/v3 v3.24.1 h1:R3t6ondCEvmARp3wxODhXMTLC/klMa87h2PHUw5m7QI=
github.com/shirou/gopsutil/v3 v3.24.1/go.mod h1:UU7a2MSBQa+kW1uuDq8DeEBS8kmrnQwsv2b5O513rwU=
github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM=
github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ=
github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k=
github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8=
@ -134,21 +116,12 @@ github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
github.com/tilinna/clock v1.0.2 h1:6BO2tyAC9JbPExKH/z9zl44FLu1lImh3nDNKA0kgrkI=
github.com/tilinna/clock v1.0.2/go.mod h1:ZsP7BcY7sEEz7ktc0IVy8Us6boDrK8VradlKRUGfOao=
github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI=
github.com/tklauser/go-sysconf v0.3.13 h1:GBUpcahXSpR2xN01jhkNAbTLRk2Yzgggk8IM08lq3r4=
github.com/tklauser/go-sysconf v0.3.13/go.mod h1:zwleP4Q4OehZHGn4CYZDipCgg9usW5IJePewFCGVEa0=
github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY=
github.com/tklauser/numcpus v0.7.0 h1:yjuerZP127QG9m5Zh/mSO4wqurYil27tHrqwRoRjpr4=
github.com/tklauser/numcpus v0.7.0/go.mod h1:bb6dMVcj8A42tSE7i32fsIUCbQNllK5iDguyOZRUzAY=
github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXVQ=
github.com/valyala/fastjson v1.6.4/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY=
github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.48.0 h1:7rkdNoXgScpSUIqBch/VOB24fk9g0wl3Tr5WPtshi9o=
go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.48.0/go.mod h1:U3t9uswWhDzieXHMNWP6zk87J4HNondiibKMdNLpnMk=
go.opentelemetry.io/contrib/instrumentation/host v0.48.0 h1:eVDLR/hletJcctz4rSWwb3QVzRnEQKuTVi6qAm7fsWs=
go.opentelemetry.io/contrib/instrumentation/host v0.48.0/go.mod h1:dWSGvPpaGKwBh/dpLJs6pczyOc82hBnxa1YTQZCic0I=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.48.0 h1:doUP+ExOpH3spVTLS0FcWGLnQrPct/hD/bCPbDRUEAU=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.48.0/go.mod h1:rdENBZMT2OE6Ne/KLwpiXudnAsbdrdBaqBvTN8M8BgA=
go.opentelemetry.io/contrib/instrumentation/runtime v0.48.0 h1:dJlCKeq+zmO5Og4kgxqPvvJrzuD/mygs1g/NYM9dAsU=
go.opentelemetry.io/contrib/instrumentation/runtime v0.48.0/go.mod h1:p+hpBCpLHpuUrR0lHgnHbUnbCBll1IhrcMIlycC+xYs=
go.opentelemetry.io/otel v1.23.1 h1:Za4UzOqJYS+MUczKI320AtqZHZb7EqxO00jAHE0jmQY=
@ -181,12 +154,6 @@ golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE=
golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=

View File

@ -8,6 +8,7 @@ func New() (*di.Container, error) {
contextDiOptions,
dbDiOptions,
handlersDiOptions,
httpClientDiOptions,
loggerDiOptions,
mojangDiOptions,
profilesDiOptions,

18
internal/di/httpClient.go Normal file
View File

@ -0,0 +1,18 @@
package di
import (
"net/http"
"github.com/defval/di"
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
)
var httpClientDiOptions = di.Options(
di.Provide(newHttpClient),
)
func newHttpClient() *http.Client {
return &http.Client{
Transport: otelhttp.NewTransport(http.DefaultTransport),
}
}

View File

@ -21,7 +21,7 @@ var mojangDiOptions = di.Options(
di.Provide(newMojangSignedTexturesProvider),
)
func newMojangApi(config *viper.Viper) (*mojang.MojangApi, error) {
func newMojangApi(config *viper.Viper, httpClient *http.Client) (*mojang.MojangApi, error) {
batchUuidsUrl := config.GetString("mojang.batch_uuids_url")
if batchUuidsUrl != "" {
if _, err := url.ParseRequestURI(batchUuidsUrl); err != nil {
@ -36,8 +36,6 @@ func newMojangApi(config *viper.Viper) (*mojang.MojangApi, error) {
}
}
httpClient := &http.Client{} // TODO: extract to the singleton dependency
return mojang.NewMojangApi(httpClient, batchUuidsUrl, profileUrl), nil
}
@ -62,21 +60,18 @@ func newMojangTexturesProviderFactory(
func newMojangTexturesProvider(
uuidsProvider mojang.UuidsProvider,
texturesProvider mojang.TexturesProvider,
) *mojang.MojangTexturesProvider {
return &mojang.MojangTexturesProvider{
UuidsProvider: uuidsProvider,
TexturesProvider: texturesProvider,
}
) (*mojang.MojangTexturesProvider, error) {
return mojang.NewMojangTexturesProvider(
uuidsProvider,
texturesProvider,
)
}
func newMojangTexturesUuidsProviderFactory(
batchProvider *mojang.BatchUuidsProvider,
uuidsStorage mojang.MojangUuidsStorage,
) mojang.UuidsProvider {
return &mojang.UuidsProviderWithCache{
Provider: batchProvider,
Storage: uuidsStorage,
}
) (mojang.UuidsProvider, error) {
return mojang.NewUuidsProviderWithCache(batchProvider, uuidsStorage)
}
func newMojangTexturesBatchUUIDsProvider(
@ -89,20 +84,19 @@ func newMojangTexturesBatchUUIDsProvider(
// TODO: healthcheck is broken
uuidsProvider := mojang.NewBatchUuidsProvider(
return mojang.NewBatchUuidsProvider(
mojangApi.UsernamesToUuids,
config.GetInt("queue.batch_size"),
config.GetDuration("queue.loop_delay"),
config.GetString("queue.strategy") == "full-bus",
)
return uuidsProvider, nil
}
func newMojangSignedTexturesProvider(mojangApi *mojang.MojangApi) mojang.TexturesProvider {
return mojang.NewTexturesProviderWithInMemoryCache(
&mojang.MojangApiTexturesProvider{
MojangApiTexturesEndpoint: mojangApi.UuidToTextures,
},
)
func newMojangSignedTexturesProvider(mojangApi *mojang.MojangApi) (mojang.TexturesProvider, error) {
provider, err := mojang.NewMojangApiTexturesProvider(mojangApi.UuidToTextures)
if err != nil {
return nil, err
}
return mojang.NewTexturesProviderWithInMemoryCache(provider)
}

View File

@ -19,9 +19,9 @@ func newProfilesManager(r profiles.ProfilesRepository) *profiles.Manager {
func newProfilesProvider(
finder profiles.ProfilesFinder,
mojangProfilesProvider profiles.MojangProfilesProvider,
) *profiles.Provider {
return &profiles.Provider{
ProfilesFinder: finder,
MojangProfilesProvider: mojangProfilesProvider,
}
) (*profiles.Provider, error) {
return profiles.NewProvider(
finder,
mojangProfilesProvider,
)
}

View File

@ -6,7 +6,9 @@ import (
"sync"
"time"
"github.com/SentimensRG/ctx/mergectx"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/metric"
"go.uber.org/multierr"
"ely.by/chrly/internal/utils"
)
@ -23,6 +25,7 @@ type BatchUuidsProvider struct {
fireChan chan any
stopChan chan any
onFirstCall sync.Once
metrics *batchUuidsProviderMetrics
}
func NewBatchUuidsProvider(
@ -30,22 +33,31 @@ func NewBatchUuidsProvider(
batchSize int,
awaitDelay time.Duration,
fireOnFull bool,
) *BatchUuidsProvider {
) (*BatchUuidsProvider, error) {
queue := utils.NewQueue[*job]()
metrics, err := newBatchUuidsProviderMetrics(otel.GetMeterProvider().Meter(ScopeName), queue)
if err != nil {
return nil, err
}
return &BatchUuidsProvider{
UsernamesToUuidsEndpoint: endpoint,
stopChan: make(chan any),
batch: batchSize,
delay: awaitDelay,
fireOnFull: fireOnFull,
queue: utils.NewQueue[*job](),
queue: queue,
fireChan: make(chan any),
}
metrics: metrics,
}, nil
}
type job struct {
Username string
Ctx context.Context
ResultChan chan<- *jobResult
Username string
Ctx context.Context
QueuingTime time.Time
ResultChan chan<- *jobResult
}
type jobResult struct {
@ -55,7 +67,7 @@ type jobResult struct {
func (p *BatchUuidsProvider) GetUuid(ctx context.Context, username string) (*ProfileInfo, error) {
resultChan := make(chan *jobResult)
n := p.queue.Enqueue(&job{username, ctx, resultChan})
n := p.queue.Enqueue(&job{username, ctx, time.Now(), resultChan})
if p.fireOnFull && n%p.batch == 0 {
p.fireChan <- struct{}{}
}
@ -92,11 +104,14 @@ func (p *BatchUuidsProvider) startQueue() {
}
func (p *BatchUuidsProvider) fireRequest() {
// Since this method is an aggregator, it uses its own context to manage its lifetime
reqCtx := context.Background()
jobs := make([]*job, 0, p.batch)
n := p.batch
for {
foundJobs, left := p.queue.Dequeue(n)
for i := range foundJobs {
p.metrics.QueueTime.Record(reqCtx, float64(time.Since(foundJobs[i].QueuingTime)))
if foundJobs[i].Ctx.Err() != nil {
// If the job context has already ended, its result will be returned in the GetUuid method
close(foundJobs[i].ResultChan)
@ -119,14 +134,14 @@ func (p *BatchUuidsProvider) fireRequest() {
return
}
ctx := context.Background()
usernames := make([]string, len(jobs))
for i, job := range jobs {
usernames[i] = job.Username
ctx = mergectx.Join(ctx, job.Ctx)
}
profiles, err := p.UsernamesToUuidsEndpoint(ctx, usernames)
p.metrics.BatchSize.Record(reqCtx, int64(len(usernames)))
profiles, err := p.UsernamesToUuidsEndpoint(reqCtx, usernames)
for _, job := range jobs {
response := &jobResult{}
if err == nil {
@ -145,3 +160,39 @@ func (p *BatchUuidsProvider) fireRequest() {
close(job.ResultChan)
}
}
func newBatchUuidsProviderMetrics(meter metric.Meter, queue *utils.Queue[*job]) (*batchUuidsProviderMetrics, error) {
m := &batchUuidsProviderMetrics{}
var errors, err error
m.QueueLength, err = meter.Int64ObservableGauge(
"queue.length", // TODO: look for better naming
metric.WithDescription(""), // TODO: description
metric.WithInt64Callback(func(_ context.Context, o metric.Int64Observer) error {
o.Observe(int64(queue.Len()))
return nil
}),
)
errors = multierr.Append(errors, err)
m.QueueTime, err = meter.Float64Histogram(
"queue.duration",
metric.WithDescription(""), // TODO: description
metric.WithUnit("ms"),
)
m.BatchSize, err = meter.Int64Histogram(
"batch.size",
metric.WithDescription(""), // TODO: write description
metric.WithUnit("1"),
)
errors = multierr.Append(errors, err)
return m, errors
}
type batchUuidsProviderMetrics struct {
QueueLength metric.Int64ObservableGauge
QueueTime metric.Float64Histogram
BatchSize metric.Int64Histogram
}

View File

@ -41,7 +41,7 @@ type batchUuidsProviderTestSuite struct {
func (s *batchUuidsProviderTestSuite) SetupTest() {
s.MojangApi = &mojangUsernamesToUuidsRequestMock{}
s.Provider = NewBatchUuidsProvider(
s.Provider, _ = NewBatchUuidsProvider(
s.MojangApi.UsernamesToUuids,
3,
awaitDelay,

View File

@ -77,6 +77,7 @@ func (c *MojangApi) UsernamesToUuids(ctx context.Context, usernames []string) ([
// Obtains textures information for provided uuid
// See https://wiki.vg/Mojang_API#UUID_-.3E_Profile_.2B_Skin.2FCape
func (c *MojangApi) UuidToTextures(ctx context.Context, uuid string, signed bool) (*ProfileResponse, error) {
// TODO: normalize request url for tracing
normalizedUuid := strings.ReplaceAll(uuid, "-", "")
url := c.profileUrl + normalizedUuid
if signed {

View File

@ -7,8 +7,13 @@ import (
"strings"
"github.com/brunomvsouza/singleflight"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/metric"
"go.uber.org/multierr"
)
const ScopeName = "ely.by/chrly/internal/mojang"
var InvalidUsername = errors.New("the username passed doesn't meet Mojang's requirements")
// https://help.minecraft.net/hc/en-us/articles/4408950195341#h_01GE5JX1Z0CZ833A7S54Y195KV
@ -22,11 +27,28 @@ type TexturesProvider interface {
GetTextures(ctx context.Context, uuid string) (*ProfileResponse, error)
}
func NewMojangTexturesProvider(
uuidsProvider UuidsProvider,
texturesProvider TexturesProvider,
) (*MojangTexturesProvider, error) {
meter, err := newProviderMetrics(otel.GetMeterProvider().Meter(ScopeName))
if err != nil {
return nil, err
}
return &MojangTexturesProvider{
UuidsProvider: uuidsProvider,
TexturesProvider: texturesProvider,
metrics: meter,
}, nil
}
type MojangTexturesProvider struct {
UuidsProvider
TexturesProvider
group singleflight.Group[string, *ProfileResponse]
metrics *providerMetrics
group singleflight.Group[string, *ProfileResponse]
}
func (p *MojangTexturesProvider) GetForUsername(ctx context.Context, username string) (*ProfileResponse, error) {
@ -36,7 +58,7 @@ func (p *MojangTexturesProvider) GetForUsername(ctx context.Context, username st
username = strings.ToLower(username)
result, err, _ := p.group.Do(username, func() (*ProfileResponse, error) {
result, err, shared := p.group.Do(username, func() (*ProfileResponse, error) {
profile, err := p.UuidsProvider.GetUuid(ctx, username)
if err != nil {
return nil, err
@ -49,12 +71,69 @@ func (p *MojangTexturesProvider) GetForUsername(ctx context.Context, username st
return p.TexturesProvider.GetTextures(ctx, profile.Id)
})
p.recordMetrics(ctx, shared, result, err)
return result, err
}
func (p *MojangTexturesProvider) recordMetrics(ctx context.Context, shared bool, result *ProfileResponse, err error) {
if shared {
p.metrics.Shared.Add(ctx, 1)
}
if err != nil {
p.metrics.Failed.Add(ctx, 1)
return
}
if result != nil {
p.metrics.Found.Add(ctx, 1)
} else {
p.metrics.Missed.Add(ctx, 1)
}
}
type NilProvider struct {
}
func (*NilProvider) GetForUsername(ctx context.Context, username string) (*ProfileResponse, error) {
return nil, nil
}
func newProviderMetrics(meter metric.Meter) (*providerMetrics, error) {
m := &providerMetrics{}
var errors, err error
m.Found, err = meter.Int64Counter(
"results.found",
metric.WithDescription(""), // TODO: description
)
errors = multierr.Append(errors, err)
m.Missed, err = meter.Int64Counter(
"results.missed",
metric.WithDescription(""), // TODO: description
)
errors = multierr.Append(errors, err)
m.Failed, err = meter.Int64Counter(
"results.failed",
metric.WithDescription(""), // TODO: description
)
errors = multierr.Append(errors, err)
m.Shared, err = meter.Int64Counter(
"singleflight.shared",
metric.WithDescription(""), // TODO: description
)
errors = multierr.Append(errors, err)
return m, errors
}
type providerMetrics struct {
Found metric.Int64Counter
Missed metric.Int64Counter
Failed metric.Int64Counter
Shared metric.Int64Counter
}

View File

@ -51,10 +51,10 @@ func (s *providerTestSuite) SetupTest() {
s.UuidsProvider = &mockUuidsProvider{}
s.TexturesProvider = &TexturesProviderMock{}
s.Provider = &MojangTexturesProvider{
UuidsProvider: s.UuidsProvider,
TexturesProvider: s.TexturesProvider,
}
s.Provider, _ = NewMojangTexturesProvider(
s.UuidsProvider,
s.TexturesProvider,
)
}
func (s *providerTestSuite) TearDownTest() {

View File

@ -6,13 +6,33 @@ import (
"time"
"github.com/jellydator/ttlcache/v3"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/metric"
"go.uber.org/multierr"
)
type MojangApiTexturesProviderFunc func(ctx context.Context, uuid string, signed bool) (*ProfileResponse, error)
func NewMojangApiTexturesProvider(endpoint MojangApiTexturesProviderFunc) (*MojangApiTexturesProvider, error) {
metrics, err := newMojangApiTexturesProviderMetrics(otel.GetMeterProvider().Meter(ScopeName))
if err != nil {
return nil, err
}
return &MojangApiTexturesProvider{
MojangApiTexturesEndpoint: endpoint,
metrics: metrics,
}, nil
}
type MojangApiTexturesProvider struct {
MojangApiTexturesEndpoint func(ctx context.Context, uuid string, signed bool) (*ProfileResponse, error)
MojangApiTexturesEndpoint MojangApiTexturesProviderFunc
metrics *mojangApiTexturesProviderMetrics
}
func (p *MojangApiTexturesProvider) GetTextures(ctx context.Context, uuid string) (*ProfileResponse, error) {
p.metrics.Requests.Add(ctx, 1)
return p.MojangApiTexturesEndpoint(ctx, uuid, true)
}
@ -22,27 +42,35 @@ type TexturesProviderWithInMemoryCache struct {
provider TexturesProvider
once sync.Once
cache *ttlcache.Cache[string, *ProfileResponse]
metrics *texturesProviderWithInMemoryCacheMetrics
}
func NewTexturesProviderWithInMemoryCache(provider TexturesProvider) *TexturesProviderWithInMemoryCache {
storage := &TexturesProviderWithInMemoryCache{
func NewTexturesProviderWithInMemoryCache(provider TexturesProvider) (*TexturesProviderWithInMemoryCache, error) {
metrics, err := newTexturesProviderWithInMemoryCacheMetrics(otel.GetMeterProvider().Meter(ScopeName))
if err != nil {
return nil, err
}
return &TexturesProviderWithInMemoryCache{
provider: provider,
cache: ttlcache.New[string, *ProfileResponse](
ttlcache.WithDisableTouchOnHit[string, *ProfileResponse](),
// I'm aware of ttlcache.WithLoader(), but it doesn't allow to return an error
),
}
return storage
metrics: metrics,
}, nil
}
func (s *TexturesProviderWithInMemoryCache) GetTextures(ctx context.Context, uuid string) (*ProfileResponse, error) {
item := s.cache.Get(uuid)
// Don't check item.IsExpired() since Get function is already did this check
if item != nil {
s.metrics.Hits.Add(ctx, 1)
return item.Value(), nil
}
s.metrics.Misses.Add(ctx, 1)
result, err := s.provider.GetTextures(ctx, uuid)
if err != nil {
return nil, err
@ -66,3 +94,47 @@ func (s *TexturesProviderWithInMemoryCache) startGcOnce() {
go s.cache.Start()
})
}
func newMojangApiTexturesProviderMetrics(meter metric.Meter) (*mojangApiTexturesProviderMetrics, error) {
m := &mojangApiTexturesProviderMetrics{}
var errors, err error
m.Requests, err = meter.Int64Counter(
"textures.requests",
metric.WithDescription(""), // TODO: write description
metric.WithUnit("1"),
)
errors = multierr.Append(errors, err)
return m, errors
}
type mojangApiTexturesProviderMetrics struct {
Requests metric.Int64Counter
}
func newTexturesProviderWithInMemoryCacheMetrics(meter metric.Meter) (*texturesProviderWithInMemoryCacheMetrics, error) {
m := &texturesProviderWithInMemoryCacheMetrics{}
var errors, err error
m.Hits, err = meter.Int64Counter(
"textures.cache.hit",
metric.WithDescription(""), // TODO: write description
metric.WithUnit("1"),
)
errors = multierr.Append(errors, err)
m.Misses, err = meter.Int64Counter(
"textures.cache.miss",
metric.WithDescription(""), // TODO: write description
metric.WithUnit("1"),
)
errors = multierr.Append(errors, err)
return m, errors
}
type texturesProviderWithInMemoryCacheMetrics struct {
Hits metric.Int64Counter
Misses metric.Int64Counter
}

View File

@ -53,9 +53,7 @@ type MojangApiTexturesProviderSuite struct {
func (s *MojangApiTexturesProviderSuite) SetupTest() {
s.MojangApi = &MojangUuidToTexturesRequestMock{}
s.Provider = &MojangApiTexturesProvider{
MojangApiTexturesEndpoint: s.MojangApi.UuidToTextures,
}
s.Provider, _ = NewMojangApiTexturesProvider(s.MojangApi.UuidToTextures)
}
func (s *MojangApiTexturesProviderSuite) TearDownTest() {
@ -95,7 +93,7 @@ type TexturesProviderWithInMemoryCacheSuite struct {
func (s *TexturesProviderWithInMemoryCacheSuite) SetupTest() {
s.Original = &TexturesProviderMock{}
s.Provider = NewTexturesProviderWithInMemoryCache(s.Original)
s.Provider, _ = NewTexturesProviderWithInMemoryCache(s.Original)
}
func (s *TexturesProviderWithInMemoryCacheSuite) TearDownTest() {

View File

@ -1,6 +1,12 @@
package mojang
import "context"
import (
"context"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/metric"
"go.uber.org/multierr"
)
type MojangUuidsStorage interface {
// The second argument must be returned as a incoming username in case,
@ -10,13 +16,32 @@ type MojangUuidsStorage interface {
StoreMojangUuid(ctx context.Context, username string, uuid string) error
}
func NewUuidsProviderWithCache(o UuidsProvider, s MojangUuidsStorage) (*UuidsProviderWithCache, error) {
metrics, err := newUuidsProviderWithCacheMetrics(otel.GetMeterProvider().Meter(ScopeName))
if err != nil {
return nil, err
}
return &UuidsProviderWithCache{
Provider: o,
Storage: s,
metrics: metrics,
}, nil
}
type UuidsProviderWithCache struct {
Provider UuidsProvider
Storage MojangUuidsStorage
metrics *uuidsProviderWithCacheMetrics
}
func (p *UuidsProviderWithCache) GetUuid(ctx context.Context, username string) (*ProfileInfo, error) {
uuid, foundUsername, err := p.Storage.GetUuidForMojangUsername(ctx, username)
var uuid, foundUsername string
var err error
defer p.recordMetrics(ctx, uuid, foundUsername, err)
uuid, foundUsername, err = p.Storage.GetUuidForMojangUsername(ctx, username)
if err != nil {
return nil, err
}
@ -45,3 +70,41 @@ func (p *UuidsProviderWithCache) GetUuid(ctx context.Context, username string) (
return profile, nil
}
func (p *UuidsProviderWithCache) recordMetrics(ctx context.Context, uuid string, username string, err error) {
if err != nil {
return
}
if username != "" {
p.metrics.Hits.Add(ctx, 1)
} else {
p.metrics.Misses.Add(ctx, 1)
}
}
func newUuidsProviderWithCacheMetrics(meter metric.Meter) (*uuidsProviderWithCacheMetrics, error) {
m := &uuidsProviderWithCacheMetrics{}
var errors, err error
m.Hits, err = meter.Int64Counter(
"uuids.cache.hit",
metric.WithDescription(""), // TODO: write description
metric.WithUnit("1"),
)
errors = multierr.Append(errors, err)
m.Misses, err = meter.Int64Counter(
"uuids.cache.miss",
metric.WithDescription(""), // TODO: write description
metric.WithUnit("1"),
)
errors = multierr.Append(errors, err)
return m, errors
}
type uuidsProviderWithCacheMetrics struct {
Hits metric.Int64Counter
Misses metric.Int64Counter
}

View File

@ -50,10 +50,7 @@ type UuidsProviderWithCacheSuite struct {
func (s *UuidsProviderWithCacheSuite) SetupTest() {
s.Original = &UuidsProviderMock{}
s.Storage = &MojangUuidsStorageMock{}
s.Provider = &UuidsProviderWithCache{
Provider: s.Original,
Storage: s.Storage,
}
s.Provider, _ = NewUuidsProviderWithCache(s.Original, s.Storage)
}
func (s *UuidsProviderWithCacheSuite) TearDownTest() {

View File

@ -16,6 +16,13 @@ type MojangProfilesProvider interface {
GetForUsername(ctx context.Context, username string) (*mojang.ProfileResponse, error)
}
func NewProvider(pf ProfilesFinder, mpf MojangProfilesProvider) (*Provider, error) {
return &Provider{
ProfilesFinder: pf,
MojangProfilesProvider: mpf,
}, nil
}
type Provider struct {
ProfilesFinder
MojangProfilesProvider

View File

@ -54,10 +54,10 @@ type CombinedProfilesProviderSuite struct {
func (t *CombinedProfilesProviderSuite) SetupSubTest() {
t.ProfilesFinder = &ProfilesFinderMock{}
t.MojangProfilesProvider = &MojangProfilesProviderMock{}
t.Provider = &Provider{
ProfilesFinder: t.ProfilesFinder,
MojangProfilesProvider: t.MojangProfilesProvider,
}
t.Provider, _ = NewProvider(
t.ProfilesFinder,
t.MojangProfilesProvider,
)
}
func (t *CombinedProfilesProviderSuite) TearDownSubTest() {

View File

@ -38,3 +38,10 @@ func (s *Queue[T]) Dequeue(n int) ([]T, int) {
return items, l - n
}
func (s *Queue[T]) Len() int {
s.lock.Lock()
defer s.lock.Unlock()
return len(s.items)
}

View File

@ -35,4 +35,13 @@ func TestQueue(t *testing.T) {
require.Equal(t, "username4", items[1])
require.Equal(t, "username5", items[2])
})
t.Run("Len", func(t *testing.T) {
s := NewQueue[string]()
s.Enqueue("username1")
s.Enqueue("username2")
s.Enqueue("username3")
require.Equal(t, 3, s.Len())
})
}