urko
/
ess-etl-go
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: urko:9fe42609bb65047e4a67d716d84d1ef05648b2d5
Branches Tags
urko:master
...
compare: urko:dcab3994b2517a2ea57e82e7548f47f763bc2c41
Branches Tags
urko:master

2 Commits
9fe42609bb ... dcab3994b2

Author SHA1 Message Date
Urko dcab3994b2 feat: rename 2023-04-24 12:03:15 +02:00
Urko d41c7fb4a5 feat: remove hardcoded values 2023-04-24 11:55:33 +02:00
9 changed files with 71 additions and 42 deletions
Show Stats Expand all files Collapse all files

45
benchmark/benchmark_test.go
View File

@ -11,6 +11,7 @@ import (
go_benchmark "gitea.urkob.com/urko/ess-etl-go/benchmark/go" go_benchmark "gitea.urkob.com/urko/ess-etl-go/benchmark/go"
nest_benchmark "gitea.urkob.com/urko/ess-etl-go/benchmark/nest" nest_benchmark "gitea.urkob.com/urko/ess-etl-go/benchmark/nest"
"gitea.urkob.com/urko/ess-etl-go/config"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
@ -24,9 +25,11 @@ func init() {
} }
} }
const bmNumberOfRequests = 100 const (
const numberOfRequestsAtOnce = 1000 bmNumberOfRequests = 100
const seconds = 15 numberOfRequestsAtOnce = 1000
seconds = 15
)
func BenchmarkGo(b *testing.B) { func BenchmarkGo(b *testing.B) {
log.SetFlags(log.Lmicroseconds) log.SetFlags(log.Lmicroseconds)
@ -34,12 +37,16 @@ func BenchmarkGo(b *testing.B) {
f, err := os.OpenFile("./dump/"+logFileName, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0o644) f, err := os.OpenFile("./dump/"+logFileName, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0o644)
require.NoError(b, err) require.NoError(b, err)
cfg := config.NewConfig(".env")
host := cfg.TestGoHost
employeeID := cfg.TestGoEmployee
defer f.Close() defer f.Close()
log.SetOutput(f) log.SetOutput(f)
b.ResetTimer() b.ResetTimer()
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {
start := time.Now() start := time.Now()
assert.NoError(b, go_benchmark.BenchmarkNoLog(bmNumberOfRequests)) assert.NoError(b, go_benchmark.BenchmarkNoLog(host, employeeID, bmNumberOfRequests))
b.Logf("Request took %v", time.Since(start)) b.Logf("Request took %v", time.Since(start))
b.Logf("i %d | b.N %d", i, b.N) b.Logf("i %d | b.N %d", i, b.N)
} }
@ -50,12 +57,16 @@ func BenchmarkNest(b *testing.B) {
f, err := os.OpenFile("./dump/"+logFileName, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0o644) f, err := os.OpenFile("./dump/"+logFileName, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0o644)
require.NoError(b, err) require.NoError(b, err)
cfg := config.NewConfig(".env")
host := cfg.TestGoHost
employeeID := cfg.TestNestEmployee
defer f.Close() defer f.Close()
log.SetOutput(f) log.SetOutput(f)
b.ResetTimer() b.ResetTimer()
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {
start := time.Now() start := time.Now()
assert.NoError(b, nest_benchmark.BenchmarkNoLog(bmNumberOfRequests)) assert.NoError(b, nest_benchmark.BenchmarkNoLog(host, employeeID, bmNumberOfRequests))
b.Logf("Request took %v", time.Since(start)) b.Logf("Request took %v", time.Since(start))
b.Logf("i %d | b.N %d", i, b.N) b.Logf("i %d | b.N %d", i, b.N)
} }
@ -67,6 +78,10 @@ func TestGoRequestsPerSecondFor15s(t *testing.T) {
f, err := os.OpenFile("./dump/"+logFileName, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0o644) f, err := os.OpenFile("./dump/"+logFileName, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0o644)
require.NoError(t, err) require.NoError(t, err)
cfg := config.NewConfig(".env")
host := cfg.TestGoHost
employeeID := cfg.TestGoEmployee
defer f.Close() defer f.Close()
log.SetOutput(f) log.SetOutput(f)
@ -75,7 +90,7 @@ func TestGoRequestsPerSecondFor15s(t *testing.T) {
for i := 0; i < seconds; i++ { for i := 0; i < seconds; i++ {
go func() { go func() {
defer wg.Done() defer wg.Done()
require.NoError(t, go_benchmark.Benchmark(numberOfRequestsAtOnce)) require.NoError(t, go_benchmark.Benchmark(host, employeeID, numberOfRequestsAtOnce))
}() }()
time.Sleep(time.Second) time.Sleep(time.Second)
} }
@ -88,6 +103,10 @@ func TestNestRequestsPerSecondFor15s(t *testing.T) {
f, err := os.OpenFile("./dump/"+logFileName, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0o644) f, err := os.OpenFile("./dump/"+logFileName, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0o644)
require.NoError(t, err) require.NoError(t, err)
cfg := config.NewConfig(".env")
host := cfg.TestGoHost
employeeID := cfg.TestNestEmployee
defer f.Close() defer f.Close()
log.SetOutput(f) log.SetOutput(f)
@ -96,7 +115,7 @@ func TestNestRequestsPerSecondFor15s(t *testing.T) {
for i := 0; i < seconds; i++ { for i := 0; i < seconds; i++ {
go func() { go func() {
defer wg.Done() defer wg.Done()
require.NoError(t, nest_benchmark.Benchmark(numberOfRequestsAtOnce)) require.NoError(t, nest_benchmark.Benchmark(host, employeeID, numberOfRequestsAtOnce))
}() }()
time.Sleep(time.Second) time.Sleep(time.Second)
} }
@ -109,10 +128,14 @@ func TestGoRequests(t *testing.T) {
f, err := os.OpenFile("./dump/"+logFileName, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0o644) f, err := os.OpenFile("./dump/"+logFileName, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0o644)
require.NoError(t, err) require.NoError(t, err)
cfg := config.NewConfig(".env")
host := cfg.TestGoHost
employeeID := cfg.TestGoEmployee
defer f.Close() defer f.Close()
log.SetOutput(f) log.SetOutput(f)
require.NoError(t, go_benchmark.Benchmark(numberOfRequestsAtOnce)) require.NoError(t, go_benchmark.Benchmark(host, employeeID, numberOfRequestsAtOnce))
} }
func TestNestRequests(t *testing.T) { func TestNestRequests(t *testing.T) {
@ -121,8 +144,12 @@ func TestNestRequests(t *testing.T) {
f, err := os.OpenFile("./dump/"+logFileName, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0o644) f, err := os.OpenFile("./dump/"+logFileName, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0o644)
require.NoError(t, err) require.NoError(t, err)
cfg := config.NewConfig(".env")
host := cfg.TestGoHost
employeeID := cfg.TestNestEmployee
defer f.Close() defer f.Close()
log.SetOutput(f) log.SetOutput(f)
require.NoError(t, nest_benchmark.Benchmark(numberOfRequestsAtOnce)) require.NoError(t, nest_benchmark.Benchmark(host, employeeID, numberOfRequestsAtOnce))
} }

13
benchmark/go/go.go
View File

@ -44,19 +44,18 @@ func doRequest(wg *sync.WaitGroup, host, employeeID string, errChan chan error)
return nil return nil
} }
func BenchmarkNoLog(totalRequests int) error { func BenchmarkNoLog(host, employeeID string, totalRequests int) error {
return benchmark(totalRequests, false) return benchmark(host, employeeID, totalRequests, false)
} }
func Benchmark(totalRequests int) error { func Benchmark(host, employeeID string, totalRequests int) error {
return benchmark(totalRequests, true) return benchmark(host, employeeID, totalRequests, true)
} }
func benchmark(totalRequests int, dolog bool) error { func benchmark(host, employeeID string, totalRequests int, dolog bool) error {
cr := crono.New() cr := crono.New()
if dolog { if dolog {
defer cr.Table() defer cr.Table()
} }
host := "http://127.0.0.1:45654"
wg := &sync.WaitGroup{} wg := &sync.WaitGroup{}
errChan := make(chan error, 1) errChan := make(chan error, 1)
@ -67,7 +66,7 @@ func benchmark(totalRequests int, dolog bool) error {
errChan <- nil errChan <- nil
}() }()
for i := 0; i < totalRequests; i++ { for i := 0; i < totalRequests; i++ {
go doRequest(wg, host, "975135", errChan) go doRequest(wg, host, employeeID, errChan)
} }
}(errChan) }(errChan)

13
benchmark/nest/nest.go
View File

@ -70,20 +70,19 @@ func doRequest(wg *sync.WaitGroup, host string, employeeID int, errChan chan err
return nil return nil
} }
func BenchmarkNoLog(totalRequests int) error { func BenchmarkNoLog(host string, employeeNumber, totalRequests int) error {
return benchmark(totalRequests, false) return benchmark(host, employeeNumber, totalRequests, false)
} }
func Benchmark(totalRequests int) error { func Benchmark(host string, employeeNumber, totalRequests int) error {
return benchmark(totalRequests, true) return benchmark(host, employeeNumber, totalRequests, true)
} }
func benchmark(totalRequests int, dolog bool) error { func benchmark(host string, employeeNumber int, totalRequests int, dolog bool) error {
cr := crono.New() cr := crono.New()
if dolog { if dolog {
defer cr.Table() defer cr.Table()
} }
host := "http://127.0.0.1:3458/graphql"
wg := &sync.WaitGroup{} wg := &sync.WaitGroup{}
errChan := make(chan error, 1) errChan := make(chan error, 1)
@ -95,7 +94,7 @@ func benchmark(totalRequests int, dolog bool) error {
}() }()
for i := 0; i < totalRequests; i++ { for i := 0; i < totalRequests; i++ {
wg.Add(1) wg.Add(1)
go doRequest(wg, host, 975153, errChan) go doRequest(wg, host, employeeNumber, errChan)
} }
}(errChan) }(errChan)

2
cmd/etl/main.go
View File

@ -78,7 +78,7 @@ func main() {
log.Fatalln("client.Connect", err) log.Fatalln("client.Connect", err)
} }
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.EmployeeWorkInformationCollection) employeeWICollection := client.Database(cfg.DbName).Collection(cfg.WorkInformationCollection)
if err = employeeWICollection.Drop(ctx); err != nil { if err = employeeWICollection.Drop(ctx); err != nil {
log.Fatalln("employeeWICollection.Drop", err) log.Fatalln("employeeWICollection.Drop", err)
} }

4
cmd/server/main.go
View File

@ -71,13 +71,13 @@ func main() {
log.Fatalln("client.Connect", err) log.Fatalln("client.Connect", err)
} }
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.EmployeeWorkInformationCollection) employeeWICollection := client.Database(cfg.DbName).Collection(cfg.WorkInformationCollection)
professionalRepo := employee_wi.NewRepo(employeeWICollection) professionalRepo := employee_wi.NewRepo(employeeWICollection)
restServer := apihttp. restServer := apihttp.
NewRestServer(cfg, cr). NewRestServer(cfg, cr).
WithEmployeeWIHandler(services.NewEmployeeWIService(ctx, professionalRepo)). WithEmployeeWIHandler(services.NewEmployeeWIService(ctx, professionalRepo)).
WithAMSHander() WithAMSHandler()
cr.MarkAndRestart("dependencies loaded") cr.MarkAndRestart("dependencies loaded")
go func() { go func() {

18
config/config.go
View File

@ -13,13 +13,17 @@ func RootDir() string {
} }
type Config struct { type Config struct {
ApiPort string `required:"true" split_words:"true"` ApiPort string `required:"true" split_words:"true"`
AmsApi string `required:"true" split_words:"true"` AmsApi string `required:"true" split_words:"true"`
AmsApiKey string `required:"true" split_words:"true"` AmsApiKey string `required:"true" split_words:"true"`
DbAddress string `required:"true" split_words:"true"` DbAddress string `required:"true" split_words:"true"`
DbName string `required:"true" split_words:"true"` DbName string `required:"true" split_words:"true"`
EmployeeWorkInformationCollection string `required:"true" split_words:"true"` WorkInformationCollection string `required:"true" split_words:"true"`
EmployeeIdList []string `required:"true" split_words:"true"` EmployeeIdList []string `required:"true" split_words:"true"`
TestGoHost string `required:"true" split_words:"true"`
TestGoEmployee string `required:"true" split_words:"true"`
TestNestHost string `required:"true" split_words:"true"`
TestNestEmployee int `required:"true" split_words:"true"`
} }
func NewConfig(envFile string) *Config { func NewConfig(envFile string) *Config {

2
internal/api/http/server.go
View File

@ -33,7 +33,7 @@ func (s *restServer) WithEmployeeWIHandler(employeeWISrv services.EmployeeWIServ
return s return s
} }
func (s *restServer) WithAMSHander() *restServer { func (s *restServer) WithAMSHandler() *restServer {
s.amsEmployeeWIHdl = handler.NewAMSEmployeeWIHandler(s.cfg.AmsApi, s.cfg.AmsApiKey) s.amsEmployeeWIHdl = handler.NewAMSEmployeeWIHandler(s.cfg.AmsApi, s.cfg.AmsApiKey)
return s return s
} }

14
internal/etl/etl_benchmark_test.go
View File

@ -41,7 +41,7 @@ func BenchmarkETLFanout(b *testing.B) {
require.NoError(b, client.Connect(ctx), "client.Connect") require.NoError(b, client.Connect(ctx), "client.Connect")
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.EmployeeWorkInformationCollection) employeeWICollection := client.Database(cfg.DbName).Collection(cfg.WorkInformationCollection)
require.NoError(b, employeeWICollection.Drop(ctx), "employeeWICollection.Drop") require.NoError(b, employeeWICollection.Drop(ctx), "employeeWICollection.Drop")
repo := employee_wi.NewRepo(employeeWICollection) repo := employee_wi.NewRepo(employeeWICollection)
@ -79,7 +79,7 @@ func BenchmarkETLFanout2(b *testing.B) {
require.NoError(b, client.Connect(ctx), "client.Connect") require.NoError(b, client.Connect(ctx), "client.Connect")
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.EmployeeWorkInformationCollection) employeeWICollection := client.Database(cfg.DbName).Collection(cfg.WorkInformationCollection)
require.NoError(b, employeeWICollection.Drop(ctx), "employeeWICollection.Drop") require.NoError(b, employeeWICollection.Drop(ctx), "employeeWICollection.Drop")
repo := employee_wi.NewRepo(employeeWICollection) repo := employee_wi.NewRepo(employeeWICollection)
@ -115,7 +115,7 @@ func BenchmarkETLMain(b *testing.B) {
require.NoError(b, client.Connect(ctx), "client.Connect") require.NoError(b, client.Connect(ctx), "client.Connect")
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.EmployeeWorkInformationCollection) employeeWICollection := client.Database(cfg.DbName).Collection(cfg.WorkInformationCollection)
require.NoError(b, employeeWICollection.Drop(ctx), "employeeWICollection.Drop") require.NoError(b, employeeWICollection.Drop(ctx), "employeeWICollection.Drop")
repo := employee_wi.NewRepo(employeeWICollection) repo := employee_wi.NewRepo(employeeWICollection)
@ -154,7 +154,7 @@ func TestETLFanout(t *testing.T) {
require.NoError(t, client.Connect(ctx), "client.Connect") require.NoError(t, client.Connect(ctx), "client.Connect")
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.EmployeeWorkInformationCollection) employeeWICollection := client.Database(cfg.DbName).Collection(cfg.WorkInformationCollection)
require.NoError(t, employeeWICollection.Drop(ctx), "employeeWICollection.Drop") require.NoError(t, employeeWICollection.Drop(ctx), "employeeWICollection.Drop")
repo := employee_wi.NewRepo(employeeWICollection) repo := employee_wi.NewRepo(employeeWICollection)
@ -191,7 +191,7 @@ func TestETLFanOut2(t *testing.T) {
require.NoError(t, client.Connect(ctx), "client.Connect") require.NoError(t, client.Connect(ctx), "client.Connect")
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.EmployeeWorkInformationCollection) employeeWICollection := client.Database(cfg.DbName).Collection(cfg.WorkInformationCollection)
require.NoError(t, employeeWICollection.Drop(ctx), "employeeWICollection.Drop") require.NoError(t, employeeWICollection.Drop(ctx), "employeeWICollection.Drop")
repo := employee_wi.NewRepo(employeeWICollection) repo := employee_wi.NewRepo(employeeWICollection)
@ -227,7 +227,7 @@ func TestETLMain(t *testing.T) {
require.NoError(t, client.Connect(ctx), "client.Connect") require.NoError(t, client.Connect(ctx), "client.Connect")
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.EmployeeWorkInformationCollection) employeeWICollection := client.Database(cfg.DbName).Collection(cfg.WorkInformationCollection)
require.NoError(t, employeeWICollection.Drop(ctx), "employeeWICollection.Drop") require.NoError(t, employeeWICollection.Drop(ctx), "employeeWICollection.Drop")
repo := employee_wi.NewRepo(employeeWICollection) repo := employee_wi.NewRepo(employeeWICollection)
@ -251,7 +251,7 @@ func TestLoad(t *testing.T) {
require.NoError(t, client.Connect(ctx), "client.Connect") require.NoError(t, client.Connect(ctx), "client.Connect")
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.EmployeeWorkInformationCollection) employeeWICollection := client.Database(cfg.DbName).Collection(cfg.WorkInformationCollection)
if err = employeeWICollection.Drop(ctx); err != nil { if err = employeeWICollection.Drop(ctx); err != nil {
log.Fatalln("employeeWICollection.Drop", err) log.Fatalln("employeeWICollection.Drop", err)
} }

2
internal/services/employee_wi.go
View File

@ -44,7 +44,7 @@ func (p *EmployeeWIService) GetByAll(ctx context.Context, id string) (*domain.Em
func (p *EmployeeWIService) InsertMany(ctx context.Context, ei []domain.EmployeeWorkInformation) error { func (p *EmployeeWIService) InsertMany(ctx context.Context, ei []domain.EmployeeWorkInformation) error {
err := p.repo.InsertMany(ctx, ei) err := p.repo.InsertMany(ctx, ei)
if err != nil { if err != nil {
return fmt.Errorf("repo.AddAppointmentTo: %s", err) return fmt.Errorf("repo.InsertMany: %s", err)
} }
return nil return nil
} }