urko
/
ess-etl-go
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: urko:9fe42609bb65047e4a67d716d84d1ef05648b2d5
Branches Tags
urko:master
...
compare: urko:dcab3994b2517a2ea57e82e7548f47f763bc2c41
Branches Tags
urko:master

2 Commits
9fe42609bb ... dcab3994b2

Author SHA1 Message Date
Urko dcab3994b2 feat: rename 2023-04-24 12:03:15 +02:00
Urko d41c7fb4a5 feat: remove hardcoded values 2023-04-24 11:55:33 +02:00
9 changed files with 71 additions and 42 deletions
Show Stats Expand all files Collapse all files

45
benchmark/benchmark_test.go
View File

@ -11,6 +11,7 @@ import (
go_benchmark "gitea.urkob.com/urko/ess-etl-go/benchmark/go"
nest_benchmark "gitea.urkob.com/urko/ess-etl-go/benchmark/nest"
"gitea.urkob.com/urko/ess-etl-go/config"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@ -24,9 +25,11 @@ func init() {
}
}
const bmNumberOfRequests = 100
const numberOfRequestsAtOnce = 1000
const seconds = 15
const (
bmNumberOfRequests = 100
numberOfRequestsAtOnce = 1000
seconds = 15
)
func BenchmarkGo(b *testing.B) {
log.SetFlags(log.Lmicroseconds)
@ -34,12 +37,16 @@ func BenchmarkGo(b *testing.B) {
f, err := os.OpenFile("./dump/"+logFileName, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0o644)
require.NoError(b, err)
cfg := config.NewConfig(".env")
host := cfg.TestGoHost
employeeID := cfg.TestGoEmployee
defer f.Close()
log.SetOutput(f)
b.ResetTimer()
for i := 0; i < b.N; i++ {
start := time.Now()
assert.NoError(b, go_benchmark.BenchmarkNoLog(bmNumberOfRequests))
assert.NoError(b, go_benchmark.BenchmarkNoLog(host, employeeID, bmNumberOfRequests))
b.Logf("Request took %v", time.Since(start))
b.Logf("i %d | b.N %d", i, b.N)
}
@ -50,12 +57,16 @@ func BenchmarkNest(b *testing.B) {
f, err := os.OpenFile("./dump/"+logFileName, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0o644)
require.NoError(b, err)
cfg := config.NewConfig(".env")
host := cfg.TestGoHost
employeeID := cfg.TestNestEmployee
defer f.Close()
log.SetOutput(f)
b.ResetTimer()
for i := 0; i < b.N; i++ {
start := time.Now()
assert.NoError(b, nest_benchmark.BenchmarkNoLog(bmNumberOfRequests))
assert.NoError(b, nest_benchmark.BenchmarkNoLog(host, employeeID, bmNumberOfRequests))
b.Logf("Request took %v", time.Since(start))
b.Logf("i %d | b.N %d", i, b.N)
}
@ -67,6 +78,10 @@ func TestGoRequestsPerSecondFor15s(t *testing.T) {
f, err := os.OpenFile("./dump/"+logFileName, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0o644)
require.NoError(t, err)
cfg := config.NewConfig(".env")
host := cfg.TestGoHost
employeeID := cfg.TestGoEmployee
defer f.Close()
log.SetOutput(f)
@ -75,7 +90,7 @@ func TestGoRequestsPerSecondFor15s(t *testing.T) {
for i := 0; i < seconds; i++ {
go func() {
defer wg.Done()
require.NoError(t, go_benchmark.Benchmark(numberOfRequestsAtOnce))
require.NoError(t, go_benchmark.Benchmark(host, employeeID, numberOfRequestsAtOnce))
}()
time.Sleep(time.Second)
}
@ -88,6 +103,10 @@ func TestNestRequestsPerSecondFor15s(t *testing.T) {
f, err := os.OpenFile("./dump/"+logFileName, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0o644)
require.NoError(t, err)
cfg := config.NewConfig(".env")
host := cfg.TestGoHost
employeeID := cfg.TestNestEmployee
defer f.Close()
log.SetOutput(f)
@ -96,7 +115,7 @@ func TestNestRequestsPerSecondFor15s(t *testing.T) {
for i := 0; i < seconds; i++ {
go func() {
defer wg.Done()
require.NoError(t, nest_benchmark.Benchmark(numberOfRequestsAtOnce))
require.NoError(t, nest_benchmark.Benchmark(host, employeeID, numberOfRequestsAtOnce))
}()
time.Sleep(time.Second)
}
@ -109,10 +128,14 @@ func TestGoRequests(t *testing.T) {
f, err := os.OpenFile("./dump/"+logFileName, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0o644)
require.NoError(t, err)
cfg := config.NewConfig(".env")
host := cfg.TestGoHost
employeeID := cfg.TestGoEmployee
defer f.Close()
log.SetOutput(f)
require.NoError(t, go_benchmark.Benchmark(numberOfRequestsAtOnce))
require.NoError(t, go_benchmark.Benchmark(host, employeeID, numberOfRequestsAtOnce))
}
func TestNestRequests(t *testing.T) {
@ -121,8 +144,12 @@ func TestNestRequests(t *testing.T) {
f, err := os.OpenFile("./dump/"+logFileName, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0o644)
require.NoError(t, err)
cfg := config.NewConfig(".env")
host := cfg.TestGoHost
employeeID := cfg.TestNestEmployee
defer f.Close()
log.SetOutput(f)
require.NoError(t, nest_benchmark.Benchmark(numberOfRequestsAtOnce))
require.NoError(t, nest_benchmark.Benchmark(host, employeeID, numberOfRequestsAtOnce))
}

13
benchmark/go/go.go
View File

@ -44,19 +44,18 @@ func doRequest(wg *sync.WaitGroup, host, employeeID string, errChan chan error)
return nil
}
func BenchmarkNoLog(totalRequests int) error {
return benchmark(totalRequests, false)
func BenchmarkNoLog(host, employeeID string, totalRequests int) error {
return benchmark(host, employeeID, totalRequests, false)
}
func Benchmark(totalRequests int) error {
return benchmark(totalRequests, true)
func Benchmark(host, employeeID string, totalRequests int) error {
return benchmark(host, employeeID, totalRequests, true)
}
func benchmark(totalRequests int, dolog bool) error {
func benchmark(host, employeeID string, totalRequests int, dolog bool) error {
cr := crono.New()
if dolog {
defer cr.Table()
}
host := "http://127.0.0.1:45654"
wg := &sync.WaitGroup{}
errChan := make(chan error, 1)
@ -67,7 +66,7 @@ func benchmark(totalRequests int, dolog bool) error {
errChan <- nil
}()
for i := 0; i < totalRequests; i++ {
go doRequest(wg, host, "975135", errChan)
go doRequest(wg, host, employeeID, errChan)
}
}(errChan)

13
benchmark/nest/nest.go
View File

@ -70,20 +70,19 @@ func doRequest(wg *sync.WaitGroup, host string, employeeID int, errChan chan err
return nil
}
func BenchmarkNoLog(totalRequests int) error {
return benchmark(totalRequests, false)
func BenchmarkNoLog(host string, employeeNumber, totalRequests int) error {
return benchmark(host, employeeNumber, totalRequests, false)
}
func Benchmark(totalRequests int) error {
return benchmark(totalRequests, true)
func Benchmark(host string, employeeNumber, totalRequests int) error {
return benchmark(host, employeeNumber, totalRequests, true)
}
func benchmark(totalRequests int, dolog bool) error {
func benchmark(host string, employeeNumber int, totalRequests int, dolog bool) error {
cr := crono.New()
if dolog {
defer cr.Table()
}
host := "http://127.0.0.1:3458/graphql"
wg := &sync.WaitGroup{}
errChan := make(chan error, 1)
@ -95,7 +94,7 @@ func benchmark(totalRequests int, dolog bool) error {
}()
for i := 0; i < totalRequests; i++ {
wg.Add(1)
go doRequest(wg, host, 975153, errChan)
go doRequest(wg, host, employeeNumber, errChan)
}
}(errChan)

2
cmd/etl/main.go
View File

@ -78,7 +78,7 @@ func main() {
log.Fatalln("client.Connect", err)
}
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.EmployeeWorkInformationCollection)
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.WorkInformationCollection)
if err = employeeWICollection.Drop(ctx); err != nil {
log.Fatalln("employeeWICollection.Drop", err)
}

4
cmd/server/main.go
View File

@ -71,13 +71,13 @@ func main() {
log.Fatalln("client.Connect", err)
}
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.EmployeeWorkInformationCollection)
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.WorkInformationCollection)
professionalRepo := employee_wi.NewRepo(employeeWICollection)
restServer := apihttp.
NewRestServer(cfg, cr).
WithEmployeeWIHandler(services.NewEmployeeWIService(ctx, professionalRepo)).
WithAMSHander()
WithAMSHandler()
cr.MarkAndRestart("dependencies loaded")
go func() {

6
config/config.go
View File

@ -18,8 +18,12 @@ type Config struct {
AmsApiKey string `required:"true" split_words:"true"`
DbAddress string `required:"true" split_words:"true"`
DbName string `required:"true" split_words:"true"`
EmployeeWorkInformationCollection string `required:"true" split_words:"true"`
WorkInformationCollection string `required:"true" split_words:"true"`
EmployeeIdList []string `required:"true" split_words:"true"`
TestGoHost string `required:"true" split_words:"true"`
TestGoEmployee string `required:"true" split_words:"true"`
TestNestHost string `required:"true" split_words:"true"`
TestNestEmployee int `required:"true" split_words:"true"`
}
func NewConfig(envFile string) *Config {

2
internal/api/http/server.go
View File

@ -33,7 +33,7 @@ func (s *restServer) WithEmployeeWIHandler(employeeWISrv services.EmployeeWIServ
return s
}
func (s *restServer) WithAMSHander() *restServer {
func (s *restServer) WithAMSHandler() *restServer {
s.amsEmployeeWIHdl = handler.NewAMSEmployeeWIHandler(s.cfg.AmsApi, s.cfg.AmsApiKey)
return s
}

14
internal/etl/etl_benchmark_test.go
View File

@ -41,7 +41,7 @@ func BenchmarkETLFanout(b *testing.B) {
require.NoError(b, client.Connect(ctx), "client.Connect")
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.EmployeeWorkInformationCollection)
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.WorkInformationCollection)
require.NoError(b, employeeWICollection.Drop(ctx), "employeeWICollection.Drop")
repo := employee_wi.NewRepo(employeeWICollection)
@ -79,7 +79,7 @@ func BenchmarkETLFanout2(b *testing.B) {
require.NoError(b, client.Connect(ctx), "client.Connect")
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.EmployeeWorkInformationCollection)
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.WorkInformationCollection)
require.NoError(b, employeeWICollection.Drop(ctx), "employeeWICollection.Drop")
repo := employee_wi.NewRepo(employeeWICollection)
@ -115,7 +115,7 @@ func BenchmarkETLMain(b *testing.B) {
require.NoError(b, client.Connect(ctx), "client.Connect")
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.EmployeeWorkInformationCollection)
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.WorkInformationCollection)
require.NoError(b, employeeWICollection.Drop(ctx), "employeeWICollection.Drop")
repo := employee_wi.NewRepo(employeeWICollection)
@ -154,7 +154,7 @@ func TestETLFanout(t *testing.T) {
require.NoError(t, client.Connect(ctx), "client.Connect")
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.EmployeeWorkInformationCollection)
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.WorkInformationCollection)
require.NoError(t, employeeWICollection.Drop(ctx), "employeeWICollection.Drop")
repo := employee_wi.NewRepo(employeeWICollection)
@ -191,7 +191,7 @@ func TestETLFanOut2(t *testing.T) {
require.NoError(t, client.Connect(ctx), "client.Connect")
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.EmployeeWorkInformationCollection)
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.WorkInformationCollection)
require.NoError(t, employeeWICollection.Drop(ctx), "employeeWICollection.Drop")
repo := employee_wi.NewRepo(employeeWICollection)
@ -227,7 +227,7 @@ func TestETLMain(t *testing.T) {
require.NoError(t, client.Connect(ctx), "client.Connect")
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.EmployeeWorkInformationCollection)
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.WorkInformationCollection)
require.NoError(t, employeeWICollection.Drop(ctx), "employeeWICollection.Drop")
repo := employee_wi.NewRepo(employeeWICollection)
@ -251,7 +251,7 @@ func TestLoad(t *testing.T) {
require.NoError(t, client.Connect(ctx), "client.Connect")
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.EmployeeWorkInformationCollection)
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.WorkInformationCollection)
if err = employeeWICollection.Drop(ctx); err != nil {
log.Fatalln("employeeWICollection.Drop", err)
}

2
internal/services/employee_wi.go
View File

@ -44,7 +44,7 @@ func (p *EmployeeWIService) GetByAll(ctx context.Context, id string) (*domain.Em
func (p *EmployeeWIService) InsertMany(ctx context.Context, ei []domain.EmployeeWorkInformation) error {
err := p.repo.InsertMany(ctx, ei)
if err != nil {
return fmt.Errorf("repo.AddAppointmentTo: %s", err)
return fmt.Errorf("repo.InsertMany: %s", err)
}
return nil
}