urko
/
ess-etl-go
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: urko:dcab3994b2517a2ea57e82e7548f47f763bc2c41
Branches Tags
urko:master
..
compare: urko:9fe42609bb65047e4a67d716d84d1ef05648b2d5
Branches Tags
urko:master

No commits in common. "dcab3994b2517a2ea57e82e7548f47f763bc2c41" and "9fe42609bb65047e4a67d716d84d1ef05648b2d5" have entirely different histories.
dcab3994b2 ... 9fe42609bb

9 changed files with 42 additions and 71 deletions
Show Stats Expand all files Collapse all files

45
benchmark/benchmark_test.go
View File

@ -11,7 +11,6 @@ import (
go_benchmark "gitea.urkob.com/urko/ess-etl-go/benchmark/go"
nest_benchmark "gitea.urkob.com/urko/ess-etl-go/benchmark/nest"
"gitea.urkob.com/urko/ess-etl-go/config"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@ -25,11 +24,9 @@ func init() {
}
}
const (
bmNumberOfRequests = 100
numberOfRequestsAtOnce = 1000
seconds = 15
)
const bmNumberOfRequests = 100
const numberOfRequestsAtOnce = 1000
const seconds = 15
func BenchmarkGo(b *testing.B) {
log.SetFlags(log.Lmicroseconds)
@ -37,16 +34,12 @@ func BenchmarkGo(b *testing.B) {
f, err := os.OpenFile("./dump/"+logFileName, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0o644)
require.NoError(b, err)
cfg := config.NewConfig(".env")
host := cfg.TestGoHost
employeeID := cfg.TestGoEmployee
defer f.Close()
log.SetOutput(f)
b.ResetTimer()
for i := 0; i < b.N; i++ {
start := time.Now()
assert.NoError(b, go_benchmark.BenchmarkNoLog(host, employeeID, bmNumberOfRequests))
assert.NoError(b, go_benchmark.BenchmarkNoLog(bmNumberOfRequests))
b.Logf("Request took %v", time.Since(start))
b.Logf("i %d | b.N %d", i, b.N)
}
@ -57,16 +50,12 @@ func BenchmarkNest(b *testing.B) {
f, err := os.OpenFile("./dump/"+logFileName, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0o644)
require.NoError(b, err)
cfg := config.NewConfig(".env")
host := cfg.TestGoHost
employeeID := cfg.TestNestEmployee
defer f.Close()
log.SetOutput(f)
b.ResetTimer()
for i := 0; i < b.N; i++ {
start := time.Now()
assert.NoError(b, nest_benchmark.BenchmarkNoLog(host, employeeID, bmNumberOfRequests))
assert.NoError(b, nest_benchmark.BenchmarkNoLog(bmNumberOfRequests))
b.Logf("Request took %v", time.Since(start))
b.Logf("i %d | b.N %d", i, b.N)
}
@ -78,10 +67,6 @@ func TestGoRequestsPerSecondFor15s(t *testing.T) {
f, err := os.OpenFile("./dump/"+logFileName, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0o644)
require.NoError(t, err)
cfg := config.NewConfig(".env")
host := cfg.TestGoHost
employeeID := cfg.TestGoEmployee
defer f.Close()
log.SetOutput(f)
@ -90,7 +75,7 @@ func TestGoRequestsPerSecondFor15s(t *testing.T) {
for i := 0; i < seconds; i++ {
go func() {
defer wg.Done()
require.NoError(t, go_benchmark.Benchmark(host, employeeID, numberOfRequestsAtOnce))
require.NoError(t, go_benchmark.Benchmark(numberOfRequestsAtOnce))
}()
time.Sleep(time.Second)
}
@ -103,10 +88,6 @@ func TestNestRequestsPerSecondFor15s(t *testing.T) {
f, err := os.OpenFile("./dump/"+logFileName, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0o644)
require.NoError(t, err)
cfg := config.NewConfig(".env")
host := cfg.TestGoHost
employeeID := cfg.TestNestEmployee
defer f.Close()
log.SetOutput(f)
@ -115,7 +96,7 @@ func TestNestRequestsPerSecondFor15s(t *testing.T) {
for i := 0; i < seconds; i++ {
go func() {
defer wg.Done()
require.NoError(t, nest_benchmark.Benchmark(host, employeeID, numberOfRequestsAtOnce))
require.NoError(t, nest_benchmark.Benchmark(numberOfRequestsAtOnce))
}()
time.Sleep(time.Second)
}
@ -128,14 +109,10 @@ func TestGoRequests(t *testing.T) {
f, err := os.OpenFile("./dump/"+logFileName, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0o644)
require.NoError(t, err)
cfg := config.NewConfig(".env")
host := cfg.TestGoHost
employeeID := cfg.TestGoEmployee
defer f.Close()
log.SetOutput(f)
require.NoError(t, go_benchmark.Benchmark(host, employeeID, numberOfRequestsAtOnce))
require.NoError(t, go_benchmark.Benchmark(numberOfRequestsAtOnce))
}
func TestNestRequests(t *testing.T) {
@ -144,12 +121,8 @@ func TestNestRequests(t *testing.T) {
f, err := os.OpenFile("./dump/"+logFileName, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0o644)
require.NoError(t, err)
cfg := config.NewConfig(".env")
host := cfg.TestGoHost
employeeID := cfg.TestNestEmployee
defer f.Close()
log.SetOutput(f)
require.NoError(t, nest_benchmark.Benchmark(host, employeeID, numberOfRequestsAtOnce))
require.NoError(t, nest_benchmark.Benchmark(numberOfRequestsAtOnce))
}

13
benchmark/go/go.go
View File

@ -44,18 +44,19 @@ func doRequest(wg *sync.WaitGroup, host, employeeID string, errChan chan error)
return nil
}
func BenchmarkNoLog(host, employeeID string, totalRequests int) error {
return benchmark(host, employeeID, totalRequests, false)
func BenchmarkNoLog(totalRequests int) error {
return benchmark(totalRequests, false)
}
func Benchmark(host, employeeID string, totalRequests int) error {
return benchmark(host, employeeID, totalRequests, true)
func Benchmark(totalRequests int) error {
return benchmark(totalRequests, true)
}
func benchmark(host, employeeID string, totalRequests int, dolog bool) error {
func benchmark(totalRequests int, dolog bool) error {
cr := crono.New()
if dolog {
defer cr.Table()
}
host := "http://127.0.0.1:45654"
wg := &sync.WaitGroup{}
errChan := make(chan error, 1)
@ -66,7 +67,7 @@ func benchmark(host, employeeID string, totalRequests int, dolog bool) error {
errChan <- nil
}()
for i := 0; i < totalRequests; i++ {
go doRequest(wg, host, employeeID, errChan)
go doRequest(wg, host, "975135", errChan)
}
}(errChan)

13
benchmark/nest/nest.go
View File

@ -70,19 +70,20 @@ func doRequest(wg *sync.WaitGroup, host string, employeeID int, errChan chan err
return nil
}
func BenchmarkNoLog(host string, employeeNumber, totalRequests int) error {
return benchmark(host, employeeNumber, totalRequests, false)
func BenchmarkNoLog(totalRequests int) error {
return benchmark(totalRequests, false)
}
func Benchmark(host string, employeeNumber, totalRequests int) error {
return benchmark(host, employeeNumber, totalRequests, true)
func Benchmark(totalRequests int) error {
return benchmark(totalRequests, true)
}
func benchmark(host string, employeeNumber int, totalRequests int, dolog bool) error {
func benchmark(totalRequests int, dolog bool) error {
cr := crono.New()
if dolog {
defer cr.Table()
}
host := "http://127.0.0.1:3458/graphql"
wg := &sync.WaitGroup{}
errChan := make(chan error, 1)
@ -94,7 +95,7 @@ func benchmark(host string, employeeNumber int, totalRequests int, dolog bool) e
}()
for i := 0; i < totalRequests; i++ {
wg.Add(1)
go doRequest(wg, host, employeeNumber, errChan)
go doRequest(wg, host, 975153, errChan)
}
}(errChan)

2
cmd/etl/main.go
View File

@ -78,7 +78,7 @@ func main() {
log.Fatalln("client.Connect", err)
}
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.WorkInformationCollection)
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.EmployeeWorkInformationCollection)
if err = employeeWICollection.Drop(ctx); err != nil {
log.Fatalln("employeeWICollection.Drop", err)
}

4
cmd/server/main.go
View File

@ -71,13 +71,13 @@ func main() {
log.Fatalln("client.Connect", err)
}
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.WorkInformationCollection)
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.EmployeeWorkInformationCollection)
professionalRepo := employee_wi.NewRepo(employeeWICollection)
restServer := apihttp.
NewRestServer(cfg, cr).
WithEmployeeWIHandler(services.NewEmployeeWIService(ctx, professionalRepo)).
WithAMSHandler()
WithAMSHander()
cr.MarkAndRestart("dependencies loaded")
go func() {

6
config/config.go
View File

@ -18,12 +18,8 @@ type Config struct {
AmsApiKey string `required:"true" split_words:"true"`
DbAddress string `required:"true" split_words:"true"`
DbName string `required:"true" split_words:"true"`
WorkInformationCollection string `required:"true" split_words:"true"`
EmployeeWorkInformationCollection string `required:"true" split_words:"true"`
EmployeeIdList []string `required:"true" split_words:"true"`
TestGoHost string `required:"true" split_words:"true"`
TestGoEmployee string `required:"true" split_words:"true"`
TestNestHost string `required:"true" split_words:"true"`
TestNestEmployee int `required:"true" split_words:"true"`
}
func NewConfig(envFile string) *Config {

2
internal/api/http/server.go
View File

@ -33,7 +33,7 @@ func (s *restServer) WithEmployeeWIHandler(employeeWISrv services.EmployeeWIServ
return s
}
func (s *restServer) WithAMSHandler() *restServer {
func (s *restServer) WithAMSHander() *restServer {
s.amsEmployeeWIHdl = handler.NewAMSEmployeeWIHandler(s.cfg.AmsApi, s.cfg.AmsApiKey)
return s
}

14
internal/etl/etl_benchmark_test.go
View File

@ -41,7 +41,7 @@ func BenchmarkETLFanout(b *testing.B) {
require.NoError(b, client.Connect(ctx), "client.Connect")
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.WorkInformationCollection)
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.EmployeeWorkInformationCollection)
require.NoError(b, employeeWICollection.Drop(ctx), "employeeWICollection.Drop")
repo := employee_wi.NewRepo(employeeWICollection)
@ -79,7 +79,7 @@ func BenchmarkETLFanout2(b *testing.B) {
require.NoError(b, client.Connect(ctx), "client.Connect")
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.WorkInformationCollection)
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.EmployeeWorkInformationCollection)
require.NoError(b, employeeWICollection.Drop(ctx), "employeeWICollection.Drop")
repo := employee_wi.NewRepo(employeeWICollection)
@ -115,7 +115,7 @@ func BenchmarkETLMain(b *testing.B) {
require.NoError(b, client.Connect(ctx), "client.Connect")
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.WorkInformationCollection)
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.EmployeeWorkInformationCollection)
require.NoError(b, employeeWICollection.Drop(ctx), "employeeWICollection.Drop")
repo := employee_wi.NewRepo(employeeWICollection)
@ -154,7 +154,7 @@ func TestETLFanout(t *testing.T) {
require.NoError(t, client.Connect(ctx), "client.Connect")
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.WorkInformationCollection)
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.EmployeeWorkInformationCollection)
require.NoError(t, employeeWICollection.Drop(ctx), "employeeWICollection.Drop")
repo := employee_wi.NewRepo(employeeWICollection)
@ -191,7 +191,7 @@ func TestETLFanOut2(t *testing.T) {
require.NoError(t, client.Connect(ctx), "client.Connect")
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.WorkInformationCollection)
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.EmployeeWorkInformationCollection)
require.NoError(t, employeeWICollection.Drop(ctx), "employeeWICollection.Drop")
repo := employee_wi.NewRepo(employeeWICollection)
@ -227,7 +227,7 @@ func TestETLMain(t *testing.T) {
require.NoError(t, client.Connect(ctx), "client.Connect")
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.WorkInformationCollection)
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.EmployeeWorkInformationCollection)
require.NoError(t, employeeWICollection.Drop(ctx), "employeeWICollection.Drop")
repo := employee_wi.NewRepo(employeeWICollection)
@ -251,7 +251,7 @@ func TestLoad(t *testing.T) {
require.NoError(t, client.Connect(ctx), "client.Connect")
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.WorkInformationCollection)
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.EmployeeWorkInformationCollection)
if err = employeeWICollection.Drop(ctx); err != nil {
log.Fatalln("employeeWICollection.Drop", err)
}

2
internal/services/employee_wi.go
View File

@ -44,7 +44,7 @@ func (p *EmployeeWIService) GetByAll(ctx context.Context, id string) (*domain.Em
func (p *EmployeeWIService) InsertMany(ctx context.Context, ei []domain.EmployeeWorkInformation) error {
err := p.repo.InsertMany(ctx, ei)
if err != nil {
return fmt.Errorf("repo.InsertMany: %s", err)
return fmt.Errorf("repo.AddAppointmentTo: %s", err)
}
return nil
}