urko
/
ess-etl-go
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: improve benchmark

This commit is contained in:
Urko 2023-04-05 19:04:52 +02:00
parent 8271f8eb19
commit 1697e354f4
9 changed files with 35 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
benchmark/go/go.go
View File

@ -7,7 +7,7 @@ import (
"net/http"
"sync"
"gitea.urkob.com/urko/ess-etl-go/pkg/crono"
"gitea.urkob.com/urko/crono"
)
func doRequest(wg *sync.WaitGroup, host, employeeID string, errChan chan error) error {
@ -44,7 +44,7 @@ func doRequest(wg *sync.WaitGroup, host, employeeID string, errChan chan error)
return nil
}
func Benchmark() {
func Benchmark() error {
cr := crono.New()
defer cr.Table()
host := "http://127.0.0.1:45654"
@ -52,9 +52,12 @@ func Benchmark() {
wg := &sync.WaitGroup{}
errChan := make(chan error, 1)
wg.Add(1)
totalRequests := 1000
totalRequests := 5000
go func(errChan chan error) {
defer wg.Done()
defer func() {
errChan <- nil
}()
for i := 0; i < totalRequests; i++ {
wg.Add(1)
go doRequest(wg, host, "975135", errChan)
@ -62,5 +65,10 @@ func Benchmark() {
}(errChan)
wg.Wait()
cr.MarkAndRestart(fmt.Sprintf("go handle %d response", totalRequests))
if err := <-errChan; err != nil {
return fmt.Errorf("Benchmark: %s", err)
}
cr.MarkAndRestart(fmt.Sprintf("go handle %d response", totalRequests))
return nil
}

22
benchmark/main.go
View File

@ -5,11 +5,9 @@ import (
"log"
"os"
"strings"
"sync"
"time"
go_benchmark "gitea.urkob.com/urko/ess-etl-go/benchmark/go"
nest_benchmark "gitea.urkob.com/urko/ess-etl-go/benchmark/nest"
)
func main() {
@ -23,17 +21,11 @@ func main() {
defer f.Close()
log.SetOutput(f)
wg := &sync.WaitGroup{}
wg.Add(2)
go func() {
defer wg.Done()
go_benchmark.Benchmark()
}()
go func() {
defer wg.Done()
nest_benchmark.Benchmark()
}()
wg.Wait()
if err := go_benchmark.Benchmark(); err != nil {
log.Fatalln("go_benchmark.Benchmark", err)
}
// if err := nest_benchmark.Benchmark(); err != nil {
// log.Fatalln("nest_benchmark.Benchmark", err)
// }
}

22
benchmark/nest/nest.go
View File

@ -9,7 +9,7 @@ import (
"net/http"
"sync"
"gitea.urkob.com/urko/ess-etl-go/pkg/crono"
"gitea.urkob.com/urko/crono"
)
type GraphqlQuery struct {
@ -70,7 +70,7 @@ func doRequest(wg *sync.WaitGroup, host string, employeeID int, errChan chan err
return nil
}
func Benchmark() {
func Benchmark() error {
cr := crono.New()
defer cr.Table()
@ -79,21 +79,23 @@ func Benchmark() {
wg := &sync.WaitGroup{}
errChan := make(chan error, 1)
wg.Add(1)
totalRequests := 1000
totalRequests := 5000
go func(errChan chan error) {
defer wg.Done()
defer func() {
errChan <- nil
}()
for i := 0; i < totalRequests; i++ {
wg.Add(1)
go doRequest(wg, host, 975153, errChan)
}
}(errChan)
go func(errChan chan error) {
if err := <-errChan; err != nil {
log.Fatalln("error process request", err)
}
}(errChan)
wg.Wait()
cr.MarkAndRestart(fmt.Sprintf("nest handle %d response", totalRequests))
if err := <-errChan; err != nil {
return fmt.Errorf("Benchmark: %s", err)
}
cr.MarkAndRestart(fmt.Sprintf("nest handle %d response", totalRequests))
return nil
}

2
cmd/etl/main.go
View File

@ -10,11 +10,11 @@ import (
"sync"
"syscall"
"gitea.urkob.com/urko/crono"
"gitea.urkob.com/urko/ess-etl-go/config"
"gitea.urkob.com/urko/ess-etl-go/internal/request"
"gitea.urkob.com/urko/ess-etl-go/internal/xml_loader"
"gitea.urkob.com/urko/ess-etl-go/pkg/adapter/repository/mongodb/employee_wi"
"gitea.urkob.com/urko/ess-etl-go/pkg/crono"
"gitea.urkob.com/urko/ess-etl-go/pkg/domain"
"go.mongodb.org/mongo-driver/mongo"

2
cmd/server/main.go
View File

@ -7,11 +7,11 @@ import (
"os/signal"
"syscall"
"gitea.urkob.com/urko/crono"
"gitea.urkob.com/urko/ess-etl-go/config"
"gitea.urkob.com/urko/ess-etl-go/internal/api/http"
"gitea.urkob.com/urko/ess-etl-go/internal/services"
"gitea.urkob.com/urko/ess-etl-go/pkg/adapter/repository/mongodb/employee_wi"
"gitea.urkob.com/urko/ess-etl-go/pkg/crono"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"

3
go.mod
View File

@ -3,9 +3,9 @@ module gitea.urkob.com/urko/ess-etl-go
go 1.19
require (
gitea.urkob.com/urko/crono v0.0.0-20230405153202-0554f3e53a4c
gitea.urkob.com/urko/go-root-dir v0.0.0-20230311113851-2f6d4355888a
github.com/gofiber/fiber/v2 v2.43.0
github.com/jedib0t/go-pretty/v6 v6.4.6
github.com/joho/godotenv v1.5.1
github.com/kelseyhightower/envconfig v1.4.0
go.mongodb.org/mongo-driver v1.11.3
@ -15,6 +15,7 @@ require (
github.com/andybalholm/brotli v1.0.5 // indirect
github.com/golang/snappy v0.0.1 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/jedib0t/go-pretty/v6 v6.4.6 // indirect
github.com/klauspost/compress v1.16.3 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.17 // indirect

2
go.sum
View File

@ -1,3 +1,5 @@
gitea.urkob.com/urko/crono v0.0.0-20230405153202-0554f3e53a4c h1:fIVALXjqe6BzhaVAq63uTp1JciHNSv9CvGdQf1wjADE=
gitea.urkob.com/urko/crono v0.0.0-20230405153202-0554f3e53a4c/go.mod h1:KVtHUEyBxndGirEJybX2Wg/Q5HrosgNE8MWlhMheOBo=
gitea.urkob.com/urko/go-root-dir v0.0.0-20230311113851-2f6d4355888a h1:s73cd3bRR6v0LGiBei841iIolbBJN2tbkUwN54X9vVg=
gitea.urkob.com/urko/go-root-dir v0.0.0-20230311113851-2f6d4355888a/go.mod h1:mU9nRHl70tBhJFbgKotpoXMV+s0wx+1uJ988p4oEpSo=
github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=

2
internal/api/http/server.go
View File

@ -4,10 +4,10 @@ import (
"fmt"
"log"
"gitea.urkob.com/urko/crono"
"gitea.urkob.com/urko/ess-etl-go/config"
"gitea.urkob.com/urko/ess-etl-go/internal/api/http/handler"
"gitea.urkob.com/urko/ess-etl-go/internal/services"
"gitea.urkob.com/urko/ess-etl-go/pkg/crono"
"github.com/gofiber/fiber/v2"
"github.com/gofiber/fiber/v2/middleware/cors"

84
pkg/crono/crono.go
View File

@ -1,84 +0,0 @@
package crono
import (
"fmt"
"os"
"time"
"github.com/jedib0t/go-pretty/v6/table"
)
type Timing struct {
Msg string
Elapsed time.Duration
}
type Crono struct {
Begin time.Time
Cursor time.Time
Timings []Timing
}
// new cronograph with autostart.
func New() *Crono {
c := new(Crono)
c.Timings = make([]Timing, 0)
c.Begin = time.Now()
c.Cursor = c.Begin
return c
}
// restart the crono
// keeps the begin time.
func (c *Crono) Restart() {
c.Cursor = time.Now()
}
// push a mark.
func (c *Crono) Mark(msg string) {
c.Timings = append(c.Timings, Timing{msg, c.GetElapsed()})
}
// push a mark and restart the cursor time.
func (c *Crono) MarkAndRestart(msg string) {
c.Mark(msg)
c.Restart()
}
// time in seconds since last start
// it doesn't restart the crono.
func (c *Crono) GetElapsed() time.Duration {
return time.Since(c.Cursor)
}
// total time sice start.
func (c *Crono) Total() time.Duration {
return time.Since(c.Begin)
}
func (c *Crono) Table() {
t := table.NewWriter()
t.SetOutputMirror(os.Stdout)
t.AppendHeader(table.Row{"#", "Description", "Elapsed"})
for i, e := range c.Timings {
t.AppendRow([]interface{}{i, e.Msg, durationToString(e.Elapsed)})
}
t.AppendSeparator()
t.AppendFooter(table.Row{"", "Total", durationToString(c.Total())})
t.Render()
}
func durationToString(t time.Duration) string {
timeString := fmt.Sprintf("%dms", t.Milliseconds())
if t.Seconds() > 3 {
timeString += " !!!"
} else if t.Seconds() > 2 {
timeString += " .!!"
} else if t.Seconds() > 1 {
timeString += " ..!"
} else {
timeString += " ..."
}
return timeString
}