ess-etl-go/internal/etl/etl_benchmark_test.go

268 lines
6.9 KiB
Go

package etl_test
import (
"context"
"log"
"os"
"runtime/trace"
"testing"
"gitea.urkob.com/urko/crono"
"gitea.urkob.com/urko/ess-etl-go/config"
"gitea.urkob.com/urko/ess-etl-go/internal/etl"
"gitea.urkob.com/urko/ess-etl-go/internal/request"
"gitea.urkob.com/urko/ess-etl-go/internal/xml_loader"
"gitea.urkob.com/urko/ess-etl-go/pkg/adapter/repository/mongodb/employee_wi"
"github.com/stretchr/testify/require"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
const traceFlag = false
func BenchmarkETLFanout(b *testing.B) {
if traceFlag {
trace.Start(os.Stdout)
}
defer func() {
if traceFlag {
trace.Stop()
}
}()
cfg := config.NewConfig(".env")
ctx := context.Background()
dbOpts := options.Client()
dbOpts.ApplyURI(cfg.DbAddress)
client, err := mongo.NewClient(dbOpts)
require.NoError(b, err, "mongo.NewClient")
require.NoError(b, client.Connect(ctx), "client.Connect")
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.WorkInformationCollection)
require.NoError(b, employeeWICollection.Drop(ctx), "employeeWICollection.Drop")
repo := employee_wi.NewRepo(employeeWICollection)
r := request.NewRequestService(cfg.AmsApi, cfg.AmsApiKey)
ewiLoader := xml_loader.NewEmployeeWILoader(r)
from, to := "2023-01-01", "2023-01-31"
etlLoader := etl.New(ewiLoader, repo)
// err = etlLoader.Main(signalContext(ctx), cr, cfg.EmployeeIdList, from, to)
for i := 0; i < b.N; i++ {
require.NoError(b, etlLoader.FanOut(ctx, cfg.EmployeeIdList, from, to))
}
}
func BenchmarkETLFanout2(b *testing.B) {
if traceFlag {
trace.Start(os.Stdout)
}
defer func() {
if traceFlag {
trace.Stop()
}
}()
cfg := config.NewConfig(".env")
ctx := context.Background()
dbOpts := options.Client()
dbOpts.ApplyURI(cfg.DbAddress)
client, err := mongo.NewClient(dbOpts)
require.NoError(b, err, "mongo.NewClient")
require.NoError(b, client.Connect(ctx), "client.Connect")
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.WorkInformationCollection)
require.NoError(b, employeeWICollection.Drop(ctx), "employeeWICollection.Drop")
repo := employee_wi.NewRepo(employeeWICollection)
r := request.NewRequestService(cfg.AmsApi, cfg.AmsApiKey)
ewiLoader := xml_loader.NewEmployeeWILoader(r)
from, to := "2023-01-01", "2023-01-31"
etlLoader := etl.New(ewiLoader, repo)
for i := 0; i < b.N; i++ {
require.NoError(b, etlLoader.FanOut2(ctx, cfg.EmployeeIdList, from, to))
}
}
func BenchmarkETLMain(b *testing.B) {
if traceFlag {
trace.Start(os.Stdout)
}
defer func() {
if traceFlag {
trace.Stop()
}
}()
cfg := config.NewConfig(".env")
ctx := context.Background()
dbOpts := options.Client()
dbOpts.ApplyURI(cfg.DbAddress)
client, err := mongo.NewClient(dbOpts)
require.NoError(b, err, "mongo.NewClient")
require.NoError(b, client.Connect(ctx), "client.Connect")
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.WorkInformationCollection)
require.NoError(b, employeeWICollection.Drop(ctx), "employeeWICollection.Drop")
repo := employee_wi.NewRepo(employeeWICollection)
r := request.NewRequestService(cfg.AmsApi, cfg.AmsApiKey)
ewiLoader := xml_loader.NewEmployeeWILoader(r)
from, to := "2023-01-01", "2023-01-31"
etlLoader := etl.New(ewiLoader, repo)
for i := 0; i < b.N; i++ {
require.NoError(b, etlLoader.Main(ctx, crono.New(), cfg.EmployeeIdList, from, to))
}
}
func TestETLFanout(t *testing.T) {
cr := crono.New()
if traceFlag {
trace.Start(os.Stdout)
} else {
defer cr.Table()
}
defer func() {
if traceFlag {
trace.Stop()
}
}()
cfg := config.NewConfig(".env")
ctx := context.Background()
dbOpts := options.Client()
dbOpts.ApplyURI(cfg.DbAddress)
client, err := mongo.NewClient(dbOpts)
require.NoError(t, err, "mongo.NewClient")
require.NoError(t, client.Connect(ctx), "client.Connect")
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.WorkInformationCollection)
require.NoError(t, employeeWICollection.Drop(ctx), "employeeWICollection.Drop")
repo := employee_wi.NewRepo(employeeWICollection)
r := request.NewRequestService(cfg.AmsApi, cfg.AmsApiKey)
ewiLoader := xml_loader.NewEmployeeWILoader(r)
from, to := "2023-01-01", "2023-01-31"
etlLoader := etl.New(ewiLoader, repo)
require.NoError(t, etlLoader.FanOut(ctx, cfg.EmployeeIdList, from, to))
}
func TestETLFanOut2(t *testing.T) {
cr := crono.New()
if traceFlag {
trace.Start(os.Stdout)
} else {
defer cr.Table()
}
defer func() {
if traceFlag {
trace.Stop()
}
}()
cfg := config.NewConfig(".env")
ctx := context.Background()
dbOpts := options.Client()
dbOpts.ApplyURI(cfg.DbAddress)
client, err := mongo.NewClient(dbOpts)
require.NoError(t, err, "mongo.NewClient")
require.NoError(t, client.Connect(ctx), "client.Connect")
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.WorkInformationCollection)
require.NoError(t, employeeWICollection.Drop(ctx), "employeeWICollection.Drop")
repo := employee_wi.NewRepo(employeeWICollection)
r := request.NewRequestService(cfg.AmsApi, cfg.AmsApiKey)
ewiLoader := xml_loader.NewEmployeeWILoader(r)
from, to := "2023-01-01", "2023-01-31"
etlLoader := etl.New(ewiLoader, repo)
require.NoError(t, etlLoader.FanOut2(ctx, cfg.EmployeeIdList, from, to))
}
func TestETLMain(t *testing.T) {
cr := crono.New()
if traceFlag {
trace.Start(os.Stdout)
} else {
defer cr.Table()
}
defer func() {
if traceFlag {
trace.Stop()
}
}()
cfg := config.NewConfig(".env")
ctx := context.Background()
dbOpts := options.Client()
dbOpts.ApplyURI(cfg.DbAddress)
client, err := mongo.NewClient(dbOpts)
require.NoError(t, err, "mongo.NewClient")
require.NoError(t, client.Connect(ctx), "client.Connect")
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.WorkInformationCollection)
require.NoError(t, employeeWICollection.Drop(ctx), "employeeWICollection.Drop")
repo := employee_wi.NewRepo(employeeWICollection)
r := request.NewRequestService(cfg.AmsApi, cfg.AmsApiKey)
ewiLoader := xml_loader.NewEmployeeWILoader(r)
from, to := "2023-01-01", "2023-01-31"
etlLoader := etl.New(ewiLoader, repo)
require.NoError(t, etlLoader.Main(ctx, cr, cfg.EmployeeIdList, from, to))
}
func TestLoad(t *testing.T) {
cfg := config.NewConfig(".env")
ctx := context.Background()
dbOpts := options.Client()
dbOpts.ApplyURI(cfg.DbAddress)
client, err := mongo.NewClient(dbOpts)
require.NoError(t, err, "mongo.NewClient")
require.NoError(t, client.Connect(ctx), "client.Connect")
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.WorkInformationCollection)
if err = employeeWICollection.Drop(ctx); err != nil {
log.Fatalln("employeeWICollection.Drop", err)
}
repo := employee_wi.NewRepo(employeeWICollection)
r := request.NewRequestService(cfg.AmsApi, cfg.AmsApiKey)
ewiLoader := xml_loader.NewEmployeeWILoader(r)
from, to := "2023-01-01", "2023-01-31"
e := etl.New(ewiLoader, repo)
require.NoError(t, e.FanOut(ctx, cfg.EmployeeIdList, from, to))
}