268 lines
7.0 KiB
Go
268 lines
7.0 KiB
Go
package etl_test
|
|
|
|
import (
|
|
"context"
|
|
"log"
|
|
"os"
|
|
"runtime/trace"
|
|
"testing"
|
|
|
|
"gitea.urkob.com/urko/crono"
|
|
"gitea.urkob.com/urko/ess-etl-go/config"
|
|
"gitea.urkob.com/urko/ess-etl-go/internal/etl"
|
|
"gitea.urkob.com/urko/ess-etl-go/internal/request"
|
|
"gitea.urkob.com/urko/ess-etl-go/internal/xml_loader"
|
|
"gitea.urkob.com/urko/ess-etl-go/pkg/adapter/repository/mongodb/employee_wi"
|
|
"github.com/stretchr/testify/require"
|
|
"go.mongodb.org/mongo-driver/mongo"
|
|
"go.mongodb.org/mongo-driver/mongo/options"
|
|
)
|
|
|
|
const traceFlag = false
|
|
|
|
func BenchmarkETLFanout(b *testing.B) {
|
|
if traceFlag {
|
|
trace.Start(os.Stdout)
|
|
}
|
|
|
|
defer func() {
|
|
if traceFlag {
|
|
trace.Stop()
|
|
}
|
|
}()
|
|
cfg := config.NewConfig(".env")
|
|
ctx := context.Background()
|
|
|
|
dbOpts := options.Client()
|
|
dbOpts.ApplyURI(cfg.DbAddress)
|
|
|
|
client, err := mongo.NewClient(dbOpts)
|
|
require.NoError(b, err, "mongo.NewClient")
|
|
|
|
require.NoError(b, client.Connect(ctx), "client.Connect")
|
|
|
|
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.EmployeeWorkInformationCollection)
|
|
require.NoError(b, employeeWICollection.Drop(ctx), "employeeWICollection.Drop")
|
|
repo := employee_wi.NewRepo(employeeWICollection)
|
|
|
|
r := request.NewRequestService(cfg.AmsApi, cfg.AmsApiKey)
|
|
|
|
ewiLoader := xml_loader.NewEmployeeWILoader(r)
|
|
from, to := "2023-01-01", "2023-01-31"
|
|
etlLoader := etl.New(ewiLoader, repo)
|
|
|
|
// err = etlLoader.Main(signalContext(ctx), cr, cfg.EmployeeIdList, from, to)
|
|
|
|
for i := 0; i < b.N; i++ {
|
|
require.NoError(b, etlLoader.FanOut(ctx, cfg.EmployeeIdList, from, to))
|
|
}
|
|
}
|
|
|
|
func BenchmarkETLFanout2(b *testing.B) {
|
|
if traceFlag {
|
|
trace.Start(os.Stdout)
|
|
}
|
|
|
|
defer func() {
|
|
if traceFlag {
|
|
trace.Stop()
|
|
}
|
|
}()
|
|
cfg := config.NewConfig(".env")
|
|
ctx := context.Background()
|
|
|
|
dbOpts := options.Client()
|
|
dbOpts.ApplyURI(cfg.DbAddress)
|
|
|
|
client, err := mongo.NewClient(dbOpts)
|
|
require.NoError(b, err, "mongo.NewClient")
|
|
|
|
require.NoError(b, client.Connect(ctx), "client.Connect")
|
|
|
|
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.EmployeeWorkInformationCollection)
|
|
require.NoError(b, employeeWICollection.Drop(ctx), "employeeWICollection.Drop")
|
|
repo := employee_wi.NewRepo(employeeWICollection)
|
|
|
|
r := request.NewRequestService(cfg.AmsApi, cfg.AmsApiKey)
|
|
|
|
ewiLoader := xml_loader.NewEmployeeWILoader(r)
|
|
from, to := "2023-01-01", "2023-01-31"
|
|
etlLoader := etl.New(ewiLoader, repo)
|
|
|
|
for i := 0; i < b.N; i++ {
|
|
require.NoError(b, etlLoader.FanOut2(ctx, cfg.EmployeeIdList, from, to))
|
|
}
|
|
}
|
|
|
|
func BenchmarkETLMain(b *testing.B) {
|
|
if traceFlag {
|
|
trace.Start(os.Stdout)
|
|
}
|
|
|
|
defer func() {
|
|
if traceFlag {
|
|
trace.Stop()
|
|
}
|
|
}()
|
|
cfg := config.NewConfig(".env")
|
|
ctx := context.Background()
|
|
|
|
dbOpts := options.Client()
|
|
dbOpts.ApplyURI(cfg.DbAddress)
|
|
|
|
client, err := mongo.NewClient(dbOpts)
|
|
require.NoError(b, err, "mongo.NewClient")
|
|
|
|
require.NoError(b, client.Connect(ctx), "client.Connect")
|
|
|
|
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.EmployeeWorkInformationCollection)
|
|
require.NoError(b, employeeWICollection.Drop(ctx), "employeeWICollection.Drop")
|
|
repo := employee_wi.NewRepo(employeeWICollection)
|
|
|
|
r := request.NewRequestService(cfg.AmsApi, cfg.AmsApiKey)
|
|
|
|
ewiLoader := xml_loader.NewEmployeeWILoader(r)
|
|
from, to := "2023-01-01", "2023-01-31"
|
|
etlLoader := etl.New(ewiLoader, repo)
|
|
|
|
for i := 0; i < b.N; i++ {
|
|
require.NoError(b, etlLoader.Main(ctx, crono.New(), cfg.EmployeeIdList, from, to))
|
|
}
|
|
}
|
|
|
|
func TestETLFanout(t *testing.T) {
|
|
cr := crono.New()
|
|
if traceFlag {
|
|
trace.Start(os.Stdout)
|
|
} else {
|
|
defer cr.Table()
|
|
}
|
|
|
|
defer func() {
|
|
if traceFlag {
|
|
trace.Stop()
|
|
}
|
|
}()
|
|
cfg := config.NewConfig(".env")
|
|
ctx := context.Background()
|
|
|
|
dbOpts := options.Client()
|
|
dbOpts.ApplyURI(cfg.DbAddress)
|
|
|
|
client, err := mongo.NewClient(dbOpts)
|
|
require.NoError(t, err, "mongo.NewClient")
|
|
|
|
require.NoError(t, client.Connect(ctx), "client.Connect")
|
|
|
|
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.EmployeeWorkInformationCollection)
|
|
require.NoError(t, employeeWICollection.Drop(ctx), "employeeWICollection.Drop")
|
|
repo := employee_wi.NewRepo(employeeWICollection)
|
|
|
|
r := request.NewRequestService(cfg.AmsApi, cfg.AmsApiKey)
|
|
|
|
ewiLoader := xml_loader.NewEmployeeWILoader(r)
|
|
from, to := "2023-01-01", "2023-01-31"
|
|
etlLoader := etl.New(ewiLoader, repo)
|
|
|
|
require.NoError(t, etlLoader.FanOut(ctx, cfg.EmployeeIdList, from, to))
|
|
}
|
|
|
|
func TestETLFanOut2(t *testing.T) {
|
|
cr := crono.New()
|
|
if traceFlag {
|
|
trace.Start(os.Stdout)
|
|
} else {
|
|
defer cr.Table()
|
|
}
|
|
|
|
defer func() {
|
|
if traceFlag {
|
|
trace.Stop()
|
|
}
|
|
}()
|
|
cfg := config.NewConfig(".env")
|
|
ctx := context.Background()
|
|
|
|
dbOpts := options.Client()
|
|
dbOpts.ApplyURI(cfg.DbAddress)
|
|
|
|
client, err := mongo.NewClient(dbOpts)
|
|
require.NoError(t, err, "mongo.NewClient")
|
|
|
|
require.NoError(t, client.Connect(ctx), "client.Connect")
|
|
|
|
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.EmployeeWorkInformationCollection)
|
|
require.NoError(t, employeeWICollection.Drop(ctx), "employeeWICollection.Drop")
|
|
repo := employee_wi.NewRepo(employeeWICollection)
|
|
|
|
r := request.NewRequestService(cfg.AmsApi, cfg.AmsApiKey)
|
|
ewiLoader := xml_loader.NewEmployeeWILoader(r)
|
|
from, to := "2023-01-01", "2023-01-31"
|
|
etlLoader := etl.New(ewiLoader, repo)
|
|
|
|
require.NoError(t, etlLoader.FanOut2(ctx, cfg.EmployeeIdList, from, to))
|
|
}
|
|
|
|
func TestETLMain(t *testing.T) {
|
|
cr := crono.New()
|
|
if traceFlag {
|
|
trace.Start(os.Stdout)
|
|
} else {
|
|
defer cr.Table()
|
|
}
|
|
|
|
defer func() {
|
|
if traceFlag {
|
|
trace.Stop()
|
|
}
|
|
}()
|
|
cfg := config.NewConfig(".env")
|
|
ctx := context.Background()
|
|
|
|
dbOpts := options.Client()
|
|
dbOpts.ApplyURI(cfg.DbAddress)
|
|
|
|
client, err := mongo.NewClient(dbOpts)
|
|
require.NoError(t, err, "mongo.NewClient")
|
|
|
|
require.NoError(t, client.Connect(ctx), "client.Connect")
|
|
|
|
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.EmployeeWorkInformationCollection)
|
|
require.NoError(t, employeeWICollection.Drop(ctx), "employeeWICollection.Drop")
|
|
repo := employee_wi.NewRepo(employeeWICollection)
|
|
|
|
r := request.NewRequestService(cfg.AmsApi, cfg.AmsApiKey)
|
|
|
|
ewiLoader := xml_loader.NewEmployeeWILoader(r)
|
|
from, to := "2023-01-01", "2023-01-31"
|
|
etlLoader := etl.New(ewiLoader, repo)
|
|
require.NoError(t, etlLoader.Main(ctx, cr, cfg.EmployeeIdList, from, to))
|
|
}
|
|
|
|
func TestLoad(t *testing.T) {
|
|
cfg := config.NewConfig(".env")
|
|
ctx := context.Background()
|
|
|
|
dbOpts := options.Client()
|
|
dbOpts.ApplyURI(cfg.DbAddress)
|
|
|
|
client, err := mongo.NewClient(dbOpts)
|
|
require.NoError(t, err, "mongo.NewClient")
|
|
|
|
require.NoError(t, client.Connect(ctx), "client.Connect")
|
|
|
|
employeeWICollection := client.Database(cfg.DbName).Collection(cfg.EmployeeWorkInformationCollection)
|
|
if err = employeeWICollection.Drop(ctx); err != nil {
|
|
log.Fatalln("employeeWICollection.Drop", err)
|
|
}
|
|
repo := employee_wi.NewRepo(employeeWICollection)
|
|
|
|
r := request.NewRequestService(cfg.AmsApi, cfg.AmsApiKey)
|
|
|
|
ewiLoader := xml_loader.NewEmployeeWILoader(r)
|
|
from, to := "2023-01-01", "2023-01-31"
|
|
e := etl.New(ewiLoader, repo)
|
|
|
|
require.NoError(t, e.FanOut(ctx, cfg.EmployeeIdList, from, to))
|
|
}
|