ess-etl-go/internal/etl/etl_benchmark_test.go

package etl_test

import (
	"context"
	"log"
	"os"
	"runtime/trace"
	"testing"

	"gitea.urkob.com/urko/crono"
	"gitea.urkob.com/urko/ess-etl-go/config"
	"gitea.urkob.com/urko/ess-etl-go/internal/etl"
	"gitea.urkob.com/urko/ess-etl-go/internal/request"
	"gitea.urkob.com/urko/ess-etl-go/internal/xml_loader"
	"gitea.urkob.com/urko/ess-etl-go/pkg/adapter/repository/mongodb/employee_wi"
	"github.com/stretchr/testify/require"
	"go.mongodb.org/mongo-driver/mongo"
	"go.mongodb.org/mongo-driver/mongo/options"
)

const traceFlag = false

func BenchmarkETLFanout(b *testing.B) {
	if traceFlag {
		trace.Start(os.Stdout)
	}

	defer func() {
		if traceFlag {
			trace.Stop()
		}
	}()
	cfg := config.NewConfig(".env")
	ctx := context.Background()

	dbOpts := options.Client()
	dbOpts.ApplyURI(cfg.DbAddress)

	client, err := mongo.NewClient(dbOpts)
	require.NoError(b, err, "mongo.NewClient")

	require.NoError(b, client.Connect(ctx), "client.Connect")

	employeeWICollection := client.Database(cfg.DbName).Collection(cfg.WorkInformationCollection)
	require.NoError(b, employeeWICollection.Drop(ctx), "employeeWICollection.Drop")
	repo := employee_wi.NewRepo(employeeWICollection)

	r := request.NewRequestService(cfg.AmsApi, cfg.AmsApiKey)

	ewiLoader := xml_loader.NewEmployeeWILoader(r)
	from, to := "2023-01-01", "2023-01-31"
	etlLoader := etl.New(ewiLoader, repo)

	// err = etlLoader.Main(signalContext(ctx), cr, cfg.EmployeeIdList, from, to)

	for i := 0; i < b.N; i++ {
		require.NoError(b, etlLoader.FanOut(ctx, cfg.EmployeeIdList, from, to))
	}
}

func BenchmarkETLFanout2(b *testing.B) {
	if traceFlag {
		trace.Start(os.Stdout)
	}

	defer func() {
		if traceFlag {
			trace.Stop()
		}
	}()
	cfg := config.NewConfig(".env")
	ctx := context.Background()

	dbOpts := options.Client()
	dbOpts.ApplyURI(cfg.DbAddress)

	client, err := mongo.NewClient(dbOpts)
	require.NoError(b, err, "mongo.NewClient")

	require.NoError(b, client.Connect(ctx), "client.Connect")

	employeeWICollection := client.Database(cfg.DbName).Collection(cfg.WorkInformationCollection)
	require.NoError(b, employeeWICollection.Drop(ctx), "employeeWICollection.Drop")
	repo := employee_wi.NewRepo(employeeWICollection)

	r := request.NewRequestService(cfg.AmsApi, cfg.AmsApiKey)

	ewiLoader := xml_loader.NewEmployeeWILoader(r)
	from, to := "2023-01-01", "2023-01-31"
	etlLoader := etl.New(ewiLoader, repo)

	for i := 0; i < b.N; i++ {
		require.NoError(b, etlLoader.FanOut2(ctx, cfg.EmployeeIdList, from, to))
	}
}

func BenchmarkETLMain(b *testing.B) {
	if traceFlag {
		trace.Start(os.Stdout)
	}

	defer func() {
		if traceFlag {
			trace.Stop()
		}
	}()
	cfg := config.NewConfig(".env")
	ctx := context.Background()

	dbOpts := options.Client()
	dbOpts.ApplyURI(cfg.DbAddress)

	client, err := mongo.NewClient(dbOpts)
	require.NoError(b, err, "mongo.NewClient")

	require.NoError(b, client.Connect(ctx), "client.Connect")

	employeeWICollection := client.Database(cfg.DbName).Collection(cfg.WorkInformationCollection)
	require.NoError(b, employeeWICollection.Drop(ctx), "employeeWICollection.Drop")
	repo := employee_wi.NewRepo(employeeWICollection)

	r := request.NewRequestService(cfg.AmsApi, cfg.AmsApiKey)

	ewiLoader := xml_loader.NewEmployeeWILoader(r)
	from, to := "2023-01-01", "2023-01-31"
	etlLoader := etl.New(ewiLoader, repo)

	for i := 0; i < b.N; i++ {
		require.NoError(b, etlLoader.Main(ctx, crono.New(), cfg.EmployeeIdList, from, to))
	}
}

func TestETLFanout(t *testing.T) {
	cr := crono.New()
	if traceFlag {
		trace.Start(os.Stdout)
	} else {
		defer cr.Table()
	}

	defer func() {
		if traceFlag {
			trace.Stop()
		}
	}()
	cfg := config.NewConfig(".env")
	ctx := context.Background()

	dbOpts := options.Client()
	dbOpts.ApplyURI(cfg.DbAddress)

	client, err := mongo.NewClient(dbOpts)
	require.NoError(t, err, "mongo.NewClient")

	require.NoError(t, client.Connect(ctx), "client.Connect")

	employeeWICollection := client.Database(cfg.DbName).Collection(cfg.WorkInformationCollection)
	require.NoError(t, employeeWICollection.Drop(ctx), "employeeWICollection.Drop")
	repo := employee_wi.NewRepo(employeeWICollection)

	r := request.NewRequestService(cfg.AmsApi, cfg.AmsApiKey)

	ewiLoader := xml_loader.NewEmployeeWILoader(r)
	from, to := "2023-01-01", "2023-01-31"
	etlLoader := etl.New(ewiLoader, repo)

	require.NoError(t, etlLoader.FanOut(ctx, cfg.EmployeeIdList, from, to))
}

func TestETLFanOut2(t *testing.T) {
	cr := crono.New()
	if traceFlag {
		trace.Start(os.Stdout)
	} else {
		defer cr.Table()
	}

	defer func() {
		if traceFlag {
			trace.Stop()
		}
	}()
	cfg := config.NewConfig(".env")
	ctx := context.Background()

	dbOpts := options.Client()
	dbOpts.ApplyURI(cfg.DbAddress)

	client, err := mongo.NewClient(dbOpts)
	require.NoError(t, err, "mongo.NewClient")

	require.NoError(t, client.Connect(ctx), "client.Connect")

	employeeWICollection := client.Database(cfg.DbName).Collection(cfg.WorkInformationCollection)
	require.NoError(t, employeeWICollection.Drop(ctx), "employeeWICollection.Drop")
	repo := employee_wi.NewRepo(employeeWICollection)

	r := request.NewRequestService(cfg.AmsApi, cfg.AmsApiKey)
	ewiLoader := xml_loader.NewEmployeeWILoader(r)
	from, to := "2023-01-01", "2023-01-31"
	etlLoader := etl.New(ewiLoader, repo)

	require.NoError(t, etlLoader.FanOut2(ctx, cfg.EmployeeIdList, from, to))
}

func TestETLMain(t *testing.T) {
	cr := crono.New()
	if traceFlag {
		trace.Start(os.Stdout)
	} else {
		defer cr.Table()
	}

	defer func() {
		if traceFlag {
			trace.Stop()
		}
	}()
	cfg := config.NewConfig(".env")
	ctx := context.Background()

	dbOpts := options.Client()
	dbOpts.ApplyURI(cfg.DbAddress)

	client, err := mongo.NewClient(dbOpts)
	require.NoError(t, err, "mongo.NewClient")

	require.NoError(t, client.Connect(ctx), "client.Connect")

	employeeWICollection := client.Database(cfg.DbName).Collection(cfg.WorkInformationCollection)
	require.NoError(t, employeeWICollection.Drop(ctx), "employeeWICollection.Drop")
	repo := employee_wi.NewRepo(employeeWICollection)

	r := request.NewRequestService(cfg.AmsApi, cfg.AmsApiKey)

	ewiLoader := xml_loader.NewEmployeeWILoader(r)
	from, to := "2023-01-01", "2023-01-31"
	etlLoader := etl.New(ewiLoader, repo)
	require.NoError(t, etlLoader.Main(ctx, cr, cfg.EmployeeIdList, from, to))
}

func TestLoad(t *testing.T) {
	cfg := config.NewConfig(".env")
	ctx := context.Background()

	dbOpts := options.Client()
	dbOpts.ApplyURI(cfg.DbAddress)

	client, err := mongo.NewClient(dbOpts)
	require.NoError(t, err, "mongo.NewClient")

	require.NoError(t, client.Connect(ctx), "client.Connect")

	employeeWICollection := client.Database(cfg.DbName).Collection(cfg.WorkInformationCollection)
	if err = employeeWICollection.Drop(ctx); err != nil {
		log.Fatalln("employeeWICollection.Drop", err)
	}
	repo := employee_wi.NewRepo(employeeWICollection)

	r := request.NewRequestService(cfg.AmsApi, cfg.AmsApiKey)

	ewiLoader := xml_loader.NewEmployeeWILoader(r)
	from, to := "2023-01-01", "2023-01-31"
	e := etl.New(ewiLoader, repo)

	require.NoError(t, e.FanOut(ctx, cfg.EmployeeIdList, from, to))
}