feat: update report

WIP: improve algorithym

Makefile

@@ -1,16 +1,4 @@
 COVERAGE_DIR=coverage
-BINARY_DIR=bin
-BINARY_NAME=ess-etl-go
-
-UNAME := $(shell uname -s)
-ifeq ($(UNAME),Darwin)
-	OS = macos
-else ifeq ($(UNAME),Linux)
-	OS = linux
-else
-$(error OS not supported by this Makefile)
-endif
-PACKAGE = $(shell head -1 go.mod | awk '{print $$2}')
 
 lint:
 	golangci-lint run ./...
@@ -24,8 +12,33 @@ test-coverage:
 	go test -v -coverprofile ${COVERAGE_DIR}/cover.out ./...
 	go tool cover -html ${COVERAGE_DIR}/cover.out -o ${COVERAGE_DIR}/cover.html
 benchmark:
-	go test -run none -bench . -benchtime 3s -benchmem	
+	go test -run none -bench . -benchtime 3s -benchmem
+benchmark_escape_analisys:
+	go test -gcflags "-m -m" -run none -bench . -benchtime 3s -benchmem -memprofile profile.out ./internal/etl
+pprof:
+	go tool pprof -alloc_space profile.out
+pprof_url:# top 40 -cum
+	go tool pprof -alloc_space http://localhost:5000/debug/pprof/allocs
+benchmark_server:
+	go test -gcflags "-m -m" -run none -bench . -benchtime 30s -benchmem -memprofile profile.out ./benchmark
+benchmark_etl:
+	go test -gcflags "-m -m" -run none -bench . -benchtime 15s -benchmem -memprofile profile.out ./internal/etl/ > t.out
+trace_etl: build_etl# go tool trace t.out	
+	$(build_etl)
+	time ./etl -trace > t.out
+build_etl:
+	env GOOS=linux CGO_ENABLED=0 GOARCH=amd64 go build -v -o etl ./cmd/etl/main.go
+run_etl: build_etl
+	./etl
 build_server:
-	env GOOS=linux CGO_ENABLED=0 GOARCH=amd64 go build -o ${BINARY_DIR}/${BINARY_NAME} ./main.go
+	env GOOS=linux CGO_ENABLED=0 GOARCH=amd64 go build -o server ./cmd/server/main.go
 run_server: build_server
-	./${BINARY_DIR}/${BINARY_NAME}
+	./server
+benchmark_go_1000req:
+	go test -run TestGoRequests ./benchmark
+benchmark_go_1000req_15s:
+	go test -run TestGoRequestsPerSecondFor15s ./benchmark
+benchmark_nest_1000req:
+	go test -run TestNestRequests ./benchmark
+benchmark_nest_1000req_15s:
+	go test -run TestNestRequestsPerSecondFor15s ./benchmark	

README.md

@@ -1,8 +1,72 @@
+## Requirements
+To run `docker-compose --env-file ./.env up` to init your container
+
+To have make installed.
+
+If you don't want to install make you can go to `Makefile` therefore copy the command then paste and execute in your terminal window.
+
+## Report
+You can watch my **[report](https://github.com/urko-iberia/ess-etl-go/blob/master/benchmark_report/Report.md)** to see benchmark results
+
+
+## Tests
+### ETL
+To see results you only have to run
+```bash
+make run_etl
+```
+
+### http server
+To see results you only have to run
+```bash
+make run_etl
+```
+
 ## Benchmark
 
-### go benchmark
-Start your server
-`go run cmd/server/main.go`
+### http comparision
+#### Start your go http server
+```bash
+go run cmd/server/main.go
+```
+#### Start your nest server
+```bash
+cd ~/nest-project-path
+```
+```bash
+npm start
+```
 
-Run your tests
-`go test -race -v --bench ./. --benchmem ./benchmark `
+Then run benchmark
+```bash
+make benchmark_server
+```
+
+### other tests
+To test different tests done on http server response you can do this way:
+
+First start your http server
+```bash
+make run_server
+```
+
+Then play running different tests:
+
+**IMPORTANT** Please do mantain your machine without extra running processes
+
+```bash
+make benchmark_go_1000req_15s
+```
+then try
+
+```bash
+make benchmark_go_1000req
+```
+
+Now try nest server, remember to first [startup your nest](#start-your-nest-server)
+```bash
+make benchmark_nest_1000req_15s
+```
+```bash
+make benchmark_nest_1000req
+```

benchmark/benchmark_test.go

@@ -5,6 +5,7 @@ import (
 	"log"
 	"os"
 	"strings"
+	"sync"
 	"testing"
 	"time"
 
@@ -14,56 +15,95 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+func init() {
+	if err := os.RemoveAll("./dump"); err != nil {
+		panic(fmt.Errorf("os.RemoveAll: %s", err))
+	}
+	if err := os.MkdirAll("./dump", os.ModeTemporary); err != nil {
+		panic(fmt.Errorf("os.MkdirAll: %s", err))
+	}
+}
+
+const bmNumberOfRequests = 100
+const numberOfRequestsAtOnce = 1000
+const seconds = 15
+
 func BenchmarkGo(b *testing.B) {
-	require.NoError(b, os.RemoveAll("./dump"))
-	require.NoError(b, os.MkdirAll("./dump", os.ModeTemporary))
 	log.SetFlags(log.Lmicroseconds)
-	logFileName := fmt.Sprintf("%s.txt", time.Now().Format(strings.ReplaceAll(time.RFC1123Z, ":", "_")))
+	logFileName := fmt.Sprintf("BenchmarkGo-%s.txt", time.Now().Format(strings.ReplaceAll(time.RFC1123Z, ":", "_")))
 	f, err := os.OpenFile("./dump/"+logFileName, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0o644)
 	require.NoError(b, err)
 
 	defer f.Close()
 	log.SetOutput(f)
-
+	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		assert.NoError(b, go_benchmark.BenchmarkNoLog(1))
+		start := time.Now()
+		assert.NoError(b, go_benchmark.BenchmarkNoLog(bmNumberOfRequests))
+		b.Logf("Request took %v", time.Since(start))
+		b.Logf("i %d | b.N %d", i, b.N)
 	}
-
 }
 func BenchmarkNest(b *testing.B) {
-	require.NoError(b, os.RemoveAll("./dump"))
-	require.NoError(b, os.MkdirAll("./dump", os.ModeTemporary))
 	log.SetFlags(log.Lmicroseconds)
-	logFileName := fmt.Sprintf("%s.txt", time.Now().Format(strings.ReplaceAll(time.RFC1123Z, ":", "_")))
+	logFileName := fmt.Sprintf("BenchmarkNest-%s.txt", time.Now().Format(strings.ReplaceAll(time.RFC1123Z, ":", "_")))
 	f, err := os.OpenFile("./dump/"+logFileName, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0o644)
 	require.NoError(b, err)
 
 	defer f.Close()
 	log.SetOutput(f)
-
+	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		assert.NoError(b, nest_benchmark.BenchmarkNoLog(1))
+		start := time.Now()
+		assert.NoError(b, nest_benchmark.BenchmarkNoLog(bmNumberOfRequests))
+		b.Logf("Request took %v", time.Since(start))
+		b.Logf("i %d | b.N %d", i, b.N)
 	}
 }
 
-func TestGoXRequestes(t *testing.T) {
+func TestGoRequestsPerSecondFor15s(t *testing.T) {
 	log.SetFlags(log.Lmicroseconds)
-	logFileName := fmt.Sprintf("%s.txt", time.Now().Format(strings.ReplaceAll(time.RFC1123Z, ":", "_")))
-	err := os.MkdirAll("./dump", os.ModeTemporary)
-	require.NoError(t, err)
+	logFileName := fmt.Sprintf("GoRequestsPerSecondFor15s-%s.txt", time.Now().Format(strings.ReplaceAll(time.RFC1123Z, ":", "_")))
 	f, err := os.OpenFile("./dump/"+logFileName, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0o644)
 	require.NoError(t, err)
 
 	defer f.Close()
 	log.SetOutput(f)
 
-	totalRequests := 1000
-	require.NoError(t, go_benchmark.Benchmark(totalRequests))
-	require.NoError(t, os.RemoveAll("./temp"))
-
+	var wg sync.WaitGroup
+	wg.Add(seconds)
+	for i := 0; i < seconds; i++ {
+		go func() {
+			defer wg.Done()
+			require.NoError(t, go_benchmark.Benchmark(numberOfRequestsAtOnce))
+		}()
+		time.Sleep(time.Second)
+	}
+	wg.Wait()
 }
 
-func TestNestXRequests(t *testing.T) {
+func TestNestRequestsPerSecondFor15s(t *testing.T) {
+	log.SetFlags(log.Lmicroseconds)
+	logFileName := fmt.Sprintf("NestRequestsPerSecondFor15s-%s.txt", time.Now().Format(strings.ReplaceAll(time.RFC1123Z, ":", "_")))
+	f, err := os.OpenFile("./dump/"+logFileName, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0o644)
+	require.NoError(t, err)
+
+	defer f.Close()
+	log.SetOutput(f)
+
+	var wg sync.WaitGroup
+	wg.Add(seconds)
+	for i := 0; i < seconds; i++ {
+		go func() {
+			defer wg.Done()
+			require.NoError(t, nest_benchmark.Benchmark(numberOfRequestsAtOnce))
+		}()
+		time.Sleep(time.Second)
+	}
+	wg.Wait()
+}
+
+func TestGoRequests(t *testing.T) {
 	log.SetFlags(log.Lmicroseconds)
 	logFileName := fmt.Sprintf("%s.txt", time.Now().Format(strings.ReplaceAll(time.RFC1123Z, ":", "_")))
 	f, err := os.OpenFile("./dump/"+logFileName, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0o644)
@@ -72,6 +112,17 @@ func TestNestXRequests(t *testing.T) {
 	defer f.Close()
 	log.SetOutput(f)
 
-	totalRequests := 1000
-	require.NoError(t, nest_benchmark.Benchmark(totalRequests))
+	require.NoError(t, go_benchmark.Benchmark(numberOfRequestsAtOnce))
+}
+
+func TestNestRequests(t *testing.T) {
+	log.SetFlags(log.Lmicroseconds)
+	logFileName := fmt.Sprintf("%s.txt", time.Now().Format(strings.ReplaceAll(time.RFC1123Z, ":", "_")))
+	f, err := os.OpenFile("./dump/"+logFileName, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0o644)
+	require.NoError(t, err)
+
+	defer f.Close()
+	log.SetOutput(f)
+
+	require.NoError(t, nest_benchmark.Benchmark(numberOfRequestsAtOnce))
 }

benchmark/nest/nest.go

@@ -26,7 +26,7 @@ func doRequest(wg *sync.WaitGroup, host string, employeeID int, errChan chan err
 	defer wg.Done()
 	var err error
 	query := GraphqlQuery{
-		Query:         `query ByEmployeeNumber($employeeNumber: Float!) { byEmployeeNumber(employeeNumber: $employeeNumber) { EmployeeNumber Date WorkInformation { EmployeeNumber Date Shifts { Start End ActualStart ActualEnd RoleCode ShiftCategoryCode _ReferenceId } DaysOff { DayOff { DayOffTypeCode Note _ReferenceId } } } Baselines { Shifts { DayOff { DayOffTypeCode } } BaselineType DaysOff { DayOff { DayOffTypeCode } } FullDayAbsences { FullDayAbsence { AbsenceTypeCode } } } CustomFields { CustomField { FullDayAbsence { AbsenceTypeCode } } } DataVersion FullDayAbsences { FullDayAbsence { AbsenceTypeCode Note _ReferenceId } } } }`,
+		Query:         `query ByEmployeeNumber($employeeNumber: Float!) { byEmployeeNumber(employeeNumber: $employeeNumber) { EmployeeNumber Date WorkInformation { EmployeeNumber Date Shifts { Start End RoleCode } DaysOff { DayOff { DayOffTypeCode Note _ReferenceId } } } Baselines { Shifts { Start End RoleCode } BaselineType DaysOff { DayOff { DayOffTypeCode } } FullDayAbsences { FullDayAbsence { AbsenceTypeCode } } } CustomFields { CustomField { Name } } DataVersion FullDayAbsences { FullDayAbsence { _ReferenceId } } } } `,
 		OperationName: "ByEmployeeNumber",
 		Variables: ByEmployeeNumberQueryVariables{
 			EmployeeNumber: float64(employeeID),

benchmark_report/Report.md

@@ -0,0 +1,742 @@
+# Benchmark results
+I’ve decided to compare same behavior done with 2 different technologies:  Go vs NestJS
+
+## Context
+
+I’ve worked with NestJS in some projects. We build a DeFi platform using NestJS+TypeORM+Postgres and project was pretty good. We had to use some raw SQL queries to perform some special tasks like obtain full network (referral program).
+After working with go since April 2020 I’ve found a super fast tool in terms of: 1- development, 2- clean code, 3- performance and list could continue.
+
+## Description
+
+I have to perform an ETL taks that will be fetching data from AMS xml API, then parse into some readable Struct/Class therefore store into some database. Database here is really important to acquire a high performance fast-response application.
+
+
+## Development spent time
+
+### go
+
+Just using go struct tags I could complete this task in just **4h** in just **5 lines of code** to load **XML** and 45 lines to define my domain entities (structs)
+
+
+**I spent around 3-4 days** working with Nest+ Apollo+MikroORM and I found really tough this task in terms of development spent time. Every time I need to add 1 console.log I had to restart server and it took around 15 and 20 seconds to execute npm start which is nest start I found this command what it does internally is first to compile typescript into javascript therefore run main.js from dist folder.
+
+### nest + graphql
+
+GraphQL api + ETL on Nestjs took me 3-4 days do some pairing with my team mates  and  
+Full restAPI + ETL process Go took me 4h to develop everything without any trouble. 
+I found quite hard using graphQL as I find selected architecture quite hard to comprehend and navigate with decission. I know if you are building a multi-tenant library or service is ok to make do more unit test, to pass mocks and use more interfaces: which is one of the most used things in go.
+
+I also would recommend to read **[this blog](https://betterprogramming.pub/graphql-from-excitement-to-deception-f81f7c95b7cf)** which I found really interesting point of view about graphQL and it's real use  case.
+
+### XML issues
+
+Standard go library is pretty easy to use as you will see in the following example of the benchmark test application
+
+## Performance
+
+### startup
+
+**NestJS** application takes between **15** and **25 seconds** to startup. I’ve created a bash script that counts the whole process since you run npm start until API is reachable
+
+![startup](./assets/1.png)
+
+**go** API server startup takes couple milliseconds to start up
+
+![startup](./assets/2.png)
+
+
+### ETL
+
+For this test I’ve decided to make 1 AMS API request for 1 employee for 1 month
+**nest**
+I’ve made a jest test: first test is to fetch database. We can see it takes around **10850ms**
+![startup](./assets/3.png)
+![startup](./assets/4.png)
+
+**go**
+With go the whole process takes **2613ms**
+![startup](./assets/5.png)
+
+### AMS direct request
+
+I didn’t go further with this comparison as I’ve seen direct request to SITA is not worth. On previous nest test execution we can see how AMS takes 1188ms to retrieve 1 employee’s work information  for 1 day.
+
+## Process 1000 request sent at same time
+
+I’ve created a go project that is responsible to send 1000 request at same time and wait until last response is sent by server to calculate how much time server took to handle 1000 requests at same time for GET operation. Well, in NestJS case it was POST itself because it is graphql  but is a request to retrieve data. I’ve added an operation to write each response into a log file to more or less simulate a real request from a browser client.
+
+
+![startup](./assets/6.png)
+
+As we can see go server  served using fiber framework takes **2874ms** to handle 1000 requests. 
+NestJS graphql took 40612ms to handle all requests. There is a huge difference: **go** is **14.13** times faster `40612/2874` than NestJS+Grahpql
+
+Testing each one isolated we can see that:
+**nest** takes **16417ms** to handle 1000 requests
+![startup](./assets/7.png)
+
+
+
+ **go** takes 1410ms to handle 1000 requests this means go is 11.64 times faster than nest handling requests
+![startup](./assets/8.png)
+
+
+
+## Process 5000 request sent at same time
+
+Testing each one isolated we can see that:
+**nest** takes 75536ms to handle 5000 requests
+![startup](./assets/9.png)
+
+
+ **go** takes 6191ms to handle 5000 requests this means go is 12.200 times faster than nest handling requests
+![startup](./assets/010.png)
+
+
+## 1000 requests per second on 3 seconds duration
+**nest**
+
+![startup](./assets/011.png)
+
+**go**
+
+![startup](./assets/012.png)
+
+
+## 1000 requests per second on 5 seconds duration
+
+**nest** it tooks really long to handle all requests. These are some output results. I had to manually stop the test because It was going to crash my machine.
+
+![startup](./assets/013.png)
+
+
+**go** the first request is just 5 seconds, and then it increases up to 20 seconds due to resources. But handling this traffic with this response times, with 4-6h of development is insane.
+![startup](./assets/014.png)
+
+
+## 1000 requests per second on 10 seconds duration
+
+**nest**
+
+```bash
++---+---------------------------+------------+
+| # | DESCRIPTION               | ELAPSED    |
++---+---------------------------+------------+
+| 0 | nest handle 1000 response | 9842ms !!! |
++---+---------------------------+------------+
+|   | TOTAL                     | 9842MS !!! |
++---+---------------------------+------------+
++---+---------------------------+-------------+
+| # | DESCRIPTION               | ELAPSED     |
++---+---------------------------+-------------+
+| 0 | nest handle 1000 response | 23957ms !!! |
++---+---------------------------+-------------+
+|   | TOTAL                     | 23957MS !!! |
++---+---------------------------+-------------+
++---+---------------------------+-------------+
+| # | DESCRIPTION               | ELAPSED     |
++---+---------------------------+-------------+
+| 0 | nest handle 1000 response | 33049ms !!! |
++---+---------------------------+-------------+
+|   | TOTAL                     | 33049MS !!! |
++---+---------------------------+-------------+
++---+---------------------------+-------------+
+| # | DESCRIPTION               | ELAPSED     |
++---+---------------------------+-------------+
+| 0 | nest handle 1000 response | 34006ms !!! |
++---+---------------------------+-------------+
+|   | TOTAL                     | 34006MS !!! |
++---+---------------------------+-------------+
++---+---------------------------+-------------+
+| # | DESCRIPTION               | ELAPSED     |
++---+---------------------------+-------------+
+| 0 | nest handle 1000 response | 40252ms !!! |
++---+---------------------------+-------------+
+|   | TOTAL                     | 40252MS !!! |
++---+---------------------------+-------------+
++---+---------------------------+-------------+
+| # | DESCRIPTION               | ELAPSED     |
++---+---------------------------+-------------+
+| 0 | nest handle 1000 response | 46852ms !!! |
++---+---------------------------+-------------+
+|   | TOTAL                     | 46852MS !!! |
++---+---------------------------+-------------+
++---+---------------------------+-------------+
+| # | DESCRIPTION               | ELAPSED     |
++---+---------------------------+-------------+
+| 0 | nest handle 1000 response | 47458ms !!! |
++---+---------------------------+-------------+
+|   | TOTAL                     | 47458MS !!! |
++---+---------------------------+-------------+
++---+---------------------------+-------------+
+| # | DESCRIPTION               | ELAPSED     |
++---+---------------------------+-------------+
+| 0 | nest handle 1000 response | 54965ms !!! |
++---+---------------------------+-------------+
+|   | TOTAL                     | 54965MS !!! |
++---+---------------------------+-------------+
++---+---------------------------+-------------+
+| # | DESCRIPTION               | ELAPSED     |
++---+---------------------------+-------------+
+| 0 | nest handle 1000 response | 57436ms !!! |
++---+---------------------------+-------------+
+|   | TOTAL                     | 57436MS !!! |
++---+---------------------------+-------------+
++---+---------------------------+-------------+
+| # | DESCRIPTION               | ELAPSED     |
++---+---------------------------+-------------+
+| 0 | nest handle 1000 response | 59447ms !!! |
++---+---------------------------+-------------+
+|   | TOTAL                     | 59447MS !!! |
++---+---------------------------+-------------+
+PASS
+```
+
+**go**
+```bash
++---+-------------------------+------------+
+| # | DESCRIPTION             | ELAPSED    |
++---+-------------------------+------------+
+| 0 | go handle 1000 response | 7918ms !!! |
++---+-------------------------+------------+
+|   | TOTAL                   | 7918MS !!! |
++---+-------------------------+------------+
++---+-------------------------+------------+
+| # | DESCRIPTION             | ELAPSED    |
++---+-------------------------+------------+
+| 0 | go handle 1000 response | 7595ms !!! |
++---+-------------------------+------------+
+|   | TOTAL                   | 7595MS !!! |
++---+-------------------------+------------+
++---+-------------------------+-------------+
+| # | DESCRIPTION             | ELAPSED     |
++---+-------------------------+-------------+
+| 0 | go handle 1000 response | 14349ms !!! |
++---+-------------------------+-------------+
+|   | TOTAL                   | 14349MS !!! |
++---+-------------------------+-------------+
++---+-------------------------+-------------+
+| # | DESCRIPTION             | ELAPSED     |
++---+-------------------------+-------------+
+| 0 | go handle 1000 response | 14500ms !!! |
++---+-------------------------+-------------+
+|   | TOTAL                   | 14500MS !!! |
++---+-------------------------+-------------+
++---+-------------------------+-------------+
+| # | DESCRIPTION             | ELAPSED     |
++---+-------------------------+-------------+
+| 0 | go handle 1000 response | 13338ms !!! |
++---+-------------------------+-------------+
+|   | TOTAL                   | 13338MS !!! |
++---+-------------------------+-------------+
++---+-------------------------+-------------+
+| # | DESCRIPTION             | ELAPSED     |
++---+-------------------------+-------------+
+| 0 | go handle 1000 response | 12395ms !!! |
++---+-------------------------+-------------+
+|   | TOTAL                   | 12395MS !!! |
++---+-------------------------+-------------+
++---+-------------------------+-------------+
+| # | DESCRIPTION             | ELAPSED     |
++---+-------------------------+-------------+
+| 0 | go handle 1000 response | 12966ms !!! |
++---+-------------------------+-------------+
+|   | TOTAL                   | 12966MS !!! |
++---+-------------------------+-------------+
++---+-------------------------+-------------+
+| # | DESCRIPTION             | ELAPSED     |
++---+-------------------------+-------------+
+| 0 | go handle 1000 response | 16075ms !!! |
++---+-------------------------+-------------+
+|   | TOTAL                   | 16075MS !!! |
++---+-------------------------+-------------+
++---+-------------------------+-------------+
+| # | DESCRIPTION             | ELAPSED     |
++---+-------------------------+-------------+
+| 0 | go handle 1000 response | 13120ms !!! |
++---+-------------------------+-------------+
+|   | TOTAL                   | 13120MS !!! |
++---+-------------------------+-------------+
++---+-------------------------+-------------+
+| # | DESCRIPTION             | ELAPSED     |
++---+-------------------------+-------------+
+| 0 | go handle 1000 response | 12161ms !!! |
++---+-------------------------+-------------+
+|   | TOTAL                   | 12161MS !!! |
++---+-------------------------+-------------+
+PASS
+```
+
+## 1000 requests per second on 15 seconds duration
+
+**nest** it took really long to handle all requests. These are some output results.
+
+```bash
++---+---------------------------+-------------+
+| # | DESCRIPTION               | ELAPSED     |
++---+---------------------------+-------------+
+| 0 | nest handle 1000 response | 17291ms !!! |
++---+---------------------------+-------------+
+|   | TOTAL                     | 17291MS !!! |
++---+---------------------------+-------------+
++---+---------------------------+-------------+
+| # | DESCRIPTION               | ELAPSED     |
++---+---------------------------+-------------+
+| 0 | nest handle 1000 response | 38445ms !!! |
++---+---------------------------+-------------+
+|   | TOTAL                     | 38445MS !!! |
++---+---------------------------+-------------+
++---+---------------------------+-------------+
+| # | DESCRIPTION               | ELAPSED     |
++---+---------------------------+-------------+
+| 0 | nest handle 1000 response | 52656ms !!! |
++---+---------------------------+-------------+
+|   | TOTAL                     | 52656MS !!! |
++---+---------------------------+-------------+
++---+---------------------------+-------------+
+| # | DESCRIPTION               | ELAPSED     |
++---+---------------------------+-------------+
+| 0 | nest handle 1000 response | 80340ms !!! |
++---+---------------------------+-------------+
+|   | TOTAL                     | 80340MS !!! |
++---+---------------------------+-------------+
++---+---------------------------+-------------+
+| # | DESCRIPTION               | ELAPSED     |
++---+---------------------------+-------------+
+| 0 | nest handle 1000 response | 88219ms !!! |
++---+---------------------------+-------------+
+|   | TOTAL                     | 88219MS !!! |
++---+---------------------------+-------------+
++---+---------------------------+-------------+
+| # | DESCRIPTION               | ELAPSED     |
++---+---------------------------+-------------+
+| 0 | nest handle 1000 response | 84223ms !!! |
++---+---------------------------+-------------+
+|   | TOTAL                     | 84223MS !!! |
++---+---------------------------+-------------+
++---+---------------------------+-------------+
+| # | DESCRIPTION               | ELAPSED     |
++---+---------------------------+-------------+
+| 0 | nest handle 1000 response | 95468ms !!! |
++---+---------------------------+-------------+
+|   | TOTAL                     | 95468MS !!! |
++---+---------------------------+-------------+
++---+---------------------------+-------------+
+| # | DESCRIPTION               | ELAPSED     |
++---+---------------------------+-------------+
+| 0 | nest handle 1000 response | 96851ms !!! |
++---+---------------------------+-------------+
+|   | TOTAL                     | 96851MS !!! |
++---+---------------------------+-------------+
++---+---------------------------+-------------+
+| # | DESCRIPTION               | ELAPSED     |
++---+---------------------------+-------------+
+| 0 | nest handle 1000 response | 96769ms !!! |
++---+---------------------------+-------------+
+|   | TOTAL                     | 96769MS !!! |
++---+---------------------------+-------------+
++---+---------------------------+--------------+
+| # | DESCRIPTION               | ELAPSED      |
++---+---------------------------+--------------+
+| 0 | nest handle 1000 response | 100684ms !!! |
++---+---------------------------+--------------+
+|   | TOTAL                     | 100684MS !!! |
++---+---------------------------+--------------+
++---+---------------------------+--------------+
+| # | DESCRIPTION               | ELAPSED      |
++---+---------------------------+--------------+
+| 0 | nest handle 1000 response | 102292ms !!! |
++---+---------------------------+--------------+
+|   | TOTAL                     | 102292MS !!! |
++---+---------------------------+--------------+
++---+---------------------------+-------------+
+| # | DESCRIPTION               | ELAPSED     |
++---+---------------------------+-------------+
+| 0 | nest handle 1000 response | 99374ms !!! |
++---+---------------------------+-------------+
+|   | TOTAL                     | 99374MS !!! |
++---+---------------------------+-------------+
++---+---------------------------+--------------+
+| # | DESCRIPTION               | ELAPSED      |
++---+---------------------------+--------------+
+| 0 | nest handle 1000 response | 103381ms !!! |
++---+---------------------------+--------------+
+|   | TOTAL                     | 103381MS !!! |
++---+---------------------------+--------------+
++---+---------------------------+-------------+
+| # | DESCRIPTION               | ELAPSED     |
++---+---------------------------+-------------+
+| 0 | nest handle 1000 response | 98271ms !!! |
++---+---------------------------+-------------+
+|   | TOTAL                     | 98271MS !!! |
++---+---------------------------+-------------+
++---+---------------------------+--------------+
+| # | DESCRIPTION               | ELAPSED      |
++---+---------------------------+--------------+
+| 0 | nest handle 1000 response | 100401ms !!! |
++---+---------------------------+--------------+
+|   | TOTAL                     | 100401MS !!! |
++---+---------------------------+--------------+
+PASS
+```
+
+**go** the first request is less than 5 seconds, and then it increases up to 20 seconds due to resources. But handling this traffic with this response times, with **4-6h** of development is insane.
+
+```bash
++---+-------------------------+------------+
+| # | DESCRIPTION             | ELAPSED    |
++---+-------------------------+------------+
+| 0 | go handle 1000 response | 4760ms !!! |
++---+-------------------------+------------+
+|   | TOTAL                   | 4760MS !!! |
++---+-------------------------+------------+
++---+-------------------------+------------+
+| # | DESCRIPTION             | ELAPSED    |
++---+-------------------------+------------+
+| 0 | go handle 1000 response | 6252ms !!! |
++---+-------------------------+------------+
+|   | TOTAL                   | 6252MS !!! |
++---+-------------------------+------------+
++---+-------------------------+-------------+
+| # | DESCRIPTION             | ELAPSED     |
++---+-------------------------+-------------+
+| 0 | go handle 1000 response | 10400ms !!! |
++---+-------------------------+-------------+
+|   | TOTAL                   | 10400MS !!! |
++---+-------------------------+-------------+
++---+-------------------------+-------------+
+| # | DESCRIPTION             | ELAPSED     |
++---+-------------------------+-------------+
+| 0 | go handle 1000 response | 18328ms !!! |
++---+-------------------------+-------------+
+|   | TOTAL                   | 18328MS !!! |
++---+-------------------------+-------------+
++---+-------------------------+-------------+
+| # | DESCRIPTION             | ELAPSED     |
++---+-------------------------+-------------+
+| 0 | go handle 1000 response | 17876ms !!! |
++---+-------------------------+-------------+
+|   | TOTAL                   | 17876MS !!! |
++---+-------------------------+-------------+
++---+-------------------------+-------------+
+| # | DESCRIPTION             | ELAPSED     |
++---+-------------------------+-------------+
+| 0 | go handle 1000 response | 20245ms !!! |
++---+-------------------------+-------------+
+|   | TOTAL                   | 20245MS !!! |
++---+-------------------------+-------------+
++---+-------------------------+-------------+
+| # | DESCRIPTION             | ELAPSED     |
++---+-------------------------+-------------+
+| 0 | go handle 1000 response | 18277ms !!! |
++---+-------------------------+-------------+
+|   | TOTAL                   | 18277MS !!! |
++---+-------------------------+-------------+
++---+-------------------------+-------------+
+| # | DESCRIPTION             | ELAPSED     |
++---+-------------------------+-------------+
+| 0 | go handle 1000 response | 17435ms !!! |
++---+-------------------------+-------------+
+|   | TOTAL                   | 17435MS !!! |
++---+-------------------------+-------------+
++---+-------------------------+-------------+
+| # | DESCRIPTION             | ELAPSED     |
++---+-------------------------+-------------+
+| 0 | go handle 1000 response | 18705ms !!! |
++---+-------------------------+-------------+
+|   | TOTAL                   | 18705MS !!! |
++---+-------------------------+-------------+
++---+-------------------------+-------------+
+| # | DESCRIPTION             | ELAPSED     |
++---+-------------------------+-------------+
+| 0 | go handle 1000 response | 19156ms !!! |
++---+-------------------------+-------------+
+|   | TOTAL                   | 19156MS !!! |
++---+-------------------------+-------------+
++---+-------------------------+-------------+
+| # | DESCRIPTION             | ELAPSED     |
++---+-------------------------+-------------+
+| 0 | go handle 1000 response | 18544ms !!! |
++---+-------------------------+-------------+
+|   | TOTAL                   | 18544MS !!! |
++---+-------------------------+-------------+
++---+-------------------------+-------------+
+| # | DESCRIPTION             | ELAPSED     |
++---+-------------------------+-------------+
+| 0 | go handle 1000 response | 18976ms !!! |
++---+-------------------------+-------------+
+|   | TOTAL                   | 18976MS !!! |
++---+-------------------------+-------------+
++---+-------------------------+-------------+
+| # | DESCRIPTION             | ELAPSED     |
++---+-------------------------+-------------+
+| 0 | go handle 1000 response | 24030ms !!! |
++---+-------------------------+-------------+
+|   | TOTAL                   | 24030MS !!! |
++---+-------------------------+-------------+
++---+-------------------------+-------------+
+| # | DESCRIPTION             | ELAPSED     |
++---+-------------------------+-------------+
+| 0 | go handle 1000 response | 18129ms !!! |
++---+-------------------------+-------------+
+|   | TOTAL                   | 18129MS !!! |
++---+-------------------------+-------------+
++---+-------------------------+-------------+
+| # | DESCRIPTION             | ELAPSED     |
++---+-------------------------+-------------+
+| 0 | go handle 1000 response | 17274ms !!! |
++---+-------------------------+-------------+
+|   | TOTAL                   | 17274MS !!! |
++---+-------------------------+-------------+
+PASS
+```
+
+## Other machine
+
+I’ve asked my teammate Jeffer to please run these tests in his machine, to compare different bandwidth latency and machine performance.
+As we saw he couldn’t launch 1000 requests due to nest was overloading then crashed, but also while go was processing he get some error we didn’t have time to inspect further. Anyway, here are results
+
+### 300 requests per second on 15 seconds duration
+
+**nest**
+
+```bash
++---+--------------------------+------------+
+| # | DESCRIPTION              | ELAPSED    |
++---+--------------------------+------------+
+| 0 | nest handle 300 response | 2712ms .!! |
++---+--------------------------+------------+
+|   | TOTAL                    | 2712MS .!! |
++---+--------------------------+------------+
++---+--------------------------+------------+
+| # | DESCRIPTION              | ELAPSED    |
++---+--------------------------+------------+
+| 0 | nest handle 300 response | 4633ms !!! |
++---+--------------------------+------------+
+|   | TOTAL                    | 4633MS !!! |
++---+--------------------------+------------+
++---+--------------------------+------------+
+| # | DESCRIPTION              | ELAPSED    |
++---+--------------------------+------------+
+| 0 | nest handle 300 response | 5346ms !!! |
++---+--------------------------+------------+
+|   | TOTAL                    | 5346MS !!! |
++---+--------------------------+------------+
++---+--------------------------+------------+
+| # | DESCRIPTION              | ELAPSED    |
++---+--------------------------+------------+
+| 0 | nest handle 300 response | 4444ms !!! |
++---+--------------------------+------------+
+|   | TOTAL                    | 4444MS !!! |
++---+--------------------------+------------+
++---+--------------------------+------------+
+| # | DESCRIPTION              | ELAPSED    |
++---+--------------------------+------------+
+| 0 | nest handle 300 response | 4757ms !!! |
++---+--------------------------+------------+
+|   | TOTAL                    | 4757MS !!! |
++---+--------------------------+------------+
++---+--------------------------+------------+
+| # | DESCRIPTION              | ELAPSED    |
++---+--------------------------+------------+
+| 0 | nest handle 300 response | 6506ms !!! |
++---+--------------------------+------------+
+|   | TOTAL                    | 6506MS !!! |
++---+--------------------------+------------+
++---+--------------------------+------------+
+| # | DESCRIPTION              | ELAPSED    |
++---+--------------------------+------------+
+| 0 | nest handle 300 response | 7631ms !!! |
++---+--------------------------+------------+
+|   | TOTAL                    | 7631MS !!! |
++---+--------------------------+------------+
++---+--------------------------+------------+
+| # | DESCRIPTION              | ELAPSED    |
++---+--------------------------+------------+
+| 0 | nest handle 300 response | 7705ms !!! |
++---+--------------------------+------------+
+|   | TOTAL                    | 7705MS !!! |
++---+--------------------------+------------+
++---+--------------------------+------------+
+| # | DESCRIPTION              | ELAPSED    |
++---+--------------------------+------------+
+| 0 | nest handle 300 response | 5511ms !!! |
++---+--------------------------+------------+
+|   | TOTAL                    | 5511MS !!! |
++---+--------------------------+------------+
++---+--------------------------+------------+
+| # | DESCRIPTION              | ELAPSED    |
++---+--------------------------+------------+
+| 0 | nest handle 300 response | 6329ms !!! |
++---+--------------------------+------------+
+|   | TOTAL                    | 6329MS !!! |
++---+--------------------------+------------+
++---+--------------------------+------------+
+| # | DESCRIPTION              | ELAPSED    |
++---+--------------------------+------------+
+| 0 | nest handle 300 response | 7177ms !!! |
++---+--------------------------+------------+
+|   | TOTAL                    | 7177MS !!! |
++---+--------------------------+------------+
++---+--------------------------+-------------+
+| # | DESCRIPTION              | ELAPSED     |
++---+--------------------------+-------------+
+| 0 | nest handle 300 response | 11947ms !!! |
++---+--------------------------+-------------+
+|   | TOTAL                    | 11947MS !!! |
++---+--------------------------+-------------+
++---+--------------------------+-------------+
+| # | DESCRIPTION              | ELAPSED     |
++---+--------------------------+-------------+
+| 0 | nest handle 300 response | 11275ms !!! |
++---+--------------------------+-------------+
+|   | TOTAL                    | 11275MS !!! |
++---+--------------------------+-------------+
++---+--------------------------+------------+
+| # | DESCRIPTION              | ELAPSED    |
++---+--------------------------+------------+
+| 0 | nest handle 300 response | 8611ms !!! |
++---+--------------------------+------------+
+|   | TOTAL                    | 8611MS !!! |
++---+--------------------------+------------+
++---+--------------------------+------------+
+| # | DESCRIPTION              | ELAPSED    |
++---+--------------------------+------------+
+| 0 | nest handle 300 response | 7731ms !!! |
++---+--------------------------+------------+
+|   | TOTAL                    | 7731MS !!! |
++---+--------------------------+------------+
+```
+**go**
+
+```bash
++---+------------------------+-----------+
+| # | DESCRIPTION            | ELAPSED   |
++---+------------------------+-----------+
+| 0 | go handle 300 response | 562ms ... |
++---+------------------------+-----------+
+|   | TOTAL                  | 562MS ... |
++---+------------------------+-----------+
++---+------------------------+-----------+
+| # | DESCRIPTION            | ELAPSED   |
++---+------------------------+-----------+
+| 0 | go handle 300 response | 471ms ... |
++---+------------------------+-----------+
+|   | TOTAL                  | 471MS ... |
++---+------------------------+-----------+
++---+------------------------+-----------+
+| # | DESCRIPTION            | ELAPSED   |
++---+------------------------+-----------+
+| 0 | go handle 300 response | 437ms ... |
++---+------------------------+-----------+
+|   | TOTAL                  | 437MS ... |
++---+------------------------+-----------+
++---+------------------------+-----------+
+| # | DESCRIPTION            | ELAPSED   |
++---+------------------------+-----------+
+| 0 | go handle 300 response | 410ms ... |
++---+------------------------+-----------+
+|   | TOTAL                  | 410MS ... |
++---+------------------------+-----------+
++---+------------------------+-----------+
+| # | DESCRIPTION            | ELAPSED   |
++---+------------------------+-----------+
+| 0 | go handle 300 response | 469ms ... |
++---+------------------------+-----------+
+|   | TOTAL                  | 469MS ... |
++---+------------------------+-----------+
++---+------------------------+-----------+
+| # | DESCRIPTION            | ELAPSED   |
++---+------------------------+-----------+
+| 0 | go handle 300 response | 450ms ... |
++---+------------------------+-----------+
+|   | TOTAL                  | 450MS ... |
++---+------------------------+-----------+
++---+------------------------+-----------+
+| # | DESCRIPTION            | ELAPSED   |
++---+------------------------+-----------+
+| 0 | go handle 300 response | 494ms ... |
++---+------------------------+-----------+
+|   | TOTAL                  | 494MS ... |
++---+------------------------+-----------+
++---+------------------------+-----------+
+| # | DESCRIPTION            | ELAPSED   |
++---+------------------------+-----------+
+| 0 | go handle 300 response | 616ms ... |
++---+------------------------+-----------+
+|   | TOTAL                  | 616MS ... |
++---+------------------------+-----------+
++---+------------------------+-----------+
+| # | DESCRIPTION            | ELAPSED   |
++---+------------------------+-----------+
+| 0 | go handle 300 response | 541ms ... |
++---+------------------------+-----------+
+|   | TOTAL                  | 541MS ... |
++---+------------------------+-----------+
++---+------------------------+------------+
+| # | DESCRIPTION            | ELAPSED    |
++---+------------------------+------------+
+| 0 | go handle 300 response | 1398ms ..! |
++---+------------------------+------------+
+|   | TOTAL                  | 1398MS ..! |
++---+------------------------+------------+
++---+------------------------+-----------+
+| # | DESCRIPTION            | ELAPSED   |
++---+------------------------+-----------+
+| 0 | go handle 300 response | 803ms ... |
++---+------------------------+-----------+
+|   | TOTAL                  | 803MS ... |
++---+------------------------+-----------+
++---+------------------------+-----------+
+| # | DESCRIPTION            | ELAPSED   |
++---+------------------------+-----------+
+| 0 | go handle 300 response | 688ms ... |
++---+------------------------+-----------+
+|   | TOTAL                  | 688MS ... |
++---+------------------------+-----------+
++---+------------------------+-----------+
+| # | DESCRIPTION            | ELAPSED   |
++---+------------------------+-----------+
+| 0 | go handle 300 response | 626ms ... |
++---+------------------------+-----------+
+|   | TOTAL                  | 626MS ... |
++---+------------------------+-----------+
++---+------------------------+-----------+
+| # | DESCRIPTION            | ELAPSED   |
++---+------------------------+-----------+
+| 0 | go handle 300 response | 580ms ... |
++---+------------------------+-----------+
+|   | TOTAL                  | 580MS ... |
++---+------------------------+-----------+
++---+------------------------+-----------+
+| # | DESCRIPTION            | ELAPSED   |
++---+------------------------+-----------+
+| 0 | go handle 300 response | 449ms ... |
++---+------------------------+-----------+
+|   | TOTAL                  | 449MS ... |
++---+------------------------+-----------+
+```
+If we compare results, for example taking last execution time we can see nest takes **7731ms** and go takes just **449ms**. If we divide `7731/449` we get **17.218** which means time difference between two process.
+
+## Summary
+
+As we can see comparing these results that go is the most efficient language to handle this kind of behavior. I would recommend to sit down and rethink about software needs, integrity, scalability, etc…
+
+## go benchmark
+
+Using go test -bench mode we are able to test both servers at same time and compare results. We can see go test can handle 1 operation in **4.42ms** which is a great response time while nest is 3 times more **13.94ms**
+![startup](./assets/015.png)
+
+
+## Database used
+
+I’ve hardly cannot finish developing with Nest as I have to do a lot of manual class mapping through 3 different classes. I couldn’t spent more time to compare databases as I think could be something to discuss later after some previous requirements has been defined.
+
+### Local Mongo
+
+I only had time to test against mongo as it is the fastest to store everything into single table without having to connect to cloud.
+
+## Source code
+
+Nest https://github.com/urko-iberia/ess-etl-nest 
+Go https://github.com/urko-iberia/ess-etl-go

cmd/etl/main.go

@@ -2,47 +2,66 @@ package main
 
 import (
 	"context"
-	"fmt"
+	"flag"
 	"log"
+	"net/http"
 	_ "net/http/pprof"
 	"os"
 	"os/signal"
-	"sync"
+	"runtime/pprof"
+	"runtime/trace"
 	"syscall"
 
 	"gitea.urkob.com/urko/crono"
 	"gitea.urkob.com/urko/ess-etl-go/config"
+	"gitea.urkob.com/urko/ess-etl-go/internal/etl"
 	"gitea.urkob.com/urko/ess-etl-go/internal/request"
 	"gitea.urkob.com/urko/ess-etl-go/internal/xml_loader"
 	"gitea.urkob.com/urko/ess-etl-go/pkg/adapter/repository/mongodb/employee_wi"
-	"gitea.urkob.com/urko/ess-etl-go/pkg/domain"
 
 	"go.mongodb.org/mongo-driver/mongo"
 	"go.mongodb.org/mongo-driver/mongo/options"
 )
 
-// var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
+var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
+var traceflag = flag.Bool("trace", false, "write trace to file")
 
 func main() {
-	// flag.Parse()
-	// if *cpuprofile != "" {
-	// 	f, err := os.Create(*cpuprofile)
-	// 	if err != nil {
-	// 		log.Fatal(err)
-	// 	}
-	// 	pprof.StartCPUProfile(f)
-	// 	defer pprof.StopCPUProfile()
-	// }
+	flag.Parse()
+	if *cpuprofile != "" {
+		f, err := os.Create(*cpuprofile)
+		if err != nil {
+			log.Fatal(err)
+		}
+		pprof.StartCPUProfile(f)
+		defer pprof.StopCPUProfile()
 
-	// // Add pprof endpoints
-	// go func() {
-	// 	log.Println(http.ListenAndServe("localhost:6060", nil))
-	// }()
+	}
+
+	// Add pprof endpoints
+	go func() {
+		if *cpuprofile != "" {
+			log.Println(http.ListenAndServe("localhost:6060", nil))
+		}
+	}()
+
+	if *traceflag {
+		log.Println("trace on")
+		trace.Start(os.Stdout)
+	}
+
+	defer func() {
+		if *traceflag {
+			trace.Stop()
+		}
+	}()
 
 	cr := crono.New()
-	defer cr.Table()
-	cfg := config.NewConfig(".env")
+	if !*traceflag {
+		defer cr.Table()
+	}
 
+	cfg := config.NewConfig(".env")
 	ctx := context.Background()
 
 	dbOpts := options.Client()
@@ -63,62 +82,24 @@ func main() {
 	if err = employeeWICollection.Drop(ctx); err != nil {
 		log.Fatalln("employeeWICollection.Drop", err)
 	}
-	professionalRepo := employee_wi.NewRepo(employeeWICollection)
+	repo := employee_wi.NewRepo(employeeWICollection)
 
 	r := request.NewRequestService(cfg.AmsApi, cfg.AmsApiKey)
 
-	employeeIDList := cfg.EmployeeIdList
-
 	ewiLoader := xml_loader.NewEmployeeWILoader(r)
 	from, to := "2023-01-01", "2023-01-31"
 	cr.MarkAndRestart("dependencies loaded")
-	ctx, cancel := context.WithCancel(signalContext(context.Background()))
-	errChan := make(chan error, 1)
-	ewiChan := make(chan []domain.EmployeeWorkInformation, len(employeeIDList))
-	wg := &sync.WaitGroup{}
-	wg.Add(1)
 
-	go func() {
-		defer wg.Done()
-		for _, v := range employeeIDList {
-			wg.Add(1)
-			go func(v string) {
-				cr.Restart()
-				defer wg.Done()
-				wi, err := ewiLoader.LoadEmployee(v, from, to)
-				if err != nil {
-					errChan <- err
-					return
-				}
-				ewiChan <- wi
-				cr.MarkAndRestart(fmt.Sprintf("ewiLoader.LoadEmployee | %s | from: %s to: %s", v, from, to))
-			}(v)
-		}
-	}()
+	etlLoader := etl.New(ewiLoader, repo)
+	err = etlLoader.FanOut(ctx, cfg.EmployeeIdList, from, to)
+	// err = etlLoader.FanOutV2(ctx, cfg.EmployeeIdList, from, to)
+	// err = etlLoader.FanOut2(ctx, cfg.EmployeeIdList, from, to)
+	// err = etlLoader.Main(signalContext(ctx), cr, cfg.EmployeeIdList, from, to)
+	if err != nil {
+		log.Fatalln("etlLoader.FanOut", err)
+	}
+	cr.MarkAndRestart("FanOut")
 
-	go func() {
-		if err := <-errChan; err != nil {
-			log.Fatalln("error while process", err)
-			cancel()
-		}
-	}()
-
-	go func() {
-		for v := range ewiChan {
-			log.Println("len v", len(v))
-			err := professionalRepo.InsertMany(ctx, v)
-			if err != nil {
-				errChan <- err
-				return
-			}
-			cr.MarkAndRestart(fmt.Sprintf("database inserted: %d", len(v)))
-		}
-
-		log.Println("cancel")
-		errChan <- nil
-	}()
-	wg.Wait()
-	// <-ctx.Done()
 	log.Println("gracefully shutdown")
 }
 

cmd/server/main.go

@@ -2,14 +2,18 @@ package main
 
 import (
 	"context"
+	"flag"
 	"log"
+	"net/http"
 	"os"
 	"os/signal"
+	"runtime/pprof"
+	"runtime/trace"
 	"syscall"
 
 	"gitea.urkob.com/urko/crono"
 	"gitea.urkob.com/urko/ess-etl-go/config"
-	"gitea.urkob.com/urko/ess-etl-go/internal/api/http"
+	apihttp "gitea.urkob.com/urko/ess-etl-go/internal/api/http"
 	"gitea.urkob.com/urko/ess-etl-go/internal/services"
 	"gitea.urkob.com/urko/ess-etl-go/pkg/adapter/repository/mongodb/employee_wi"
 
@@ -17,7 +21,36 @@ import (
 	"go.mongodb.org/mongo-driver/mongo/options"
 )
 
+var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
+var traceflag = flag.String("trace", "", "write trace to file")
+
 func main() {
+	flag.Parse()
+	if *cpuprofile != "" {
+		f, err := os.Create(*cpuprofile)
+		if err != nil {
+			log.Fatal(err)
+		}
+		pprof.StartCPUProfile(f)
+		defer pprof.StopCPUProfile()
+	}
+
+	// Add pprof endpoints
+	go func() {
+		if *cpuprofile != "" {
+			log.Println(http.ListenAndServe("localhost:6060", nil))
+		}
+	}()
+
+	if *traceflag != "" {
+		log.Println("trace on")
+		trace.Start(os.Stdout)
+		defer func() {
+			log.Println("on stop")
+			trace.Stop()
+		}()
+	}
+
 	cr := crono.New()
 	defer cr.Table()
 	cfg := config.NewConfig(".env")
@@ -41,13 +74,12 @@ func main() {
 	employeeWICollection := client.Database(cfg.DbName).Collection(cfg.EmployeeWorkInformationCollection)
 	professionalRepo := employee_wi.NewRepo(employeeWICollection)
 
-	restServer := http.
+	restServer := apihttp.
 		NewRestServer(cfg, cr).
 		WithEmployeeWIHandler(services.NewEmployeeWIService(ctx, professionalRepo)).
 		WithAMSHander()
 
 	cr.MarkAndRestart("dependencies loaded")
-	log.Println(cfg)
 	go func() {
 		if err = restServer.Start(cfg.ApiPort, ""); err != nil {
 			log.Fatalln("restServer.Start", err)

go.mod

@@ -5,6 +5,7 @@ go 1.19
 require (
 	gitea.urkob.com/urko/crono v0.0.0-20230405153202-0554f3e53a4c
 	gitea.urkob.com/urko/go-root-dir v0.0.0-20230311113851-2f6d4355888a
+	github.com/docker/go-units v0.5.0
 	github.com/gofiber/fiber/v2 v2.43.0
 	github.com/joho/godotenv v1.5.1
 	github.com/kelseyhightower/envconfig v1.4.0

go.sum

@@ -7,6 +7,8 @@ github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHG
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
+github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/gofiber/fiber/v2 v2.43.0 h1:yit3E4kHf178B60p5CQBa/3v+WVuziWMa/G2ZNyLJB0=
 github.com/gofiber/fiber/v2 v2.43.0/go.mod h1:mpS1ZNE5jU+u+BA4FbM+KKnUzJ4wzTK+FT2tG3tU+6I=
 github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=

internal/etl/etl.go

@@ -0,0 +1,227 @@
+package etl
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"runtime"
+	"sync"
+
+	"gitea.urkob.com/urko/crono"
+	"gitea.urkob.com/urko/ess-etl-go/internal/xml_loader"
+	"gitea.urkob.com/urko/ess-etl-go/pkg/adapter/repository/mongodb/employee_wi"
+	"gitea.urkob.com/urko/ess-etl-go/pkg/domain"
+)
+
+type Etl struct {
+	ewiLoader xml_loader.EmployeeWILoader
+	repo      *employee_wi.Repo
+}
+
+func New(ewiLoader xml_loader.EmployeeWILoader, repo *employee_wi.Repo) *Etl {
+	return &Etl{
+		ewiLoader: ewiLoader,
+		repo:      repo,
+	}
+}
+
+func (etl *Etl) FanOut(ctx context.Context, employeeNumber []string, from, to string) error {
+	g := runtime.GOMAXPROCS(0)
+	var wg sync.WaitGroup
+	wg.Add(g)
+
+	employeeWIChan := make(chan []domain.EmployeeWorkInformation, g)
+	errChan := make(chan error, 1)
+
+	for i := 0; i < g; i++ {
+		go func() {
+			defer func() {
+				wg.Done()
+			}()
+			for v := range employeeWIChan {
+				func() {
+					err := etl.repo.InsertMany(ctx, v)
+					if err != nil {
+						errChan <- err
+						return
+					}
+				}()
+			}
+		}()
+	}
+	go func() {
+		var wg2 sync.WaitGroup
+		wg2.Add(len(employeeNumber))
+		for i := range employeeNumber {
+			go func(v string) {
+				defer wg2.Done()
+				wi, err := etl.ewiLoader.LoadEmployee(v, from, to)
+				if err != nil {
+					log.Println("err", err)
+					errChan <- err
+					return
+				}
+				employeeWIChan <- wi
+			}(employeeNumber[i])
+		}
+		wg2.Wait()
+		close(employeeWIChan)
+	}()
+
+	wg.Wait()
+	errChan <- nil
+	return <-errChan
+}
+
+func (etl *Etl) FanOutV2(ctx context.Context, employeeNumber []string, from, to string) error {
+	employeeWIChan := make(chan []domain.EmployeeWorkInformation, len(employeeNumber))
+	errChan := make(chan error, 1)
+
+	go func() {
+		for {
+			select {
+			case v, ok := <-employeeWIChan:
+				if !ok {
+					errChan <- nil
+					return
+				}
+				err := etl.repo.InsertMany(ctx, v)
+				if err != nil {
+					errChan <- err
+					return
+				}
+			}
+		}
+	}()
+
+	go func() {
+		var wg sync.WaitGroup
+		wg.Add(len(employeeNumber))
+		for i := range employeeNumber {
+			go func(v string) {
+				defer wg.Done()
+				wi, err := etl.ewiLoader.LoadEmployee(v, from, to)
+				if err != nil {
+					log.Println("err", err)
+					errChan <- err
+					return
+				}
+				employeeWIChan <- wi
+			}(employeeNumber[i])
+		}
+		wg.Wait()
+		close(employeeWIChan)
+	}()
+
+	return <-errChan
+}
+
+func (etl *Etl) FanOut2(ctx context.Context, employeeNumber []string, from, to string) error {
+	employeeWIChan := make(chan []domain.EmployeeWorkInformation, len(employeeNumber))
+	xmlChan := make(chan []byte, len(employeeNumber))
+	errChan := make(chan error, 1)
+
+	go func() {
+		for {
+			select {
+			case v, ok := <-employeeWIChan:
+				if !ok {
+					errChan <- nil
+					return
+				}
+				err := etl.repo.InsertMany(ctx, v)
+				if err != nil {
+					errChan <- err
+					return
+				}
+			}
+		}
+	}()
+
+	go func() {
+		for {
+			select {
+			case bts, ok := <-xmlChan:
+				if !ok {
+					close(employeeWIChan)
+					return
+				}
+				wi, err := xml_loader.GoLoadFromXML(bts)
+				if err != nil {
+					errChan <- err
+					return
+				}
+				employeeWIChan <- wi
+			}
+		}
+	}()
+
+	go func() {
+		var wg sync.WaitGroup
+		wg.Add(len(employeeNumber))
+		for i := range employeeNumber {
+			go func(v string) {
+				defer wg.Done()
+				bts, err := etl.ewiLoader.GoLoadEmployee(v, from, to)
+				if err != nil {
+					errChan <- err
+					return
+				}
+				xmlChan <- bts
+			}(employeeNumber[i])
+		}
+		wg.Wait()
+		close(xmlChan)
+	}()
+	return <-errChan
+}
+
+func (etl *Etl) Main(ctx context.Context, cr *crono.Crono, employeeNumber []string, from, to string) error {
+	ctx, cancel := context.WithCancel(ctx)
+
+	errChan := make(chan error, 1)
+	ewiChan := make(chan []domain.EmployeeWorkInformation, len(employeeNumber))
+	var wg sync.WaitGroup
+	wg.Add(1 + len(employeeNumber))
+
+	go func() {
+		defer wg.Done()
+		for _, v := range employeeNumber {
+			go func(v string) {
+				cr.Restart()
+				defer wg.Done()
+				wi, err := etl.ewiLoader.LoadEmployee(v, from, to)
+				if err != nil {
+					errChan <- err
+					return
+				}
+				ewiChan <- wi
+				cr.MarkAndRestart(fmt.Sprintf("ewiLoader.LoadEmployee | %s | from: %s to: %s", v, from, to))
+			}(v)
+		}
+	}()
+
+	go func() {
+		if err := <-errChan; err != nil {
+			log.Fatalln("error while process", err)
+			cancel()
+			return
+		}
+	}()
+
+	go func() {
+		for v := range ewiChan {
+			// log.Println("len v", len(v))
+			err := etl.repo.InsertMany(ctx, v)
+			if err != nil {
+				errChan <- err
+				return
+			}
+			cr.MarkAndRestart(fmt.Sprintf("database inserted: %d", len(v)))
+		}
+		errChan <- nil
+	}()
+
+	wg.Wait()
+	return nil
+}

internal/etl/etl_benchmark_test.go

@@ -0,0 +1,267 @@
+package etl_test
+
+import (
+	"context"
+	"log"
+	"os"
+	"runtime/trace"
+	"testing"
+
+	"gitea.urkob.com/urko/crono"
+	"gitea.urkob.com/urko/ess-etl-go/config"
+	"gitea.urkob.com/urko/ess-etl-go/internal/etl"
+	"gitea.urkob.com/urko/ess-etl-go/internal/request"
+	"gitea.urkob.com/urko/ess-etl-go/internal/xml_loader"
+	"gitea.urkob.com/urko/ess-etl-go/pkg/adapter/repository/mongodb/employee_wi"
+	"github.com/stretchr/testify/require"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+)
+
+const traceFlag = false
+
+func BenchmarkETLFanout(b *testing.B) {
+	if traceFlag {
+		trace.Start(os.Stdout)
+	}
+
+	defer func() {
+		if traceFlag {
+			trace.Stop()
+		}
+	}()
+	cfg := config.NewConfig(".env")
+	ctx := context.Background()
+
+	dbOpts := options.Client()
+	dbOpts.ApplyURI(cfg.DbAddress)
+
+	client, err := mongo.NewClient(dbOpts)
+	require.NoError(b, err, "mongo.NewClient")
+
+	require.NoError(b, client.Connect(ctx), "client.Connect")
+
+	employeeWICollection := client.Database(cfg.DbName).Collection(cfg.EmployeeWorkInformationCollection)
+	require.NoError(b, employeeWICollection.Drop(ctx), "employeeWICollection.Drop")
+	repo := employee_wi.NewRepo(employeeWICollection)
+
+	r := request.NewRequestService(cfg.AmsApi, cfg.AmsApiKey)
+
+	ewiLoader := xml_loader.NewEmployeeWILoader(r)
+	from, to := "2023-01-01", "2023-01-31"
+	etlLoader := etl.New(ewiLoader, repo)
+
+	// err = etlLoader.Main(signalContext(ctx), cr, cfg.EmployeeIdList, from, to)
+
+	for i := 0; i < b.N; i++ {
+		require.NoError(b, etlLoader.FanOut(ctx, cfg.EmployeeIdList, from, to))
+	}
+}
+
+func BenchmarkETLFanout2(b *testing.B) {
+	if traceFlag {
+		trace.Start(os.Stdout)
+	}
+
+	defer func() {
+		if traceFlag {
+			trace.Stop()
+		}
+	}()
+	cfg := config.NewConfig(".env")
+	ctx := context.Background()
+
+	dbOpts := options.Client()
+	dbOpts.ApplyURI(cfg.DbAddress)
+
+	client, err := mongo.NewClient(dbOpts)
+	require.NoError(b, err, "mongo.NewClient")
+
+	require.NoError(b, client.Connect(ctx), "client.Connect")
+
+	employeeWICollection := client.Database(cfg.DbName).Collection(cfg.EmployeeWorkInformationCollection)
+	require.NoError(b, employeeWICollection.Drop(ctx), "employeeWICollection.Drop")
+	repo := employee_wi.NewRepo(employeeWICollection)
+
+	r := request.NewRequestService(cfg.AmsApi, cfg.AmsApiKey)
+
+	ewiLoader := xml_loader.NewEmployeeWILoader(r)
+	from, to := "2023-01-01", "2023-01-31"
+	etlLoader := etl.New(ewiLoader, repo)
+
+	for i := 0; i < b.N; i++ {
+		require.NoError(b, etlLoader.FanOut2(ctx, cfg.EmployeeIdList, from, to))
+	}
+}
+
+func BenchmarkETLMain(b *testing.B) {
+	if traceFlag {
+		trace.Start(os.Stdout)
+	}
+
+	defer func() {
+		if traceFlag {
+			trace.Stop()
+		}
+	}()
+	cfg := config.NewConfig(".env")
+	ctx := context.Background()
+
+	dbOpts := options.Client()
+	dbOpts.ApplyURI(cfg.DbAddress)
+
+	client, err := mongo.NewClient(dbOpts)
+	require.NoError(b, err, "mongo.NewClient")
+
+	require.NoError(b, client.Connect(ctx), "client.Connect")
+
+	employeeWICollection := client.Database(cfg.DbName).Collection(cfg.EmployeeWorkInformationCollection)
+	require.NoError(b, employeeWICollection.Drop(ctx), "employeeWICollection.Drop")
+	repo := employee_wi.NewRepo(employeeWICollection)
+
+	r := request.NewRequestService(cfg.AmsApi, cfg.AmsApiKey)
+
+	ewiLoader := xml_loader.NewEmployeeWILoader(r)
+	from, to := "2023-01-01", "2023-01-31"
+	etlLoader := etl.New(ewiLoader, repo)
+
+	for i := 0; i < b.N; i++ {
+		require.NoError(b, etlLoader.Main(ctx, crono.New(), cfg.EmployeeIdList, from, to))
+	}
+}
+
+func TestETLFanout(t *testing.T) {
+	cr := crono.New()
+	if traceFlag {
+		trace.Start(os.Stdout)
+	} else {
+		defer cr.Table()
+	}
+
+	defer func() {
+		if traceFlag {
+			trace.Stop()
+		}
+	}()
+	cfg := config.NewConfig(".env")
+	ctx := context.Background()
+
+	dbOpts := options.Client()
+	dbOpts.ApplyURI(cfg.DbAddress)
+
+	client, err := mongo.NewClient(dbOpts)
+	require.NoError(t, err, "mongo.NewClient")
+
+	require.NoError(t, client.Connect(ctx), "client.Connect")
+
+	employeeWICollection := client.Database(cfg.DbName).Collection(cfg.EmployeeWorkInformationCollection)
+	require.NoError(t, employeeWICollection.Drop(ctx), "employeeWICollection.Drop")
+	repo := employee_wi.NewRepo(employeeWICollection)
+
+	r := request.NewRequestService(cfg.AmsApi, cfg.AmsApiKey)
+
+	ewiLoader := xml_loader.NewEmployeeWILoader(r)
+	from, to := "2023-01-01", "2023-01-31"
+	etlLoader := etl.New(ewiLoader, repo)
+
+	require.NoError(t, etlLoader.FanOut(ctx, cfg.EmployeeIdList, from, to))
+}
+
+func TestETLFanOut2(t *testing.T) {
+	cr := crono.New()
+	if traceFlag {
+		trace.Start(os.Stdout)
+	} else {
+		defer cr.Table()
+	}
+
+	defer func() {
+		if traceFlag {
+			trace.Stop()
+		}
+	}()
+	cfg := config.NewConfig(".env")
+	ctx := context.Background()
+
+	dbOpts := options.Client()
+	dbOpts.ApplyURI(cfg.DbAddress)
+
+	client, err := mongo.NewClient(dbOpts)
+	require.NoError(t, err, "mongo.NewClient")
+
+	require.NoError(t, client.Connect(ctx), "client.Connect")
+
+	employeeWICollection := client.Database(cfg.DbName).Collection(cfg.EmployeeWorkInformationCollection)
+	require.NoError(t, employeeWICollection.Drop(ctx), "employeeWICollection.Drop")
+	repo := employee_wi.NewRepo(employeeWICollection)
+
+	r := request.NewRequestService(cfg.AmsApi, cfg.AmsApiKey)
+	ewiLoader := xml_loader.NewEmployeeWILoader(r)
+	from, to := "2023-01-01", "2023-01-31"
+	etlLoader := etl.New(ewiLoader, repo)
+
+	require.NoError(t, etlLoader.FanOut2(ctx, cfg.EmployeeIdList, from, to))
+}
+
+func TestETLMain(t *testing.T) {
+	cr := crono.New()
+	if traceFlag {
+		trace.Start(os.Stdout)
+	} else {
+		defer cr.Table()
+	}
+
+	defer func() {
+		if traceFlag {
+			trace.Stop()
+		}
+	}()
+	cfg := config.NewConfig(".env")
+	ctx := context.Background()
+
+	dbOpts := options.Client()
+	dbOpts.ApplyURI(cfg.DbAddress)
+
+	client, err := mongo.NewClient(dbOpts)
+	require.NoError(t, err, "mongo.NewClient")
+
+	require.NoError(t, client.Connect(ctx), "client.Connect")
+
+	employeeWICollection := client.Database(cfg.DbName).Collection(cfg.EmployeeWorkInformationCollection)
+	require.NoError(t, employeeWICollection.Drop(ctx), "employeeWICollection.Drop")
+	repo := employee_wi.NewRepo(employeeWICollection)
+
+	r := request.NewRequestService(cfg.AmsApi, cfg.AmsApiKey)
+
+	ewiLoader := xml_loader.NewEmployeeWILoader(r)
+	from, to := "2023-01-01", "2023-01-31"
+	etlLoader := etl.New(ewiLoader, repo)
+	require.NoError(t, etlLoader.Main(ctx, cr, cfg.EmployeeIdList, from, to))
+}
+
+func TestLoad(t *testing.T) {
+	cfg := config.NewConfig(".env")
+	ctx := context.Background()
+
+	dbOpts := options.Client()
+	dbOpts.ApplyURI(cfg.DbAddress)
+
+	client, err := mongo.NewClient(dbOpts)
+	require.NoError(t, err, "mongo.NewClient")
+
+	require.NoError(t, client.Connect(ctx), "client.Connect")
+
+	employeeWICollection := client.Database(cfg.DbName).Collection(cfg.EmployeeWorkInformationCollection)
+	if err = employeeWICollection.Drop(ctx); err != nil {
+		log.Fatalln("employeeWICollection.Drop", err)
+	}
+	repo := employee_wi.NewRepo(employeeWICollection)
+
+	r := request.NewRequestService(cfg.AmsApi, cfg.AmsApiKey)
+
+	ewiLoader := xml_loader.NewEmployeeWILoader(r)
+	from, to := "2023-01-01", "2023-01-31"
+	e := etl.New(ewiLoader, repo)
+
+	require.NoError(t, e.FanOut(ctx, cfg.EmployeeIdList, from, to))
+}

internal/request/request.go

@@ -1,7 +1,6 @@
 package request
 
 import (
-	"bytes"
 	"fmt"
 	"io"
 	"net/http"
@@ -38,12 +37,12 @@ func getPayload(employeeIDList []string) (string, error) {
 	return employees.String(), nil
 }
 
-func (r RequestService) EmployeeWorkInformation(employeeIDList []string, from, to string) (io.Reader, error) {
+func (r RequestService) EmployeeWorkInformation(data *[]byte, employeeIDList []string, from, to string) error {
 	url := r.api + "/EmployeeWorkInformation/Search/" + from + "/" + to + "/"
 
 	stringPayload, err := getPayload(employeeIDList)
 	if err != nil {
-		return nil, fmt.Errorf("getPayload: %s", err)
+		return fmt.Errorf("getPayload: %s", err)
 	}
 
 	payload := strings.NewReader(stringPayload)
@@ -51,7 +50,7 @@ func (r RequestService) EmployeeWorkInformation(employeeIDList []string, from, t
 	req, err := http.NewRequest("POST", url, payload)
 
 	if err != nil {
-		return nil, fmt.Errorf("http.NewRequest: %s", err)
+		return fmt.Errorf("http.NewRequest: %s", err)
 	}
 	req.Header.Add("Cache-Control", "no-cache")
 	req.Header.Add("Authorization", r.apiKey)
@@ -59,16 +58,30 @@ func (r RequestService) EmployeeWorkInformation(employeeIDList []string, from, t
 
 	res, err := client.Do(req)
 	if err != nil {
-		return nil, fmt.Errorf("client.Do: %s", err)
+		return fmt.Errorf("client.Do: %s", err)
 	}
 	defer res.Body.Close()
 
-	body, err := io.ReadAll(res.Body)
-	if err != nil {
-		return nil, fmt.Errorf("ioutil.ReadAll: %s", err)
+	for {
+		if len(*data) == cap(*data) {
+			// Add more capacity (let append pick how much).
+			*data = append(*data, 0)[:len(*data)]
+		}
+		n, err := res.Body.Read((*data)[len(*data):cap(*data)])
+		*data = (*data)[:len(*data)+n]
+		if err != nil {
+			if err == io.EOF {
+				err = nil
+			}
+			return err
+		}
 	}
 
-	//log.Println("readed", string(body))
+	// data, err = io.ReadAll(res.Body)
+	// if err != nil {
+	// 	return fmt.Errorf("ioutil.ReadAll: %s", err)
+	// }
 
-	return bytes.NewReader(body), nil
+	//log.Println("readed", string(body))
+	return nil
 }

internal/xml_loader/employee_wi.go

@@ -3,10 +3,10 @@ package xml_loader
 import (
 	"encoding/xml"
 	"fmt"
-	"io"
 
 	"gitea.urkob.com/urko/ess-etl-go/internal/request"
 	"gitea.urkob.com/urko/ess-etl-go/pkg/domain"
+	"github.com/docker/go-units"
 )
 
 type EmployeeWILoader struct {
@@ -18,31 +18,56 @@ func NewEmployeeWILoader(r request.RequestService) EmployeeWILoader {
 }
 
 func (e EmployeeWILoader) LoadEmployeeList(employeeIDList []string, from, to string) ([]domain.EmployeeWorkInformation, error) {
-	reader, err := e.r.EmployeeWorkInformation(employeeIDList, from, to)
+	xmlBts := make([]byte, 0, units.MiB*5)
+	err := e.r.EmployeeWorkInformation(&xmlBts, employeeIDList, from, to)
 	if err != nil {
 		return nil, fmt.Errorf("r.EmployeeWorkInformation: %s", err)
 	}
+	if len(xmlBts) <= 0 {
+		return nil, fmt.Errorf("couldn't load xml ")
+	}
 
-	return loadFromXML(reader)
+	return loadFromXML(xmlBts)
 }
 
 func (e EmployeeWILoader) LoadEmployee(employeeID, from, to string) ([]domain.EmployeeWorkInformation, error) {
 	employeeIDList := []string{employeeID}
-
-	reader, err := e.r.EmployeeWorkInformation(employeeIDList, from, to)
+	xmlBts := make([]byte, 0, units.MiB*5)
+	err := e.r.EmployeeWorkInformation(&xmlBts, employeeIDList, from, to)
 	if err != nil {
 		return nil, fmt.Errorf("r.EmployeeWorkInformation: %s", err)
 	}
-
-	return loadFromXML(reader)
+	if len(xmlBts) <= 0 {
+		return nil, fmt.Errorf("couldn't load xml ")
+	}
+	return loadFromXML(xmlBts)
 }
 
-func loadFromXML(xmlFile io.Reader) ([]domain.EmployeeWorkInformation, error) {
+func loadFromXML(xmlFile []byte) ([]domain.EmployeeWorkInformation, error) {
 	var awi domain.ArrayOfEmployeeWorkInformation
-
-	if err := xml.NewDecoder(xmlFile).Decode(&awi); err != nil {
-		return nil, fmt.Errorf("xml.NewDecoder.Decode: %s", err)
+	if err := xml.Unmarshal(xmlFile, &awi); err != nil {
+		return nil, fmt.Errorf("xml.Unmarshal: %s", err)
+	}
+	return awi.EmployeeWorkInfos, nil
+}
+
+func (e EmployeeWILoader) GoLoadEmployee(employeeID, from, to string) ([]byte, error) {
+	employeeIDList := []string{employeeID}
+	xmlBts := make([]byte, 0, units.MiB*5)
+	err := e.r.EmployeeWorkInformation(&xmlBts, employeeIDList, from, to)
+	if err != nil {
+		return nil, fmt.Errorf("r.EmployeeWorkInformation: %s", err)
+	}
+	if len(xmlBts) <= 0 {
+		return nil, fmt.Errorf("couldn't load xml ")
+	}
+	return xmlBts, nil
+}
+
+func GoLoadFromXML(xmlFile []byte) ([]domain.EmployeeWorkInformation, error) {
+	var awi domain.ArrayOfEmployeeWorkInformation
+	if err := xml.Unmarshal(xmlFile, &awi); err != nil {
+		return nil, fmt.Errorf("xml.Unmarshal: %s", err)
 	}
-
 	return awi.EmployeeWorkInfos, nil
 }

internal/xml_loader/employee_wi_test.go

@@ -0,0 +1,22 @@
+package xml_loader_test
+
+import (
+	"testing"
+
+	"gitea.urkob.com/urko/ess-etl-go/config"
+	"gitea.urkob.com/urko/ess-etl-go/internal/request"
+	"gitea.urkob.com/urko/ess-etl-go/internal/xml_loader"
+	"github.com/stretchr/testify/require"
+)
+
+func TestEmployeeWILoader_LoadEmployee(t *testing.T) {
+	cfg := config.NewConfig(".env")
+
+	r := request.NewRequestService(cfg.AmsApi, cfg.AmsApiKey)
+
+	loader := xml_loader.NewEmployeeWILoader(r)
+	employeeID, from, to := cfg.EmployeeIdList[0], "2023-01-01", "2023-01-31"
+	got, err := loader.LoadEmployee(employeeID, from, to)
+	require.NoError(t, err)
+	require.Greater(t, len(got), 0)
+}