Performance issue when generating 10k elements in array #42
Description
Problem
Hello first of all thanks for the great library! it is really useful
Im using faker to seed my local database and i want to generate 10k elements the issue is if i use fakers default method faker.FakeData it hangs & takes up all of my memory
users := []domain.User{}
err = faker.FakeData(&users, options.WithRandomMapAndSliceMinSize(5000), options.WithRandomMapAndSliceMaxSize(10000))But if i do it manually like this it finishes in under 1 second
for i := 0; i < 10000; i++ {
user := domain.User{}
// err = faker.FakeData(&user)
user.Name = faker.Name()
user.Email = faker.Email()
user.PhoneNumber = faker.Phonenumber()
user.OrganizationID = org.ID
user.Locations = []domain.Location{}
user.Services = []domain.Service{}
_ = faker.FakeData(&user.Locations, options.WithRandomMapAndSliceMinSize(5), options.WithRandomMapAndSliceMaxSize(10))
_ = faker.FakeData(&user.Services, options.WithRandomMapAndSliceMinSize(5), options.WithRandomMapAndSliceMaxSize(10))
if err != nil {
panic(err)
}
users = append(users, user)
}here is the full code & the models
domain.go
type User struct {
ID uuid.UUID `json:"id" gorm:"primary_key; unique; type:uuid; column:id; default:uuid_generate_v4();"`
OrganizationID uuid.UUID `json:"organization_id" faker:"orgID"`
Organization *Organization `json:"organization"`
Name string `json:"name" faker:"name"`
Email string `json:"email" faker:"email"`
PhoneNumber string `json:"phone_number" faker:"phone_number"`
Photo string `json:"photo" faker:"-"`
AuditLogs []AuditLog `json:"audit_logs" gorm:"foreignKey:PerformerID;references:ID;" faker:"-"`
Locations []Location `json:"locations" gorm:"many2many:location_user;" faker:"locations"`
Services []Service `json:"services" gorm:"many2many:service_user;" faker:"services"`
FilterPresets datatypes.JSON `json:"filter_presets" gorm:"type:jsonb" faker:"-"`
Role userRole `json:"role" sql:"type:user_role" faker:"-"`
CreatedAt time.Time `json:"created_at" gorm:"autoCreateTime" faker:"-"`
}
type Location struct {
ID uuid.UUID `json:"id" gorm:"primary_key; unique; type:uuid; column:id; default:uuid_generate_v4();" faker:"-"`
OrganizationID uuid.UUID `json:"organization_id" gorm:"type:uuid;" faker:"orgID"`
Name string `json:"name" gorm:"type:varchar(200); not null;" faker:"cityName"`
Description string `json:"description" faker:"sentence"`
Addresses pq.StringArray `json:"addresses" gorm:"type:text[]" faker:"-"`
Logo string `json:"logo" faker:"-"`
SocialLinks pq.StringArray `json:"social_links" gorm:"type:text[]" faker:"-"`
WorkingHours datatypes.JSON `json:"working_hours" gorm:"type:jsonb; not null;" faker:"jsonb"`
ServiceSettings datatypes.JSON `json:"service_settings" gorm:"type:jsonb; not null;" faker:"jsonb"`
Kiosks datatypes.JSON `json:"kiosks" gorm:"type:jsonb; not null;" faker:"jsonb"`
TV datatypes.JSON `json:"tv" gorm:"type:jsonb; not null;" faker:"jsonb"`
Bookings []*Booking `json:"bookings" gorm:"foreignKey:LocationID;references:ID;" faker:"-"`
Services []*Service `json:"services" gorm:"many2many:location_service;" faker:"-"`
Users []User `json:"users" gorm:"many2many:location_user;" faker:"-"`
CreatedAt time.Time `json:"created_at" gorm:"autoCreateTime" faker:"-"`
}
type Service struct {
ID uuid.UUID `json:"id" gorm:"primary_key; unique; type:uuid; column:id; default:uuid_generate_v4();" faker:"orgID"`
Name string `json:"name" gorm:"type:varchar(200); not null;" faker:"name"`
Description string `json:"description" faker:"sentence"`
Bookings []*Booking `json:"bookings" gorm:"foreignKey:ServiceID;references:ID;" faker:"-"`
Locations []*Location `json:"locations" gorm:"many2many:location_service;" faker:"-"`
Users []User `json:"users" gorm:"many2many:service_user;" faker:"-"`
BookingForm datatypes.JSON `json:"booking_form" gorm:"type:jsonb" faker:"jsonb"`
WelcomePage datatypes.JSON `json:"welcome_page" gorm:"type:jsonb" faker:"jsonb"`
ConfirmationPage datatypes.JSON `json:"confirmation_page" gorm:"type:jsonb" faker:"jsonb"`
CreatedAt time.Time `json:"created_at" gorm:"autoCreateTime" faker:"-"`
}main.go
package main
import (
"log"
"os"
"reflect"
"time"
"github.com/IBM/fp-go/option"
"github.com/go-faker/faker/v4"
"github.com/go-faker/faker/v4/pkg/options"
"github.com/makerstudio-io/livqueue/libs/domain"
"github.com/makerstudio-io/livqueue/libs/utils"
"gorm.io/driver/postgres"
"gorm.io/gorm"
"gorm.io/gorm/logger"
)
var (
_ = faker.AddProvider("cityName", func(v reflect.Value) (interface{}, error) {
return faker.GetRealAddress().City, nil
})
_ = faker.AddProvider("locations", func(v reflect.Value) (interface{}, error) {
locations := []domain.Location{}
err := faker.FakeData(&locations, options.WithRandomMapAndSliceMinSize(5), options.WithRandomMapAndSliceMaxSize(10))
return locations, err
})
_ = faker.AddProvider("services", func(v reflect.Value) (interface{}, error) {
services := []domain.Service{}
err := faker.FakeData(&services, options.WithRandomMapAndSliceMinSize(5), options.WithRandomMapAndSliceMaxSize(10))
return services, err
})
_ = faker.AddProvider("jsonb", func(v reflect.Value) (interface{}, error) {
return []byte{'{', '}'}, nil
})
_ = faker.AddProvider("users", func(v reflect.Value) (interface{}, error) {
user := domain.User{}
err := faker.FakeData(&user)
return user, err
})
)
// _ = faker.AddProvider()
func main() {
conn := utils.ConstructDBConnString(utils.PostgresConfig{
SslMode: option.Some("disable"),
})
newLogger := logger.New(
log.New(os.Stdout, "\r\n", log.LstdFlags), // io writer
logger.Config{
SlowThreshold: time.Second, // Slow SQL threshold
LogLevel: logger.Silent, // Log level
IgnoreRecordNotFoundError: true, // Ignore ErrRecordNotFound error for logger
ParameterizedQueries: false, // Don't include params in the SQL log
Colorful: true, // Disable color
},
)
opts := gorm.Config{
Logger: newLogger,
}
db, err := gorm.Open(postgres.Open(conn), &opts)
if err != nil {
panic(err)
}
var org domain.Organization
res := db.Model(&domain.Organization{}).First(&org)
if res.Error != nil {
panic(res.Error)
}
_ = faker.AddProvider("orgID", func(v reflect.Value) (interface{}, error) {
return org.ID, nil
})
users := []domain.User{}
// err = faker.FakeData(&users, options.WithRandomMapAndSliceMinSize(5000), options.WithRandomMapAndSliceMaxSize(10000))
// for loop over 10000
for i := 0; i < 10000; i++ {
user := domain.User{}
// err = faker.FakeData(&user)
user.Name = faker.Name()
user.Email = faker.Email()
user.PhoneNumber = faker.Phonenumber()
user.OrganizationID = org.ID
user.Locations = []domain.Location{}
user.Services = []domain.Service{}
_ = faker.FakeData(&user.Locations, options.WithRandomMapAndSliceMinSize(5), options.WithRandomMapAndSliceMaxSize(10))
_ = faker.FakeData(&user.Services, options.WithRandomMapAndSliceMinSize(5), options.WithRandomMapAndSliceMaxSize(10))
if err != nil {
panic(err)
}
users = append(users, user)
}
if err != nil {
panic(err)
}
println(len(users), len(users[0].Locations), users[0].Locations[0].WorkingHours, len(users[0].Services))
req := db.CreateInBatches(&users, 500)
if req.Error != nil {
panic(req.Error)
}
}