From af2ed0a29e291c000590940994d706c6709777b4 Mon Sep 17 00:00:00 2001 From: Adrien Aury <44274230+adrienaury@users.noreply.github.com> Date: Wed, 13 Mar 2024 16:27:33 +0100 Subject: [PATCH 01/13] feat(pull): add maxLifetime maxOpen and maxIdle options parameters --- cmd/lino/dep_pull.go | 13 +++++++ cmd/lino/main.go | 2 +- internal/app/pull/cli.go | 37 +++++++++++++++++-- internal/app/pull/http.go | 2 +- internal/infra/pull/datasource_db2.go | 10 ++++-- internal/infra/pull/datasource_db2_dummy.go | 10 ++++-- internal/infra/pull/datasource_mariadb.go | 10 ++++-- internal/infra/pull/datasource_oracle.go | 10 ++++-- internal/infra/pull/datasource_postgres.go | 10 ++++-- internal/infra/pull/datasource_sqlserver.go | 10 ++++-- internal/infra/pull/datasource_ws.go | 2 +- internal/infra/pull/http_datasource.go | 2 +- internal/infra/pull/sql_datasource.go | 40 ++++++++++++++++++--- pkg/pull/driven.go | 4 ++- 14 files changed, 137 insertions(+), 25 deletions(-) diff --git a/cmd/lino/dep_pull.go b/cmd/lino/dep_pull.go index f982aa5a..311d07a1 100755 --- a/cmd/lino/dep_pull.go +++ b/cmd/lino/dep_pull.go @@ -20,6 +20,7 @@ package main import ( "io" "os" + "time" infra "github.com/cgi-fr/lino/internal/infra/pull" domain "github.com/cgi-fr/lino/pkg/pull" @@ -59,3 +60,15 @@ func pullKeyStoreFactory() func(file io.ReadCloser, keys []string) (domain.KeySt func traceListner(file *os.File) domain.TraceListener { return infra.NewJSONTraceListener(file) } + +func maxLifeTime(maxLifetimeInSeconds int64) domain.DataSourceOption { + return infra.WithMaxLifetime(time.Duration(maxLifetimeInSeconds) * time.Second) +} + +func maxOpenConns(maxOpenConns int) domain.DataSourceOption { + return infra.WithMaxOpenConns(maxOpenConns) +} + +func maxIdleConns(maxIdleConns int) domain.DataSourceOption { + return infra.WithMaxIdleConns(maxIdleConns) +} diff --git a/cmd/lino/main.go b/cmd/lino/main.go index 23783d0f..718a0f33 100755 --- a/cmd/lino/main.go +++ b/cmd/lino/main.go @@ -210,7 +210,7 @@ func initConfig() { table.Inject(dataconnectorStorage(), tableStorage(), tableExtractorFactory()) sequence.Inject(dataconnectorStorage(), tableStorage(), sequenceStorage(), sequenceUpdatorFactory()) id.Inject(idStorageFile, relationStorage(), idExporter(), idJSONStorage(*os.Stdout)) - pull.Inject(dataconnectorStorage(), relationStorage(), tableStorage(), idStorageFactory(), pullDataSourceFactory(), pullRowExporterFactory(), pullRowReaderFactory(), pullKeyStoreFactory(), traceListner(os.Stderr)) + pull.Inject(dataconnectorStorage(), relationStorage(), tableStorage(), idStorageFactory(), pullDataSourceFactory(), pullRowExporterFactory(), pullRowReaderFactory(), pullKeyStoreFactory(), traceListner(os.Stderr), maxLifeTime, maxOpenConns, maxIdleConns) push.Inject(dataconnectorStorage(), relationStorage(), tableStorage(), idStorageFactory(), pushDataDestinationFactory(), pushRowIteratorFactory(), pushRowExporterFactory(), pushTranslator()) } diff --git a/internal/app/pull/cli.go b/internal/app/pull/cli.go index dec11256..6e43dcce 100755 --- a/internal/app/pull/cli.go +++ b/internal/app/pull/cli.go @@ -44,6 +44,9 @@ var ( pullExporterFactory func(io.Writer) pull.RowExporter rowReaderFactory func(io.ReadCloser) pull.RowReader keyStoreFactory func(io.ReadCloser, []string) (pull.KeyStore, error) + maxLifeTimeOption func(int64) pull.DataSourceOption + maxOpenConnsOption func(int) pull.DataSourceOption + maxIdleConnsOption func(int) pull.DataSourceOption ) var traceListener pull.TraceListener @@ -59,6 +62,9 @@ func Inject( rrf func(io.ReadCloser) pull.RowReader, ksf func(io.ReadCloser, []string) (pull.KeyStore, error), tl pull.TraceListener, + mltOpt func(int64) pull.DataSourceOption, + mocOpt func(int) pull.DataSourceOption, + micOpt func(int) pull.DataSourceOption, ) { dataconnectorStorage = dbas relStorage = rs @@ -69,6 +75,9 @@ func Inject( rowReaderFactory = rrf keyStoreFactory = ksf traceListener = tl + maxLifeTimeOption = mltOpt + maxOpenConnsOption = mocOpt + maxIdleConnsOption = micOpt } // NewCommand implements the cli pull command @@ -85,6 +94,8 @@ func NewCommand(fullName string, err *os.File, out *os.File, in *os.File) *cobra var diagnostic bool var filters pull.RowReader var parallel uint + var maxLifeTimeInSeconds int64 + var maxOpenConns, maxIdleConns int cmd := &cobra.Command{ Use: "pull [DB Alias Name]", @@ -103,6 +114,9 @@ func NewCommand(fullName string, err *os.File, out *os.File, in *os.File) *cobra Str("table", table). Str("where", where). Uint("parallel", parallel). + Int64("maxLifeTimeInSeconds", maxLifeTimeInSeconds). + Int("maxOpenConns", maxOpenConns). + Int("maxIdleConns", maxIdleConns). Msg("Pull mode") }, Run: func(cmd *cobra.Command, args []string) { @@ -111,7 +125,7 @@ func NewCommand(fullName string, err *os.File, out *os.File, in *os.File) *cobra startTime := time.Now() - datasource, e1 := getDataSource(args[0], out) + datasource, e1 := getDataSource(args[0], out, maxLifeTimeInSeconds, maxOpenConns, maxIdleConns) if e1 != nil { fmt.Fprintln(err, e1.Error()) os.Exit(1) @@ -197,13 +211,16 @@ func NewCommand(fullName string, err *os.File, out *os.File, in *os.File) *cobra cmd.Flags().StringVarP(&where, "where", "w", "", "Advanced SQL where clause to filter") cmd.Flags().StringVarP(&ingressDescriptor, "ingress-descriptor", "i", "ingress-descriptor.yaml", "pull content using ingress descriptor definition") cmd.Flags().UintVarP(¶llel, "parallel", "p", 1, "number of parallel workers") + cmd.Flags().Int64Var(&maxLifeTimeInSeconds, "conn-max-lifetime", -1, "sets the maximum amount of time (in seconds) a connection may be reused") + cmd.Flags().IntVar(&maxOpenConns, "conn-max-open", -1, "sets the maximum number of open connections to the database") + cmd.Flags().IntVar(&maxIdleConns, "conn-max-idle", -1, "sets the maximum number of connections in the idle connection pool") cmd.SetOut(out) cmd.SetErr(err) cmd.SetIn(in) return cmd } -func getDataSource(dataconnectorName string, out io.Writer) (pull.DataSource, error) { +func getDataSource(dataconnectorName string, out io.Writer, maxLifeTimeInSeconds int64, maxOpenConns, maxIdleConns int) (pull.DataSource, error) { alias, e1 := dataconnector.Get(dataconnectorStorage, dataconnectorName) if e1 != nil { return nil, e1 @@ -219,7 +236,21 @@ func getDataSource(dataconnectorName string, out io.Writer) (pull.DataSource, er return nil, fmt.Errorf("no datasource found for database type") } - return datasourceFactory.New(u.URL.String(), alias.Schema), nil + options := []pull.DataSourceOption{} + + if maxLifeTimeInSeconds >= 0 { + options = append(options, maxLifeTimeOption(maxLifeTimeInSeconds)) + } + + if maxOpenConns >= 0 { + options = append(options, maxOpenConnsOption(maxOpenConns)) + } + + if maxIdleConns >= 0 { + options = append(options, maxIdleConnsOption(maxIdleConns)) + } + + return datasourceFactory.New(u.URL.String(), alias.Schema, options...), nil } func getPullerPlan(idStorage id.Storage) (pull.Plan, pull.Table, error) { diff --git a/internal/app/pull/http.go b/internal/app/pull/http.go index 165033db..3fb9d379 100644 --- a/internal/app/pull/http.go +++ b/internal/app/pull/http.go @@ -114,7 +114,7 @@ func HandlerFactory(ingressDescriptor string) func(w http.ResponseWriter, r *htt return } - datasource, err = getDataSource(datasourceName, w) + datasource, err = getDataSource(datasourceName, w, -1, -1, -1) if err != nil { log.Error().Err(err).Msg("") w.WriteHeader(http.StatusNotFound) diff --git a/internal/infra/pull/datasource_db2.go b/internal/infra/pull/datasource_db2.go index 42368513..bc6f5445 100644 --- a/internal/infra/pull/datasource_db2.go +++ b/internal/infra/pull/datasource_db2.go @@ -36,10 +36,16 @@ func NewDb2DataSourceFactory() *Db2DataSourceFactory { } // New return a Db2 puller -func (e *Db2DataSourceFactory) New(url string, schema string) pull.DataSource { - return &SQLDataSource{ +func (e *Db2DataSourceFactory) New(url string, schema string, options ...pull.DataSourceOption) pull.DataSource { + ds := &SQLDataSource{ url: url, schema: schema, dialect: commonsql.Db2Dialect{}, } + + for _, option := range options { + option(ds) + } + + return ds } diff --git a/internal/infra/pull/datasource_db2_dummy.go b/internal/infra/pull/datasource_db2_dummy.go index 1d4db4bf..f5d8c9b1 100644 --- a/internal/infra/pull/datasource_db2_dummy.go +++ b/internal/infra/pull/datasource_db2_dummy.go @@ -34,10 +34,16 @@ func NewDb2DataSourceFactory() *Db2DataSourceFactory { } // New return a Db2 puller -func (e *Db2DataSourceFactory) New(url string, schema string) pull.DataSource { - return &SQLDataSource{ +func (e *Db2DataSourceFactory) New(url string, schema string, options ...pull.DataSourceOption) pull.DataSource { + ds := &SQLDataSource{ url: url, schema: schema, dialect: commonsql.Db2Dialect{}, } + + for _, option := range options { + option(ds) + } + + return ds } diff --git a/internal/infra/pull/datasource_mariadb.go b/internal/infra/pull/datasource_mariadb.go index fffd59c8..4fb4828e 100644 --- a/internal/infra/pull/datasource_mariadb.go +++ b/internal/infra/pull/datasource_mariadb.go @@ -33,10 +33,16 @@ func NewMariadbDataSourceFactory() *MariadbDataSourceFactory { } // New return a Mariadb puller -func (e *MariadbDataSourceFactory) New(url string, schema string) pull.DataSource { - return &SQLDataSource{ +func (e *MariadbDataSourceFactory) New(url string, schema string, options ...pull.DataSourceOption) pull.DataSource { + ds := &SQLDataSource{ url: url, schema: schema, dialect: commonsql.MariadbDialect{}, } + + for _, option := range options { + option(ds) + } + + return ds } diff --git a/internal/infra/pull/datasource_oracle.go b/internal/infra/pull/datasource_oracle.go index 6804282c..62b6a377 100755 --- a/internal/infra/pull/datasource_oracle.go +++ b/internal/infra/pull/datasource_oracle.go @@ -14,10 +14,16 @@ func NewOracleDataSourceFactory() *OracleDataSourceFactory { } // New return a Oracle puller -func (e *OracleDataSourceFactory) New(url string, schema string) pull.DataSource { - return &SQLDataSource{ +func (e *OracleDataSourceFactory) New(url string, schema string, options ...pull.DataSourceOption) pull.DataSource { + ds := &SQLDataSource{ url: url, schema: schema, dialect: commonsql.OracleDialect{}, } + + for _, option := range options { + option(ds) + } + + return ds } diff --git a/internal/infra/pull/datasource_postgres.go b/internal/infra/pull/datasource_postgres.go index 74037f07..e965e1cd 100755 --- a/internal/infra/pull/datasource_postgres.go +++ b/internal/infra/pull/datasource_postgres.go @@ -33,10 +33,16 @@ func NewPostgresDataSourceFactory() *PostgresDataSourceFactory { } // New return a Postgres puller -func (e *PostgresDataSourceFactory) New(url string, schema string) pull.DataSource { - return &SQLDataSource{ +func (e *PostgresDataSourceFactory) New(url string, schema string, options ...pull.DataSourceOption) pull.DataSource { + ds := &SQLDataSource{ url: url, schema: schema, dialect: commonsql.PostgresDialect{}, } + + for _, option := range options { + option(ds) + } + + return ds } diff --git a/internal/infra/pull/datasource_sqlserver.go b/internal/infra/pull/datasource_sqlserver.go index e4a5053a..df5abb6f 100644 --- a/internal/infra/pull/datasource_sqlserver.go +++ b/internal/infra/pull/datasource_sqlserver.go @@ -33,10 +33,16 @@ func NewSQLServerDataSourceFactory() *SQLServerDataSourceFactory { } // New return a SQLServer puller -func (e *SQLServerDataSourceFactory) New(url string, schema string) pull.DataSource { - return &SQLDataSource{ +func (e *SQLServerDataSourceFactory) New(url string, schema string, options ...pull.DataSourceOption) pull.DataSource { + ds := &SQLDataSource{ url: url, schema: schema, dialect: commonsql.SQLServerDialect{}, } + + for _, option := range options { + option(ds) + } + + return ds } diff --git a/internal/infra/pull/datasource_ws.go b/internal/infra/pull/datasource_ws.go index f81d9ba0..516fbe9e 100644 --- a/internal/infra/pull/datasource_ws.go +++ b/internal/infra/pull/datasource_ws.go @@ -69,7 +69,7 @@ func NewWSDataSourceFactory() *WSDataSourceFactory { } // New return a WS puller -func (e *WSDataSourceFactory) New(url string, schema string) pull.DataSource { +func (e *WSDataSourceFactory) New(url string, schema string, options ...pull.DataSourceOption) pull.DataSource { return &WSDataSource{ url: url, schema: schema, diff --git a/internal/infra/pull/http_datasource.go b/internal/infra/pull/http_datasource.go index 06cb20ac..4dcb8fe1 100644 --- a/internal/infra/pull/http_datasource.go +++ b/internal/infra/pull/http_datasource.go @@ -38,7 +38,7 @@ func NewHTTPDataSourceFactory() *HTTPDataSourceFactory { } // New return a HTTP puller -func (e *HTTPDataSourceFactory) New(url string, schema string) pull.DataSource { +func (e *HTTPDataSourceFactory) New(url string, schema string, options ...pull.DataSourceOption) pull.DataSource { return &HTTPDataSource{ url: url, schema: schema, diff --git a/internal/infra/pull/sql_datasource.go b/internal/infra/pull/sql_datasource.go index 0f94c6ba..7ba78ce5 100644 --- a/internal/infra/pull/sql_datasource.go +++ b/internal/infra/pull/sql_datasource.go @@ -21,6 +21,7 @@ import ( "database/sql" "fmt" "strings" + "time" "github.com/cgi-fr/lino/internal/infra/commonsql" "github.com/cgi-fr/lino/pkg/pull" @@ -30,13 +31,37 @@ import ( "github.com/xo/dburl" ) +func WithMaxLifetime(maxLifeTime time.Duration) pull.DataSourceOption { + return func(ds pull.DataSource) { + log.Info().Int64("maxLifetime", int64(maxLifeTime.Seconds())).Msg("setting database connection parameter") + ds.(*SQLDataSource).maxLifetime = maxLifeTime + } +} + +func WithMaxOpenConns(maxOpenConns int) pull.DataSourceOption { + return func(ds pull.DataSource) { + log.Info().Int("maxOpenConns", maxOpenConns).Msg("setting database connection parameter") + ds.(*SQLDataSource).maxOpenConns = maxOpenConns + } +} + +func WithMaxIdleConns(maxIdleConns int) pull.DataSourceOption { + return func(ds pull.DataSource) { + log.Info().Int("maxIdleConns", maxIdleConns).Msg("setting database connection parameter") + ds.(*SQLDataSource).maxIdleConns = maxIdleConns + } +} + // SQLDataSource to read in the pull process. type SQLDataSource struct { - url string - schema string - dbx *sqlx.DB - db *sql.DB - dialect commonsql.Dialect + url string + schema string + dbx *sqlx.DB + db *sql.DB + dialect commonsql.Dialect + maxLifetime time.Duration + maxOpenConns int + maxIdleConns int } // Open a connection to the SQL DB @@ -46,6 +71,11 @@ func (ds *SQLDataSource) Open() error { return err } + // database handle settings + db.SetConnMaxLifetime(ds.maxLifetime) + db.SetMaxOpenConns(ds.maxOpenConns) + db.SetMaxIdleConns(ds.maxIdleConns) + ds.db = db u, err := dburl.Parse(ds.url) diff --git a/pkg/pull/driven.go b/pkg/pull/driven.go index 2a34757f..ab9dc8db 100755 --- a/pkg/pull/driven.go +++ b/pkg/pull/driven.go @@ -22,9 +22,11 @@ type RowExporter interface { Export(ExportedRow) error } +type DataSourceOption func(DataSource) + // DataSourceFactory exposes methods to create new datasources. type DataSourceFactory interface { - New(url string, schema string) DataSource + New(url string, schema string, options ...DataSourceOption) DataSource } // DataSource to read in the pull process. From 22d7c6f56dcb29f7b82004982d92a33d4da04e98 Mon Sep 17 00:00:00 2001 From: Adrien Aury <44274230+adrienaury@users.noreply.github.com> Date: Wed, 13 Mar 2024 16:43:32 +0100 Subject: [PATCH 02/13] feat(push): add maxLifetime maxOpen and maxIdle options parameters --- cmd/lino/dep_pull.go | 6 +- cmd/lino/dep_push.go | 13 +++++ cmd/lino/main.go | 4 +- internal/app/push/cli.go | 57 ++++++++++++++----- internal/app/push/cli_test.go | 5 +- internal/app/push/http.go | 2 +- internal/infra/push/datadestination_db2.go | 4 +- .../infra/push/datadestination_db2_dummy.go | 4 +- internal/infra/push/datadestination_http.go | 2 +- .../infra/push/datadestination_mariadb.go | 4 +- internal/infra/push/datadestination_oracle.go | 4 +- .../infra/push/datadestination_postgres.go | 4 +- internal/infra/push/datadestination_sql.go | 40 ++++++++++++- .../infra/push/datadestination_sqlserver.go | 4 +- internal/infra/push/datadestination_ws.go | 4 +- pkg/push/driven.go | 4 +- 16 files changed, 123 insertions(+), 38 deletions(-) diff --git a/cmd/lino/dep_pull.go b/cmd/lino/dep_pull.go index 311d07a1..3eeb4428 100755 --- a/cmd/lino/dep_pull.go +++ b/cmd/lino/dep_pull.go @@ -61,14 +61,14 @@ func traceListner(file *os.File) domain.TraceListener { return infra.NewJSONTraceListener(file) } -func maxLifeTime(maxLifetimeInSeconds int64) domain.DataSourceOption { +func pullMaxLifeTime(maxLifetimeInSeconds int64) domain.DataSourceOption { return infra.WithMaxLifetime(time.Duration(maxLifetimeInSeconds) * time.Second) } -func maxOpenConns(maxOpenConns int) domain.DataSourceOption { +func pullMaxOpenConns(maxOpenConns int) domain.DataSourceOption { return infra.WithMaxOpenConns(maxOpenConns) } -func maxIdleConns(maxIdleConns int) domain.DataSourceOption { +func pullMaxIdleConns(maxIdleConns int) domain.DataSourceOption { return infra.WithMaxIdleConns(maxIdleConns) } diff --git a/cmd/lino/dep_push.go b/cmd/lino/dep_push.go index b2b03b28..86f45dd0 100755 --- a/cmd/lino/dep_push.go +++ b/cmd/lino/dep_push.go @@ -19,6 +19,7 @@ package main import ( "io" + "time" infra "github.com/cgi-fr/lino/internal/infra/push" domain "github.com/cgi-fr/lino/pkg/push" @@ -48,3 +49,15 @@ func pushRowExporterFactory() func(io.Writer) domain.RowWriter { func pushTranslator() domain.Translator { return infra.NewFileTranslator() } + +func pushMaxLifeTime(maxLifetimeInSeconds int64) domain.DataDestinationOption { + return infra.WithMaxLifetime(time.Duration(maxLifetimeInSeconds) * time.Second) +} + +func pushMaxOpenConns(maxOpenConns int) domain.DataDestinationOption { + return infra.WithMaxOpenConns(maxOpenConns) +} + +func pushMaxIdleConns(maxIdleConns int) domain.DataDestinationOption { + return infra.WithMaxIdleConns(maxIdleConns) +} diff --git a/cmd/lino/main.go b/cmd/lino/main.go index 718a0f33..8ee689b9 100755 --- a/cmd/lino/main.go +++ b/cmd/lino/main.go @@ -210,8 +210,8 @@ func initConfig() { table.Inject(dataconnectorStorage(), tableStorage(), tableExtractorFactory()) sequence.Inject(dataconnectorStorage(), tableStorage(), sequenceStorage(), sequenceUpdatorFactory()) id.Inject(idStorageFile, relationStorage(), idExporter(), idJSONStorage(*os.Stdout)) - pull.Inject(dataconnectorStorage(), relationStorage(), tableStorage(), idStorageFactory(), pullDataSourceFactory(), pullRowExporterFactory(), pullRowReaderFactory(), pullKeyStoreFactory(), traceListner(os.Stderr), maxLifeTime, maxOpenConns, maxIdleConns) - push.Inject(dataconnectorStorage(), relationStorage(), tableStorage(), idStorageFactory(), pushDataDestinationFactory(), pushRowIteratorFactory(), pushRowExporterFactory(), pushTranslator()) + pull.Inject(dataconnectorStorage(), relationStorage(), tableStorage(), idStorageFactory(), pullDataSourceFactory(), pullRowExporterFactory(), pullRowReaderFactory(), pullKeyStoreFactory(), traceListner(os.Stderr), pullMaxLifeTime, pullMaxOpenConns, pullMaxIdleConns) + push.Inject(dataconnectorStorage(), relationStorage(), tableStorage(), idStorageFactory(), pushDataDestinationFactory(), pushRowIteratorFactory(), pushRowExporterFactory(), pushTranslator(), pushMaxLifeTime, pushMaxOpenConns, pushMaxIdleConns) } func writeMetricsToFile(statsFile string, statsByte []byte) { diff --git a/internal/app/push/cli.go b/internal/app/push/cli.go index 552947b1..4bc4a66f 100755 --- a/internal/app/push/cli.go +++ b/internal/app/push/cli.go @@ -45,6 +45,9 @@ var ( rowIteratorFactory func(io.ReadCloser) push.RowIterator rowExporterFactory func(io.Writer) push.RowWriter translator push.Translator + maxLifeTimeOption func(int64) push.DataDestinationOption + maxOpenConnsOption func(int) push.DataDestinationOption + maxIdleConnsOption func(int) push.DataDestinationOption ) // Inject dependencies @@ -57,6 +60,9 @@ func Inject( rif func(io.ReadCloser) push.RowIterator, ref func(io.Writer) push.RowWriter, trnsltor push.Translator, + mltOpt func(int64) push.DataDestinationOption, + mocOpt func(int) push.DataDestinationOption, + micOpt func(int) push.DataDestinationOption, ) { dataconnectorStorage = dbas relStorage = rs @@ -66,21 +72,26 @@ func Inject( rowIteratorFactory = rif rowExporterFactory = ref translator = trnsltor + maxLifeTimeOption = mltOpt + maxOpenConnsOption = mocOpt + maxIdleConnsOption = micOpt } // NewCommand implements the cli pull command func NewCommand(fullName string, err *os.File, out *os.File, in *os.File) *cobra.Command { var ( - commitSize uint - disableConstraints bool - catchErrors string - table string - ingressDescriptor string - rowExporter push.RowWriter - pkTranslations map[string]string - whereField string - savepoint string - autoTruncate bool + commitSize uint + disableConstraints bool + catchErrors string + table string + ingressDescriptor string + rowExporter push.RowWriter + pkTranslations map[string]string + whereField string + savepoint string + autoTruncate bool + maxLifeTimeInSeconds int64 + maxOpenConns, maxIdleConns int ) cmd := &cobra.Command{ @@ -106,6 +117,9 @@ func NewCommand(fullName string, err *os.File, out *os.File, in *os.File) *cobra Bool("disable-constraints", disableConstraints). Str("catch-errors", catchErrors). Str("table", table). + Int64("maxLifeTimeInSeconds", maxLifeTimeInSeconds). + Int("maxOpenConns", maxOpenConns). + Int("maxIdleConns", maxIdleConns). Msg("Push mode") }, Run: func(cmd *cobra.Command, args []string) { @@ -122,7 +136,7 @@ func NewCommand(fullName string, err *os.File, out *os.File, in *os.File) *cobra mode, _ = push.ParseMode(args[0]) } - datadestination, e1 := getDataDestination(dcDestination) + datadestination, e1 := getDataDestination(dcDestination, maxLifeTimeInSeconds, maxOpenConns, maxIdleConns) if e1 != nil { fmt.Fprintln(err, e1.Error()) os.Exit(1) @@ -174,6 +188,9 @@ func NewCommand(fullName string, err *os.File, out *os.File, in *os.File) *cobra cmd.Flags().StringVar(&whereField, "using-pk-field", "__usingpk__", "Name of the data field that can be used as pk for update queries") cmd.Flags().StringVar(&savepoint, "savepoint", "", "Name of a file to write primary keys of effectively processed lines (commit to database)") cmd.Flags().BoolVarP(&autoTruncate, "autotruncate", "a", false, "Automatically truncate values to the maximum length defined in table.yaml") + cmd.Flags().Int64Var(&maxLifeTimeInSeconds, "conn-max-lifetime", -1, "sets the maximum amount of time (in seconds) a connection may be reused") + cmd.Flags().IntVar(&maxOpenConns, "conn-max-open", -1, "sets the maximum number of open connections to the database") + cmd.Flags().IntVar(&maxIdleConns, "conn-max-idle", -1, "sets the maximum number of connections in the idle connection pool") cmd.SetOut(out) cmd.SetErr(err) cmd.SetIn(in) @@ -221,7 +238,7 @@ func loadTranslator(pkTranslations map[string]string) error { return nil } -func getDataDestination(dataconnectorName string) (push.DataDestination, *push.Error) { +func getDataDestination(dataconnectorName string, maxLifeTimeInSeconds int64, maxOpenConns, maxIdleConns int) (push.DataDestination, *push.Error) { alias, e1 := dataconnector.Get(dataconnectorStorage, dataconnectorName) if e1 != nil { return nil, &push.Error{Description: e1.Error()} @@ -240,7 +257,21 @@ func getDataDestination(dataconnectorName string) (push.DataDestination, *push.E return nil, &push.Error{Description: "no datadestination found for database type " + u.UnaliasedDriver} } - return datadestinationFactory.New(u.URL.String(), alias.Schema), nil + options := []push.DataDestinationOption{} + + if maxLifeTimeInSeconds >= 0 { + options = append(options, maxLifeTimeOption(maxLifeTimeInSeconds)) + } + + if maxOpenConns >= 0 { + options = append(options, maxOpenConnsOption(maxOpenConns)) + } + + if maxIdleConns >= 0 { + options = append(options, maxIdleConnsOption(maxIdleConns)) + } + + return datadestinationFactory.New(u.URL.String(), alias.Schema, options...), nil } func getPlan(idStorage id.Storage, autoTruncate bool) (push.Plan, *push.Error) { diff --git a/internal/app/push/cli_test.go b/internal/app/push/cli_test.go index a7cdf07b..7b5cb700 100755 --- a/internal/app/push/cli_test.go +++ b/internal/app/push/cli_test.go @@ -43,6 +43,9 @@ func Test_getDataDestination(t *testing.T) { func(io.ReadCloser) push.RowIterator { return &push.MockRowIterator{} }, func(io.Writer) push.RowWriter { return &push.MockRowWriter{} }, push.NewMockTranslator(), + maxLifeTimeOption, + maxOpenConnsOption, + maxIdleConnsOption, ) type args struct { @@ -63,7 +66,7 @@ func Test_getDataDestination(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, got1 := getDataDestination(tt.args.dataconnectorName) + got, got1 := getDataDestination(tt.args.dataconnectorName, -1, -1, -1) if !reflect.DeepEqual(got, tt.want) { t.Errorf("getDataDestination() got = %v, want %v", got, tt.want) } diff --git a/internal/app/push/http.go b/internal/app/push/http.go index 92b732f3..2087af5c 100644 --- a/internal/app/push/http.go +++ b/internal/app/push/http.go @@ -69,7 +69,7 @@ func Handler(w http.ResponseWriter, r *http.Request, mode push.Mode, ingressDesc return } - datadestination, err := getDataDestination(dcDestination) + datadestination, err := getDataDestination(dcDestination, -1, -1, -1) if err != nil { log.Error().Err(err).Msg("") w.WriteHeader(http.StatusNotFound) diff --git a/internal/infra/push/datadestination_db2.go b/internal/infra/push/datadestination_db2.go index 9b94fe38..c396e069 100644 --- a/internal/infra/push/datadestination_db2.go +++ b/internal/infra/push/datadestination_db2.go @@ -40,8 +40,8 @@ func NewDb2DataDestinationFactory() *Db2DataDestinationFactory { } // New return a Db2 pusher -func (e *Db2DataDestinationFactory) New(url string, schema string) push.DataDestination { - return NewSQLDataDestination(url, schema, Db2Dialect{}) +func (e *Db2DataDestinationFactory) New(url string, schema string, options ...push.DataDestinationOption) push.DataDestination { + return NewSQLDataDestination(url, schema, Db2Dialect{}, options...) } // Db2Dialect inject oracle variations diff --git a/internal/infra/push/datadestination_db2_dummy.go b/internal/infra/push/datadestination_db2_dummy.go index 73e4e0f3..0ea73054 100644 --- a/internal/infra/push/datadestination_db2_dummy.go +++ b/internal/infra/push/datadestination_db2_dummy.go @@ -35,8 +35,8 @@ func NewDb2DataDestinationFactory() *Db2DataDestinationFactory { } // New return a Db2 pusher -func (e *Db2DataDestinationFactory) New(url string, schema string) push.DataDestination { - return NewSQLDataDestination(url, schema, Db2Dialect{}) +func (e *Db2DataDestinationFactory) New(url string, schema string, options ...push.DataDestinationOption) push.DataDestination { + return NewSQLDataDestination(url, schema, Db2Dialect{}, options...) } // Db2Dialect inject oracle variations diff --git a/internal/infra/push/datadestination_http.go b/internal/infra/push/datadestination_http.go index dba1991c..5755c552 100644 --- a/internal/infra/push/datadestination_http.go +++ b/internal/infra/push/datadestination_http.go @@ -38,7 +38,7 @@ func NewHTTPDataDestinationFactory() *HTTPDataDestinationFactory { } // New return a HTTP pusher -func (e *HTTPDataDestinationFactory) New(url string, schema string) push.DataDestination { +func (e *HTTPDataDestinationFactory) New(url string, schema string, options ...push.DataDestinationOption) push.DataDestination { return NewHTTPDataDestination(url, schema) } diff --git a/internal/infra/push/datadestination_mariadb.go b/internal/infra/push/datadestination_mariadb.go index d088c425..330230f6 100644 --- a/internal/infra/push/datadestination_mariadb.go +++ b/internal/infra/push/datadestination_mariadb.go @@ -34,8 +34,8 @@ func NewMariadbDataDestinationFactory() *MariadbDataDestinationFactory { } // New return a Mariadb pusher -func (e *MariadbDataDestinationFactory) New(url string, schema string) push.DataDestination { - return NewSQLDataDestination(url, schema, MariadbDialect{}) +func (e *MariadbDataDestinationFactory) New(url string, schema string, options ...push.DataDestinationOption) push.DataDestination { + return NewSQLDataDestination(url, schema, MariadbDialect{}, options...) } // MariadbDialect inject mariadb variations diff --git a/internal/infra/push/datadestination_oracle.go b/internal/infra/push/datadestination_oracle.go index 59815727..f84c1328 100644 --- a/internal/infra/push/datadestination_oracle.go +++ b/internal/infra/push/datadestination_oracle.go @@ -21,8 +21,8 @@ func NewOracleDataDestinationFactory() *OracleDataDestinationFactory { } // New return a Oracle pusher -func (e *OracleDataDestinationFactory) New(url string, schema string) push.DataDestination { - return NewSQLDataDestination(url, schema, OracleDialect{}) +func (e *OracleDataDestinationFactory) New(url string, schema string, options ...push.DataDestinationOption) push.DataDestination { + return NewSQLDataDestination(url, schema, OracleDialect{}, options...) } // OracleDialect inject oracle variations diff --git a/internal/infra/push/datadestination_postgres.go b/internal/infra/push/datadestination_postgres.go index c8f2abb7..01dcf94f 100755 --- a/internal/infra/push/datadestination_postgres.go +++ b/internal/infra/push/datadestination_postgres.go @@ -34,8 +34,8 @@ func NewPostgresDataDestinationFactory() *PostgresDataDestinationFactory { } // New return a Postgres pusher -func (e *PostgresDataDestinationFactory) New(url string, schema string) push.DataDestination { - return NewSQLDataDestination(url, schema, PostgresDialect{}) +func (e *PostgresDataDestinationFactory) New(url string, schema string, options ...push.DataDestinationOption) push.DataDestination { + return NewSQLDataDestination(url, schema, PostgresDialect{}, options...) } // PostgresDialect inject postgres variations diff --git a/internal/infra/push/datadestination_sql.go b/internal/infra/push/datadestination_sql.go index 24234bf5..b347b323 100644 --- a/internal/infra/push/datadestination_sql.go +++ b/internal/infra/push/datadestination_sql.go @@ -21,6 +21,7 @@ import ( "database/sql" "fmt" "strings" + "time" "github.com/cgi-fr/lino/pkg/push" "github.com/jmoiron/sqlx" @@ -28,6 +29,27 @@ import ( "github.com/xo/dburl" ) +func WithMaxLifetime(maxLifeTime time.Duration) push.DataDestinationOption { + return func(ds push.DataDestination) { + log.Info().Int64("maxLifetime", int64(maxLifeTime.Seconds())).Msg("setting database connection parameter") + ds.(*SQLDataDestination).maxLifetime = maxLifeTime + } +} + +func WithMaxOpenConns(maxOpenConns int) push.DataDestinationOption { + return func(ds push.DataDestination) { + log.Info().Int("maxOpenConns", maxOpenConns).Msg("setting database connection parameter") + ds.(*SQLDataDestination).maxOpenConns = maxOpenConns + } +} + +func WithMaxIdleConns(maxIdleConns int) push.DataDestinationOption { + return func(ds push.DataDestination) { + log.Info().Int("maxIdleConns", maxIdleConns).Msg("setting database connection parameter") + ds.(*SQLDataDestination).maxIdleConns = maxIdleConns + } +} + // SQLDataDestination read data from a SQL database. type SQLDataDestination struct { url string @@ -38,16 +60,25 @@ type SQLDataDestination struct { mode push.Mode disableConstraints bool dialect SQLDialect + maxLifetime time.Duration + maxOpenConns int + maxIdleConns int } // NewSQLDataDestination creates a new SQL datadestination. -func NewSQLDataDestination(url string, schema string, dialect SQLDialect) *SQLDataDestination { - return &SQLDataDestination{ +func NewSQLDataDestination(url string, schema string, dialect SQLDialect, options ...push.DataDestinationOption) *SQLDataDestination { + dd := &SQLDataDestination{ url: url, schema: schema, rowWriter: map[string]*SQLRowWriter{}, dialect: dialect, } + + for _, option := range options { + option(dd) + } + + return dd } // Close SQL connections @@ -130,6 +161,11 @@ func (dd *SQLDataDestination) Open(plan push.Plan, mode push.Mode, disableConstr return &push.Error{Description: err.Error()} } + // database handle settings + db.SetConnMaxLifetime(dd.maxLifetime) + db.SetMaxOpenConns(dd.maxOpenConns) + db.SetMaxIdleConns(dd.maxIdleConns) + u, err := dburl.Parse(dd.url) if err != nil { return &push.Error{Description: err.Error()} diff --git a/internal/infra/push/datadestination_sqlserver.go b/internal/infra/push/datadestination_sqlserver.go index cb3a28e9..a9153002 100644 --- a/internal/infra/push/datadestination_sqlserver.go +++ b/internal/infra/push/datadestination_sqlserver.go @@ -36,8 +36,8 @@ func NewSQLServerDataDestinationFactory() *SQLServerDataDestinationFactory { } // New return a SQLServer pusher -func (e *SQLServerDataDestinationFactory) New(url string, schema string) push.DataDestination { - return NewSQLDataDestination(url, schema, SQLServerDialect{}) +func (e *SQLServerDataDestinationFactory) New(url string, schema string, options ...push.DataDestinationOption) push.DataDestination { + return NewSQLDataDestination(url, schema, SQLServerDialect{}, options...) } // SQLServerDialect inject SQLServer variations diff --git a/internal/infra/push/datadestination_ws.go b/internal/infra/push/datadestination_ws.go index 21e065f6..4d954d21 100644 --- a/internal/infra/push/datadestination_ws.go +++ b/internal/infra/push/datadestination_ws.go @@ -34,7 +34,7 @@ import ( type Action string -// "pull_open", "push_open", "push_data", "push_commit", "push_close" +// "pull_open", "push_open", "push_data", "push_commit", "push_close" const ( PushOpen Action = "push_open" PushData Action = "push_data" @@ -77,7 +77,7 @@ func NewWebSocketDataDestinationFactory() *WebSocketDataDestinationFactory { } // New return a web socket pusher -func (e *WebSocketDataDestinationFactory) New(url string, schema string) push.DataDestination { +func (e *WebSocketDataDestinationFactory) New(url string, schema string, options ...push.DataDestinationOption) push.DataDestination { return NewWebSocketDataDestination(url, schema) } diff --git a/pkg/push/driven.go b/pkg/push/driven.go index 57e5f7ef..dbd50757 100755 --- a/pkg/push/driven.go +++ b/pkg/push/driven.go @@ -17,9 +17,11 @@ package push +type DataDestinationOption func(DataDestination) + // DataDestinationFactory exposes methods to create new datadestinations. type DataDestinationFactory interface { - New(url string, schema string) DataDestination + New(url string, schema string, options ...DataDestinationOption) DataDestination } // DataDestination to write in the push process. From 69cc706f50f892dac8ea279060d980e4c8ad027d Mon Sep 17 00:00:00 2001 From: Adrien Aury <44274230+adrienaury@users.noreply.github.com> Date: Wed, 13 Mar 2024 16:54:20 +0100 Subject: [PATCH 03/13] fix: regression in venom test --- tests/suites/pull/stats.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/suites/pull/stats.yml b/tests/suites/pull/stats.yml index 75d54356..28d17383 100644 --- a/tests/suites/pull/stats.yml +++ b/tests/suites/pull/stats.yml @@ -38,7 +38,7 @@ testcases: - script: lino pull -v 5 source --where "address_id > 13" --log-json assertions: - result.code ShouldEqual 0 - - result.systemerr ShouldContainSubstring {"level":"info","limit":1,"filter":{},"diagnostic":false,"distinct":false,"filter-from-file":"","exclude-from-file":"","table":"","where":"address_id > 13","parallel":1,"message":"Pull mode"} + - result.systemerr ShouldContainSubstring {"level":"info","limit":1,"filter":{},"diagnostic":false,"distinct":false,"filter-from-file":"","exclude-from-file":"","table":"","where":"address_id > 13","parallel":1, - name: pull with filter from file and where clause steps: From 5ce46853dd5ce6e4cb33a1e7e89ff17468601592 Mon Sep 17 00:00:00 2001 From: Adrien Aury <44274230+adrienaury@users.noreply.github.com> Date: Wed, 13 Mar 2024 18:15:03 +0100 Subject: [PATCH 04/13] fix: clean close open --- internal/infra/pull/datasource_ws.go | 6 ++++++ internal/infra/pull/http_datasource.go | 2 ++ internal/infra/pull/rowreader_json.go | 4 ++++ internal/infra/pull/sql_datasource.go | 13 +++++++++++++ internal/infra/push/datadestination_sql.go | 22 ++++++++++++++++++---- pkg/pull/driven.go | 1 + pkg/pull/driven_DatasourceInMemory.go | 3 +++ pkg/pull/driven_OneEmptyRowReader.go | 3 +++ pkg/pull/driver.go | 2 ++ pkg/pull/driver_parallel.go | 2 ++ 10 files changed, 54 insertions(+), 4 deletions(-) diff --git a/internal/infra/pull/datasource_ws.go b/internal/infra/pull/datasource_ws.go index 516fbe9e..9d681e0b 100644 --- a/internal/infra/pull/datasource_ws.go +++ b/internal/infra/pull/datasource_ws.go @@ -152,6 +152,8 @@ func (ds *WSDataSource) Read(source pull.Table, filter pull.Filter) (pull.RowSet return nil, err } + defer reader.Close() + result := pull.RowSet{} for reader.Next() { result = append(result, reader.Value()) @@ -289,3 +291,7 @@ func (rs *ResultStream) Value() pull.Row { func (rs *ResultStream) Error() error { return rs.err } + +func (rs *ResultStream) Close() error { + return nil +} diff --git a/internal/infra/pull/http_datasource.go b/internal/infra/pull/http_datasource.go index 4dcb8fe1..a267cea9 100644 --- a/internal/infra/pull/http_datasource.go +++ b/internal/infra/pull/http_datasource.go @@ -63,6 +63,8 @@ func (ds *HTTPDataSource) Read(source pull.Table, filter pull.Filter) (pull.RowS return nil, err } + defer reader.Close() + result := pull.RowSet{} for reader.Next() { result = append(result, reader.Value()) diff --git a/internal/infra/pull/rowreader_json.go b/internal/infra/pull/rowreader_json.go index 240c1343..b18e8899 100644 --- a/internal/infra/pull/rowreader_json.go +++ b/internal/infra/pull/rowreader_json.go @@ -68,3 +68,7 @@ func (jrr *JSONRowReader) Value() pull.Row { func (jrr *JSONRowReader) Error() error { return jrr.err } + +func (jrr *JSONRowReader) Close() error { + return nil +} diff --git a/internal/infra/pull/sql_datasource.go b/internal/infra/pull/sql_datasource.go index 7ba78ce5..3b112575 100644 --- a/internal/infra/pull/sql_datasource.go +++ b/internal/infra/pull/sql_datasource.go @@ -71,6 +71,8 @@ func (ds *SQLDataSource) Open() error { return err } + log.Error().Msg("open database connection pool") + // database handle settings db.SetConnMaxLifetime(ds.maxLifetime) db.SetMaxOpenConns(ds.maxOpenConns) @@ -129,6 +131,8 @@ func (ds *SQLDataSource) Read(source pull.Table, filter pull.Filter) (pull.RowSe return nil, err } + defer reader.Close() + result := pull.RowSet{} for reader.Next() { result = append(result, reader.Value()) @@ -160,6 +164,8 @@ func (ds *SQLDataSource) RowReader(source pull.Table, filter pull.Filter) (pull. return nil, err } + log.Error().Msg("open database rows iterator") + return &SQLDataIterator{rows, nil, nil}, nil } @@ -221,6 +227,7 @@ func (ds *SQLDataSource) Close() error { if err != nil { return err } + log.Error().Msg("close database connection pool") return nil } @@ -272,6 +279,12 @@ func (di *SQLDataIterator) Error() error { return di.err } +// Close returns the iterator +func (di *SQLDataIterator) Close() error { + defer log.Error().Msg("close database rows iterator") + return di.rows.Close() +} + func NewSQLDataSource(url, schema string, dbx *sqlx.DB, db *sql.DB, dialect commonsql.Dialect) *SQLDataSource { return &SQLDataSource{ url: url, diff --git a/internal/infra/push/datadestination_sql.go b/internal/infra/push/datadestination_sql.go index b347b323..484bb82a 100644 --- a/internal/infra/push/datadestination_sql.go +++ b/internal/infra/push/datadestination_sql.go @@ -115,6 +115,8 @@ func (dd *SQLDataDestination) Close() *push.Error { errors = append(errors, &push.Error{Description: err2.Error()}) } + log.Error().Msg("close database connection pool") + if len(errors) > 0 { allErrors := &push.Error{} for _, err := range errors { @@ -141,11 +143,15 @@ func (dd *SQLDataDestination) Commit() *push.Error { } log.Debug().Msg("transaction committed") + log.Error().Msg("commit database transaction") + tx, err := dd.db.Begin() if err != nil { return &push.Error{Description: err.Error()} } + log.Error().Msg("open database transaction") + dd.tx = tx return nil @@ -161,6 +167,8 @@ func (dd *SQLDataDestination) Open(plan push.Plan, mode push.Mode, disableConstr return &push.Error{Description: err.Error()} } + log.Error().Msg("open database connection pool") + // database handle settings db.SetConnMaxLifetime(dd.maxLifetime) db.SetMaxOpenConns(dd.maxOpenConns) @@ -186,6 +194,8 @@ func (dd *SQLDataDestination) Open(plan push.Plan, mode push.Mode, disableConstr } dd.tx = tx + log.Error().Msg("open database transaction") + for _, table := range plan.Tables() { rw := NewSQLRowWriter(table, dd) err := rw.open() @@ -263,6 +273,7 @@ func (rw *SQLRowWriter) close() *push.Error { if err != nil { return &push.Error{Description: err.Error()} } + log.Error().Msg("close database statement") rw.statement = nil log.Debug().Msg(fmt.Sprintf("close statement %s", rw.dd.mode)) } @@ -320,6 +331,7 @@ func (rw *SQLRowWriter) createStatement(row push.Row, where push.Row) *push.Erro if err != nil { return &push.Error{Description: err.Error()} } + log.Error().Msg("open database statement") rw.statement = stmt return nil } @@ -386,12 +398,12 @@ func (rw *SQLRowWriter) Write(row push.Row, where push.Row) *push.Error { _, err2 := rw.statement.Exec(values...) if err2 != nil { // reset statement after error - if err := rw.close(); err != nil { - return &push.Error{Description: err.Error() + "\noriginal error :\n" + err2.Error()} - } if rw.dd.dialect.IsDuplicateError(err2) { log.Trace().Msg(fmt.Sprintf("duplicate key %v (%s) for %s", row, rw.table.PrimaryKey(), rw.table.Name())) } else { + if err := rw.close(); err != nil { + return &push.Error{Description: err.Error() + "\noriginal error :\n" + err2.Error()} + } return &push.Error{Description: err2.Error()} } } @@ -419,7 +431,9 @@ func (rw *SQLRowWriter) disableConstraints() *push.Error { return &push.Error{Description: err.Error()} } - defer result.Close() + log.Error().Msg("open database rows iterator") + + defer func() { result.Close(); log.Error().Msg("close database rows iterator") }() var tableName, constraintName string for result.Next() { diff --git a/pkg/pull/driven.go b/pkg/pull/driven.go index ab9dc8db..3fdc46eb 100755 --- a/pkg/pull/driven.go +++ b/pkg/pull/driven.go @@ -42,6 +42,7 @@ type RowReader interface { Next() bool Value() Row Error() error + Close() error } // TraceListener receives diagnostic trace. diff --git a/pkg/pull/driven_DatasourceInMemory.go b/pkg/pull/driven_DatasourceInMemory.go index 196c5f2e..a050439c 100644 --- a/pkg/pull/driven_DatasourceInMemory.go +++ b/pkg/pull/driven_DatasourceInMemory.go @@ -40,6 +40,8 @@ func (ds DataSourceInMemory) Read(source Table, filter Filter) (RowSet, error) { return nil, err } + defer reader.Close() + result := RowSet{} for reader.Next() { result = append(result, reader.Value()) @@ -104,3 +106,4 @@ func (rr *RowReaderInMemory) Value() Row { return row } func (rr *RowReaderInMemory) Error() error { return nil } +func (rr *RowReaderInMemory) Close() error { return nil } diff --git a/pkg/pull/driven_OneEmptyRowReader.go b/pkg/pull/driven_OneEmptyRowReader.go index 0ab9d88e..43291596 100644 --- a/pkg/pull/driven_OneEmptyRowReader.go +++ b/pkg/pull/driven_OneEmptyRowReader.go @@ -38,3 +38,6 @@ func (r OneEmptyRowReader) Value() Row { return Row{} } // Error return always nil func (r OneEmptyRowReader) Error() error { return nil } + +// Close return always nil +func (r OneEmptyRowReader) Close() error { return nil } diff --git a/pkg/pull/driver.go b/pkg/pull/driver.go index c35b582b..5957dc8d 100755 --- a/pkg/pull/driver.go +++ b/pkg/pull/driver.go @@ -110,6 +110,8 @@ func (p *puller) Pull(start Table, filter Filter, filterCohort RowReader, exclud return fmt.Errorf("%w", err) } + defer reader.Close() + for reader.Next() { IncLinesPerStepCount(string(start.Name)) row := start.export(reader.Value()) diff --git a/pkg/pull/driver_parallel.go b/pkg/pull/driver_parallel.go index ea55c821..9d1daf49 100644 --- a/pkg/pull/driver_parallel.go +++ b/pkg/pull/driver_parallel.go @@ -121,6 +121,8 @@ func (p *pullerParallel) Pull(start Table, filter Filter, filterCohort RowReader return fmt.Errorf("%w", err) } + defer reader.Close() + for reader.Next() { IncLinesPerStepCount(string(start.Name)) p.inChan <- reader.Value() From 706d59787afb2878ea0b1018f6c6cd8af7f0f31f Mon Sep 17 00:00:00 2001 From: Adrien Aury <44274230+adrienaury@users.noreply.github.com> Date: Wed, 13 Mar 2024 18:24:22 +0100 Subject: [PATCH 05/13] fix: clean close open log info --- internal/infra/pull/sql_datasource.go | 8 ++++---- internal/infra/push/datadestination_sql.go | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/internal/infra/pull/sql_datasource.go b/internal/infra/pull/sql_datasource.go index 3b112575..dbb20c62 100644 --- a/internal/infra/pull/sql_datasource.go +++ b/internal/infra/pull/sql_datasource.go @@ -71,7 +71,7 @@ func (ds *SQLDataSource) Open() error { return err } - log.Error().Msg("open database connection pool") + log.Info().Msg("open database connection pool") // database handle settings db.SetConnMaxLifetime(ds.maxLifetime) @@ -164,7 +164,7 @@ func (ds *SQLDataSource) RowReader(source pull.Table, filter pull.Filter) (pull. return nil, err } - log.Error().Msg("open database rows iterator") + log.Info().Msg("open database rows iterator") return &SQLDataIterator{rows, nil, nil}, nil } @@ -227,7 +227,7 @@ func (ds *SQLDataSource) Close() error { if err != nil { return err } - log.Error().Msg("close database connection pool") + log.Info().Msg("close database connection pool") return nil } @@ -281,7 +281,7 @@ func (di *SQLDataIterator) Error() error { // Close returns the iterator func (di *SQLDataIterator) Close() error { - defer log.Error().Msg("close database rows iterator") + defer log.Info().Msg("close database rows iterator") return di.rows.Close() } diff --git a/internal/infra/push/datadestination_sql.go b/internal/infra/push/datadestination_sql.go index 484bb82a..9622a576 100644 --- a/internal/infra/push/datadestination_sql.go +++ b/internal/infra/push/datadestination_sql.go @@ -115,7 +115,7 @@ func (dd *SQLDataDestination) Close() *push.Error { errors = append(errors, &push.Error{Description: err2.Error()}) } - log.Error().Msg("close database connection pool") + log.Info().Msg("close database connection pool") if len(errors) > 0 { allErrors := &push.Error{} @@ -150,7 +150,7 @@ func (dd *SQLDataDestination) Commit() *push.Error { return &push.Error{Description: err.Error()} } - log.Error().Msg("open database transaction") + log.Info().Msg("open database transaction") dd.tx = tx @@ -167,7 +167,7 @@ func (dd *SQLDataDestination) Open(plan push.Plan, mode push.Mode, disableConstr return &push.Error{Description: err.Error()} } - log.Error().Msg("open database connection pool") + log.Info().Msg("open database connection pool") // database handle settings db.SetConnMaxLifetime(dd.maxLifetime) @@ -194,7 +194,7 @@ func (dd *SQLDataDestination) Open(plan push.Plan, mode push.Mode, disableConstr } dd.tx = tx - log.Error().Msg("open database transaction") + log.Info().Msg("open database transaction") for _, table := range plan.Tables() { rw := NewSQLRowWriter(table, dd) @@ -273,7 +273,7 @@ func (rw *SQLRowWriter) close() *push.Error { if err != nil { return &push.Error{Description: err.Error()} } - log.Error().Msg("close database statement") + log.Info().Msg("close database statement") rw.statement = nil log.Debug().Msg(fmt.Sprintf("close statement %s", rw.dd.mode)) } @@ -331,7 +331,7 @@ func (rw *SQLRowWriter) createStatement(row push.Row, where push.Row) *push.Erro if err != nil { return &push.Error{Description: err.Error()} } - log.Error().Msg("open database statement") + log.Info().Msg("open database statement") rw.statement = stmt return nil } @@ -431,9 +431,9 @@ func (rw *SQLRowWriter) disableConstraints() *push.Error { return &push.Error{Description: err.Error()} } - log.Error().Msg("open database rows iterator") + log.Info().Msg("open database rows iterator") - defer func() { result.Close(); log.Error().Msg("close database rows iterator") }() + defer func() { result.Close(); log.Info().Msg("close database rows iterator") }() var tableName, constraintName string for result.Next() { From 843534b061392fcc707a793b946a440164925e2f Mon Sep 17 00:00:00 2001 From: Adrien Aury <44274230+adrienaury@users.noreply.github.com> Date: Wed, 13 Mar 2024 18:32:04 +0100 Subject: [PATCH 06/13] fix: clean close open log info --- internal/infra/push/datadestination_sql.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/infra/push/datadestination_sql.go b/internal/infra/push/datadestination_sql.go index 9622a576..2396d18e 100644 --- a/internal/infra/push/datadestination_sql.go +++ b/internal/infra/push/datadestination_sql.go @@ -143,7 +143,7 @@ func (dd *SQLDataDestination) Commit() *push.Error { } log.Debug().Msg("transaction committed") - log.Error().Msg("commit database transaction") + log.Info().Msg("close (commit) database transaction") tx, err := dd.db.Begin() if err != nil { From e64d1d35d9c46c89cf35afc8237114a86694f9d9 Mon Sep 17 00:00:00 2001 From: Adrien Aury <44274230+adrienaury@users.noreply.github.com> Date: Wed, 13 Mar 2024 18:59:28 +0100 Subject: [PATCH 07/13] docs: changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ede9c45..c69be571 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,10 @@ Types of changes - `Fixed` for any bug fixes. - `Security` in case of vulnerabilities. +## [2.8.0] + +- `Added` flags `--conn-max-lifetime`, `--conn-max-open` and `--conn-max-idle` to `lino pull` and `lino push` commands + ## [2.7.1] - `Fixed` panic during push on Oracle database with a `null` column value From 7541df99865822e244e512340c19fffe0e401b52 Mon Sep 17 00:00:00 2001 From: Adrien Aury <44274230+adrienaury@users.noreply.github.com> Date: Mon, 18 Mar 2024 14:41:43 +0100 Subject: [PATCH 08/13] feat: try other mysql driver --- go.mod | 8 ++- go.sum | 56 ++++++++++++++++++-- internal/infra/pull/datasource_mariadb.go | 2 +- internal/infra/relation/extractor_mariadb.go | 2 +- internal/infra/table/extractor_mariadb.go | 2 +- 5 files changed, 61 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 9d2c98b5..9b2fb9b1 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/cgi-fr/jsonline v0.5.0 github.com/cgi-fr/rimo v0.4.0 github.com/docker/docker-credential-helpers v0.8.1 - github.com/go-sql-driver/mysql v1.8.0 + github.com/go-mysql-org/go-mysql v1.7.0 github.com/gorilla/mux v1.8.1 github.com/hashicorp/go-multierror v1.1.1 github.com/ibmdb/go_ibm_db v0.4.5 @@ -30,10 +30,10 @@ require ( ) require ( - filippo.io/edwards25519 v1.1.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect github.com/golang-sql/sqlexp v0.1.0 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/iancoleman/orderedmap v0.3.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect @@ -41,9 +41,13 @@ require ( github.com/kr/text v0.2.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-sqlite3 v2.0.3+incompatible // indirect + github.com/pingcap/errors v0.11.5-0.20210425183316-da1aaba5fb63 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 // indirect + github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/objx v0.5.2 // indirect + go.uber.org/atomic v1.7.0 // indirect golang.org/x/crypto v0.18.0 // indirect golang.org/x/sys v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect diff --git a/go.sum b/go.sum index 502a3d0e..0859fa99 100644 --- a/go.sum +++ b/go.sum @@ -43,8 +43,6 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= -filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 h1:lGlwhPtrX6EVml1hO0ivjkUxsSyl4dsiw9qcA1k/3IQ= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 h1:sO0/P7g68FrryJzljemN+6GTssUXdANk6aJ7T1ZxnsQ= github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 h1:6oNBlSdi1QqM1PNW7FPA6xOGA5UNsXnkaYZz9vdPGhA= @@ -66,6 +64,7 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/awalterschulze/gographviz v2.0.3+incompatible h1:9sVEXJBJLwGX7EQVhLm2elIKCm7P2YHFC8v6096G09E= github.com/awalterschulze/gographviz v2.0.3+incompatible/go.mod h1:GEV5wmg4YquNw7v1kkyoX9etIk8yVmXj+AkDHuuETHs= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -89,6 +88,9 @@ github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= +github.com/cznic/sortutil v0.0.0-20181122101858-f5f958428db8/go.mod h1:q2w6Bg5jeox1B+QkJ6Wp/+Vn0G/bo3f1uY7Fn3vivIQ= +github.com/cznic/strutil v0.0.0-20171016134553-529a34b1c186/go.mod h1:AHHPPPXTw0h6pVabbcbyGRK1DckRn7r/STdZEeIDzZc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -110,9 +112,11 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-mysql-org/go-mysql v1.7.0 h1:qE5FTRb3ZeTQmlk3pjE+/m2ravGxxRDrVDTyDe9tvqI= +github.com/go-mysql-org/go-mysql v1.7.0/go.mod h1:9cRWLtuXNKhamUPMkrDVzBhaomGvqLRLtBiyjvjc4pk= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= -github.com/go-sql-driver/mysql v1.8.0 h1:UtktXaU2Nb64z/pLiGIxY4431SJ4/dR5cjMmlVHgnT4= -github.com/go-sql-driver/mysql v1.8.0/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= @@ -187,7 +191,9 @@ github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= @@ -239,6 +245,7 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/invopop/jsonschema v0.7.0 h1:2vgQcBz1n256N+FpX3Jq7Y17AjYt46Ig3zIWyy770So= github.com/invopop/jsonschema v0.7.0/go.mod h1:O9uiLokuu0+MGFlyiaqtWxwqJm41/+8Nj0lD7A36YH0= +github.com/jmoiron/sqlx v1.3.3/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ= github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -302,6 +309,13 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8 h1:USx2/E1bX46VG32FIw034Au6seQ2fY9NEILmNh/UlQg= +github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ= +github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= +github.com/pingcap/errors v0.11.5-0.20210425183316-da1aaba5fb63 h1:+FZIDR/D97YOPik4N4lPDaUcLDF/EQPogxtlHB2ZZRM= +github.com/pingcap/errors v0.11.5-0.20210425183316-da1aaba5fb63/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= +github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM= +github.com/pingcap/tidb/parser v0.0.0-20221126021158-6b02a5d8ba7d/go.mod h1:ElJiub4lRy6UZDb+0JHDkGEdr6aOli+ykhyej7VCLoI= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -311,6 +325,7 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= @@ -326,7 +341,12 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sagikazarmark/crypt v0.1.0/go.mod h1:B/mN0msZuINBtQ1zZLEQcegFJJf9vnYIR88KRMEuODE= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 h1:xT+JlYxNGqyT+XcU8iUrN18JYed2TvG9yN5ULG2jATM= +github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726/go.mod h1:3yhqj7WBBfRhbBlzyOC3gUxftwsU0u8gqevxwIHQpMw= +github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 h1:oI+RNwuC9jF2g2lP0u0cVEEZrc/AYBCuFdvwrLWM/6Q= +github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07/go.mod h1:yFdBgwXP24JziuRl2NMUahT7nGLNOKi1SIiFxMttVD4= github.com/sijms/go-ora/v2 v2.8.9 h1:XIghcG8hjtYu+G9H235VEe5JXPLJtdzzj7pQm7JucVo= github.com/sijms/go-ora/v2 v2.8.9/go.mod h1:EHxlY6x7y9HAsdfumurRfTd+v8NrEOTR3Xl4FWlH6xk= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= @@ -344,6 +364,7 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= github.com/spf13/viper v1.9.0/go.mod h1:+i6ajR7OX2XaiBkrcZJFK21htRk7eDeLg7+O6bhUPP4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -353,6 +374,8 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= @@ -375,9 +398,16 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -389,6 +419,7 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= +golang.org/x/exp v0.0.0-20181106170214-d68db9428509/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -587,6 +618,8 @@ golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -616,6 +649,7 @@ golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201125231158-b5590deeca9b/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -764,6 +798,7 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.63.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -779,6 +814,19 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +modernc.org/fileutil v1.0.0/go.mod h1:JHsWpkrk/CnVV1H/eGlFf85BEpfkrp56ro8nojIq9Q8= +modernc.org/golex v1.0.1/go.mod h1:QCA53QtsT1NdGkaZZkF5ezFwk4IXh4BGNafAARTC254= +modernc.org/lex v1.0.0/go.mod h1:G6rxMTy3cH2iA0iXL/HRRv4Znu8MK4higxph/lE7ypk= +modernc.org/lexer v1.0.0/go.mod h1:F/Dld0YKYdZCLQ7bD0USbWL4YKCyTDRDHiDTOs0q0vk= +modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= +modernc.org/mathutil v1.4.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/parser v1.0.0/go.mod h1:H20AntYJ2cHHL6MHthJ8LZzXCdDCHMWt1KZXtIMjejA= +modernc.org/parser v1.0.2/go.mod h1:TXNq3HABP3HMaqLK7brD1fLA/LfN0KS6JxZn71QdDqs= +modernc.org/scanner v1.0.1/go.mod h1:OIzD2ZtjYk6yTuyqZr57FmifbM9fIH74SumloSsajuE= +modernc.org/sortutil v1.0.0/go.mod h1:1QO0q8IlIlmjBIwm6t/7sof874+xCfZouyqZMLIAtxM= +modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= +modernc.org/strutil v1.1.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= +modernc.org/y v1.0.1/go.mod h1:Ho86I+LVHEI+LYXoUKlmOMAM1JTXOCfj8qi1T8PsClE= nhooyr.io/websocket v1.8.10 h1:mv4p+MnGrLDcPlBoWsvPP7XCzTYMXP9F9eIGoKbgx7Q= nhooyr.io/websocket v1.8.10/go.mod h1:rN9OFWIUwuxg4fR5tELlYC04bXYowCP9GX47ivo2l+c= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/internal/infra/pull/datasource_mariadb.go b/internal/infra/pull/datasource_mariadb.go index 4fb4828e..8b2da715 100644 --- a/internal/infra/pull/datasource_mariadb.go +++ b/internal/infra/pull/datasource_mariadb.go @@ -21,7 +21,7 @@ import ( "github.com/cgi-fr/lino/internal/infra/commonsql" "github.com/cgi-fr/lino/pkg/pull" - _ "github.com/go-sql-driver/mysql" + _ "github.com/go-mysql-org/go-mysql/driver" ) // MariadbDataSourceFactory exposes methods to create new Mariadb pullers. diff --git a/internal/infra/relation/extractor_mariadb.go b/internal/infra/relation/extractor_mariadb.go index e3ec42d0..05b43bff 100644 --- a/internal/infra/relation/extractor_mariadb.go +++ b/internal/infra/relation/extractor_mariadb.go @@ -22,7 +22,7 @@ import ( // import mariadb connector "fmt" - _ "github.com/go-sql-driver/mysql" + _ "github.com/go-mysql-org/go-mysql/driver" "github.com/cgi-fr/lino/pkg/relation" ) diff --git a/internal/infra/table/extractor_mariadb.go b/internal/infra/table/extractor_mariadb.go index b3414c36..6464a73e 100644 --- a/internal/infra/table/extractor_mariadb.go +++ b/internal/infra/table/extractor_mariadb.go @@ -21,7 +21,7 @@ import ( "fmt" // import mariadbsql connector - _ "github.com/go-sql-driver/mysql" + _ "github.com/go-mysql-org/go-mysql/driver" "github.com/cgi-fr/lino/internal/infra/commonsql" "github.com/cgi-fr/lino/pkg/table" From 6ffd0fdcda1aeb22b084be2c820c5c580c15dfcb Mon Sep 17 00:00:00 2001 From: Adrien Aury <44274230+adrienaury@users.noreply.github.com> Date: Mon, 18 Mar 2024 15:42:10 +0100 Subject: [PATCH 09/13] feat: try other mysql driver --- internal/app/urlbuilder/urlbuilder.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/internal/app/urlbuilder/urlbuilder.go b/internal/app/urlbuilder/urlbuilder.go index faa45dba..bb348de8 100644 --- a/internal/app/urlbuilder/urlbuilder.go +++ b/internal/app/urlbuilder/urlbuilder.go @@ -80,6 +80,15 @@ func genJDBCOracle(u *dburl.URL) (string, string, error) { return go_ora.BuildJDBC("", "", u.Host, map[string]string{}), "", nil } +func genMySQL(u *dburl.URL) (string, string, error) { + dsn, other, err := dburl.GenMysql(u) + if err != nil { + return dsn, other, err + } + + return strings.ReplaceAll(dsn, "/", "?"), other, err +} + func init() { oracleScheme := dburl.Scheme{ Driver: "godror-raw", @@ -125,6 +134,16 @@ func init() { Override: "", } dburl.Register(wsScheme) + + mySQLScheme := dburl.Scheme{ + Driver: "mysql-test", + Generator: genMySQL, + Transport: dburl.TransportAny, + Opaque: false, + Aliases: []string{}, + Override: "", + } + dburl.Register(mySQLScheme) } func BuildURL(dc *dataconnector.DataConnector, out io.Writer) *dburl.URL { From 58d0f625e19005e593d47116488a762c2691d10b Mon Sep 17 00:00:00 2001 From: Adrien Aury <44274230+adrienaury@users.noreply.github.com> Date: Mon, 18 Mar 2024 15:52:53 +0100 Subject: [PATCH 10/13] feat: try other mysql driver --- internal/app/urlbuilder/urlbuilder.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/app/urlbuilder/urlbuilder.go b/internal/app/urlbuilder/urlbuilder.go index bb348de8..2b3b8ff6 100644 --- a/internal/app/urlbuilder/urlbuilder.go +++ b/internal/app/urlbuilder/urlbuilder.go @@ -135,8 +135,9 @@ func init() { } dburl.Register(wsScheme) + dburl.Unregister("mysql") mySQLScheme := dburl.Scheme{ - Driver: "mysql-test", + Driver: "mysql", Generator: genMySQL, Transport: dburl.TransportAny, Opaque: false, From d1dcb6a4d5c748b0c35ac4ba280f65c673754029 Mon Sep 17 00:00:00 2001 From: Adrien Aury <44274230+adrienaury@users.noreply.github.com> Date: Thu, 21 Mar 2024 09:30:18 +0100 Subject: [PATCH 11/13] fix: MySQL driver connection string --- internal/app/urlbuilder/urlbuilder.go | 34 +++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/internal/app/urlbuilder/urlbuilder.go b/internal/app/urlbuilder/urlbuilder.go index 2b3b8ff6..48e1b2fb 100644 --- a/internal/app/urlbuilder/urlbuilder.go +++ b/internal/app/urlbuilder/urlbuilder.go @@ -81,12 +81,36 @@ func genJDBCOracle(u *dburl.URL) (string, string, error) { } func genMySQL(u *dburl.URL) (string, string, error) { - dsn, other, err := dburl.GenMysql(u) - if err != nil { - return dsn, other, err + host, port, dbname := u.Hostname(), u.Port(), strings.TrimPrefix(u.Path, "/") + // build dsn + var dsn string + if u.User != nil { + if n := u.User.Username(); n != "" { + if p, ok := u.User.Password(); ok { + n += ":" + p + } + dsn += n + "@" + } } - - return strings.ReplaceAll(dsn, "/", "?"), other, err + // if host or proto is not empty + if u.Transport != "unix" { + if host == "" { + host = "localhost" + } + if port == "" { + port = "3306" + } + } + if port != "" { + port = ":" + port + } + // add proto and database + dsn += host + port + "/" + dbname + // query + if s := u.Query().Encode(); s != "" { + dsn += "?" + s + } + return dsn, "", nil } func init() { From 5f2f3146ea18ac626410472684bc434f7a7fccb1 Mon Sep 17 00:00:00 2001 From: Adrien Aury <44274230+adrienaury@users.noreply.github.com> Date: Thu, 21 Mar 2024 11:30:19 +0100 Subject: [PATCH 12/13] feat: test mymysql driver --- cmd/lino/dep_analyse.go | 1 + cmd/lino/dep_dataconnector.go | 1 + cmd/lino/dep_pull.go | 1 + cmd/lino/dep_push.go | 1 + go.mod | 7 +-- go.sum | 53 +------------------- internal/app/urlbuilder/urlbuilder.go | 20 ++++---- internal/infra/pull/datasource_mariadb.go | 2 +- internal/infra/relation/extractor_mariadb.go | 2 +- internal/infra/table/extractor_mariadb.go | 2 +- 10 files changed, 20 insertions(+), 70 deletions(-) diff --git a/cmd/lino/dep_analyse.go b/cmd/lino/dep_analyse.go index 92830c96..f88fba01 100644 --- a/cmd/lino/dep_analyse.go +++ b/cmd/lino/dep_analyse.go @@ -27,6 +27,7 @@ func analyseDataSourceFactory() map[string]infra.SQLExtractorFactory { "godror": infra.NewOracleExtractorFactory(), "godror-raw": infra.NewOracleExtractorFactory(), "mysql": infra.NewMariaDBExtractorFactory(), + "mymysql": infra.NewMariaDBExtractorFactory(), "db2": infra.NewDB2ExtractorFactory(), "sqlserver": infra.NewSQLServerExtractorFactory(), } diff --git a/cmd/lino/dep_dataconnector.go b/cmd/lino/dep_dataconnector.go index a06f7ba1..4194e248 100755 --- a/cmd/lino/dep_dataconnector.go +++ b/cmd/lino/dep_dataconnector.go @@ -32,6 +32,7 @@ func dataPingerFactory() map[string]domain.DataPingerFactory { "godror": infra.NewSQLDataPingerFactory(), "godror-raw": infra.NewSQLDataPingerFactory(), "mysql": infra.NewSQLDataPingerFactory(), + "mymysql": infra.NewSQLDataPingerFactory(), "db2": infra.NewSQLDataPingerFactory(), "http": infra.NewHTTPDataPingerFactory(), "ws": infra.NewWSDataPingerFactory(), diff --git a/cmd/lino/dep_pull.go b/cmd/lino/dep_pull.go index 3eeb4428..8224c055 100755 --- a/cmd/lino/dep_pull.go +++ b/cmd/lino/dep_pull.go @@ -32,6 +32,7 @@ func pullDataSourceFactory() map[string]domain.DataSourceFactory { "godror": infra.NewOracleDataSourceFactory(), "godror-raw": infra.NewOracleDataSourceFactory(), "mysql": infra.NewMariadbDataSourceFactory(), + "mymysql": infra.NewMariadbDataSourceFactory(), "db2": infra.NewDb2DataSourceFactory(), "http": infra.NewHTTPDataSourceFactory(), "ws": infra.NewWSDataSourceFactory(), diff --git a/cmd/lino/dep_push.go b/cmd/lino/dep_push.go index 86f45dd0..017c18fb 100755 --- a/cmd/lino/dep_push.go +++ b/cmd/lino/dep_push.go @@ -31,6 +31,7 @@ func pushDataDestinationFactory() map[string]domain.DataDestinationFactory { "godror": infra.NewOracleDataDestinationFactory(), "godror-raw": infra.NewOracleDataDestinationFactory(), "mysql": infra.NewMariadbDataDestinationFactory(), + "mymysql": infra.NewMariadbDataDestinationFactory(), "db2": infra.NewDb2DataDestinationFactory(), "http": infra.NewHTTPDataDestinationFactory(), "ws": infra.NewWebSocketDataDestinationFactory(), diff --git a/go.mod b/go.mod index 9b2fb9b1..0517dff3 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,6 @@ require ( github.com/cgi-fr/jsonline v0.5.0 github.com/cgi-fr/rimo v0.4.0 github.com/docker/docker-credential-helpers v0.8.1 - github.com/go-mysql-org/go-mysql v1.7.0 github.com/gorilla/mux v1.8.1 github.com/hashicorp/go-multierror v1.1.1 github.com/ibmdb/go_ibm_db v0.4.5 @@ -23,6 +22,7 @@ require ( github.com/spf13/cobra v1.8.0 github.com/stretchr/testify v1.9.0 github.com/xo/dburl v0.21.1 + github.com/ziutek/mymysql v1.5.4 golang.org/x/exp v0.0.0-20231006140011-7918f672742d golang.org/x/term v0.18.0 gopkg.in/yaml.v3 v3.0.1 @@ -33,7 +33,6 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect github.com/golang-sql/sqlexp v0.1.0 // indirect - github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/iancoleman/orderedmap v0.3.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect @@ -41,13 +40,9 @@ require ( github.com/kr/text v0.2.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-sqlite3 v2.0.3+incompatible // indirect - github.com/pingcap/errors v0.11.5-0.20210425183316-da1aaba5fb63 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 // indirect - github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/objx v0.5.2 // indirect - go.uber.org/atomic v1.7.0 // indirect golang.org/x/crypto v0.18.0 // indirect golang.org/x/sys v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect diff --git a/go.sum b/go.sum index 0859fa99..9ea01f96 100644 --- a/go.sum +++ b/go.sum @@ -64,7 +64,6 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/awalterschulze/gographviz v2.0.3+incompatible h1:9sVEXJBJLwGX7EQVhLm2elIKCm7P2YHFC8v6096G09E= github.com/awalterschulze/gographviz v2.0.3+incompatible/go.mod h1:GEV5wmg4YquNw7v1kkyoX9etIk8yVmXj+AkDHuuETHs= -github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -88,9 +87,6 @@ github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= -github.com/cznic/sortutil v0.0.0-20181122101858-f5f958428db8/go.mod h1:q2w6Bg5jeox1B+QkJ6Wp/+Vn0G/bo3f1uY7Fn3vivIQ= -github.com/cznic/strutil v0.0.0-20171016134553-529a34b1c186/go.mod h1:AHHPPPXTw0h6pVabbcbyGRK1DckRn7r/STdZEeIDzZc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -112,9 +108,6 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-mysql-org/go-mysql v1.7.0 h1:qE5FTRb3ZeTQmlk3pjE+/m2ravGxxRDrVDTyDe9tvqI= -github.com/go-mysql-org/go-mysql v1.7.0/go.mod h1:9cRWLtuXNKhamUPMkrDVzBhaomGvqLRLtBiyjvjc4pk= -github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= @@ -191,9 +184,7 @@ github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= @@ -245,7 +236,6 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/invopop/jsonschema v0.7.0 h1:2vgQcBz1n256N+FpX3Jq7Y17AjYt46Ig3zIWyy770So= github.com/invopop/jsonschema v0.7.0/go.mod h1:O9uiLokuu0+MGFlyiaqtWxwqJm41/+8Nj0lD7A36YH0= -github.com/jmoiron/sqlx v1.3.3/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ= github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -309,13 +299,6 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8 h1:USx2/E1bX46VG32FIw034Au6seQ2fY9NEILmNh/UlQg= -github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ= -github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= -github.com/pingcap/errors v0.11.5-0.20210425183316-da1aaba5fb63 h1:+FZIDR/D97YOPik4N4lPDaUcLDF/EQPogxtlHB2ZZRM= -github.com/pingcap/errors v0.11.5-0.20210425183316-da1aaba5fb63/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= -github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM= -github.com/pingcap/tidb/parser v0.0.0-20221126021158-6b02a5d8ba7d/go.mod h1:ElJiub4lRy6UZDb+0JHDkGEdr6aOli+ykhyej7VCLoI= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -325,7 +308,6 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= @@ -341,12 +323,7 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sagikazarmark/crypt v0.1.0/go.mod h1:B/mN0msZuINBtQ1zZLEQcegFJJf9vnYIR88KRMEuODE= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 h1:xT+JlYxNGqyT+XcU8iUrN18JYed2TvG9yN5ULG2jATM= -github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726/go.mod h1:3yhqj7WBBfRhbBlzyOC3gUxftwsU0u8gqevxwIHQpMw= -github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 h1:oI+RNwuC9jF2g2lP0u0cVEEZrc/AYBCuFdvwrLWM/6Q= -github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07/go.mod h1:yFdBgwXP24JziuRl2NMUahT7nGLNOKi1SIiFxMttVD4= github.com/sijms/go-ora/v2 v2.8.9 h1:XIghcG8hjtYu+G9H235VEe5JXPLJtdzzj7pQm7JucVo= github.com/sijms/go-ora/v2 v2.8.9/go.mod h1:EHxlY6x7y9HAsdfumurRfTd+v8NrEOTR3Xl4FWlH6xk= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= @@ -364,7 +341,6 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= github.com/spf13/viper v1.9.0/go.mod h1:+i6ajR7OX2XaiBkrcZJFK21htRk7eDeLg7+O6bhUPP4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -374,8 +350,6 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= @@ -387,6 +361,8 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= +github.com/ziutek/mymysql v1.5.4 h1:GB0qdRGsTwQSBVYuVShFBKaXSnSnYYC2d9knnE1LHFs= +github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= @@ -398,16 +374,9 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= -go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -419,7 +388,6 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= -golang.org/x/exp v0.0.0-20181106170214-d68db9428509/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -618,8 +586,6 @@ golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -649,7 +615,6 @@ golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201125231158-b5590deeca9b/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -798,7 +763,6 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.63.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -814,19 +778,6 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -modernc.org/fileutil v1.0.0/go.mod h1:JHsWpkrk/CnVV1H/eGlFf85BEpfkrp56ro8nojIq9Q8= -modernc.org/golex v1.0.1/go.mod h1:QCA53QtsT1NdGkaZZkF5ezFwk4IXh4BGNafAARTC254= -modernc.org/lex v1.0.0/go.mod h1:G6rxMTy3cH2iA0iXL/HRRv4Znu8MK4higxph/lE7ypk= -modernc.org/lexer v1.0.0/go.mod h1:F/Dld0YKYdZCLQ7bD0USbWL4YKCyTDRDHiDTOs0q0vk= -modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= -modernc.org/mathutil v1.4.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= -modernc.org/parser v1.0.0/go.mod h1:H20AntYJ2cHHL6MHthJ8LZzXCdDCHMWt1KZXtIMjejA= -modernc.org/parser v1.0.2/go.mod h1:TXNq3HABP3HMaqLK7brD1fLA/LfN0KS6JxZn71QdDqs= -modernc.org/scanner v1.0.1/go.mod h1:OIzD2ZtjYk6yTuyqZr57FmifbM9fIH74SumloSsajuE= -modernc.org/sortutil v1.0.0/go.mod h1:1QO0q8IlIlmjBIwm6t/7sof874+xCfZouyqZMLIAtxM= -modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= -modernc.org/strutil v1.1.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= -modernc.org/y v1.0.1/go.mod h1:Ho86I+LVHEI+LYXoUKlmOMAM1JTXOCfj8qi1T8PsClE= nhooyr.io/websocket v1.8.10 h1:mv4p+MnGrLDcPlBoWsvPP7XCzTYMXP9F9eIGoKbgx7Q= nhooyr.io/websocket v1.8.10/go.mod h1:rN9OFWIUwuxg4fR5tELlYC04bXYowCP9GX47ivo2l+c= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/internal/app/urlbuilder/urlbuilder.go b/internal/app/urlbuilder/urlbuilder.go index 48e1b2fb..ccfc644a 100644 --- a/internal/app/urlbuilder/urlbuilder.go +++ b/internal/app/urlbuilder/urlbuilder.go @@ -159,16 +159,16 @@ func init() { } dburl.Register(wsScheme) - dburl.Unregister("mysql") - mySQLScheme := dburl.Scheme{ - Driver: "mysql", - Generator: genMySQL, - Transport: dburl.TransportAny, - Opaque: false, - Aliases: []string{}, - Override: "", - } - dburl.Register(mySQLScheme) + // dburl.Unregister("mysql") + // mySQLScheme := dburl.Scheme{ + // Driver: "mysql", + // Generator: genMySQL, + // Transport: dburl.TransportAny, + // Opaque: false, + // Aliases: []string{}, + // Override: "", + // } + // dburl.Register(mySQLScheme) } func BuildURL(dc *dataconnector.DataConnector, out io.Writer) *dburl.URL { diff --git a/internal/infra/pull/datasource_mariadb.go b/internal/infra/pull/datasource_mariadb.go index 8b2da715..d2debc69 100644 --- a/internal/infra/pull/datasource_mariadb.go +++ b/internal/infra/pull/datasource_mariadb.go @@ -21,7 +21,7 @@ import ( "github.com/cgi-fr/lino/internal/infra/commonsql" "github.com/cgi-fr/lino/pkg/pull" - _ "github.com/go-mysql-org/go-mysql/driver" + _ "github.com/ziutek/mymysql/godrv" ) // MariadbDataSourceFactory exposes methods to create new Mariadb pullers. diff --git a/internal/infra/relation/extractor_mariadb.go b/internal/infra/relation/extractor_mariadb.go index 05b43bff..2d0beecc 100644 --- a/internal/infra/relation/extractor_mariadb.go +++ b/internal/infra/relation/extractor_mariadb.go @@ -22,7 +22,7 @@ import ( // import mariadb connector "fmt" - _ "github.com/go-mysql-org/go-mysql/driver" + _ "github.com/ziutek/mymysql/godrv" "github.com/cgi-fr/lino/pkg/relation" ) diff --git a/internal/infra/table/extractor_mariadb.go b/internal/infra/table/extractor_mariadb.go index 6464a73e..07c39431 100644 --- a/internal/infra/table/extractor_mariadb.go +++ b/internal/infra/table/extractor_mariadb.go @@ -21,7 +21,7 @@ import ( "fmt" // import mariadbsql connector - _ "github.com/go-mysql-org/go-mysql/driver" + _ "github.com/ziutek/mymysql/godrv" "github.com/cgi-fr/lino/internal/infra/commonsql" "github.com/cgi-fr/lino/pkg/table" From 95f8e82f8fae2901365736d5a255ecfcf75e43e4 Mon Sep 17 00:00:00 2001 From: Adrien Aury <44274230+adrienaury@users.noreply.github.com> Date: Thu, 21 Mar 2024 11:36:10 +0100 Subject: [PATCH 13/13] feat: test mymysql driver --- internal/app/urlbuilder/urlbuilder.go | 44 --------------------------- 1 file changed, 44 deletions(-) diff --git a/internal/app/urlbuilder/urlbuilder.go b/internal/app/urlbuilder/urlbuilder.go index ccfc644a..faa45dba 100644 --- a/internal/app/urlbuilder/urlbuilder.go +++ b/internal/app/urlbuilder/urlbuilder.go @@ -80,39 +80,6 @@ func genJDBCOracle(u *dburl.URL) (string, string, error) { return go_ora.BuildJDBC("", "", u.Host, map[string]string{}), "", nil } -func genMySQL(u *dburl.URL) (string, string, error) { - host, port, dbname := u.Hostname(), u.Port(), strings.TrimPrefix(u.Path, "/") - // build dsn - var dsn string - if u.User != nil { - if n := u.User.Username(); n != "" { - if p, ok := u.User.Password(); ok { - n += ":" + p - } - dsn += n + "@" - } - } - // if host or proto is not empty - if u.Transport != "unix" { - if host == "" { - host = "localhost" - } - if port == "" { - port = "3306" - } - } - if port != "" { - port = ":" + port - } - // add proto and database - dsn += host + port + "/" + dbname - // query - if s := u.Query().Encode(); s != "" { - dsn += "?" + s - } - return dsn, "", nil -} - func init() { oracleScheme := dburl.Scheme{ Driver: "godror-raw", @@ -158,17 +125,6 @@ func init() { Override: "", } dburl.Register(wsScheme) - - // dburl.Unregister("mysql") - // mySQLScheme := dburl.Scheme{ - // Driver: "mysql", - // Generator: genMySQL, - // Transport: dburl.TransportAny, - // Opaque: false, - // Aliases: []string{}, - // Override: "", - // } - // dburl.Register(mySQLScheme) } func BuildURL(dc *dataconnector.DataConnector, out io.Writer) *dburl.URL {