fix(pull): refactor select query with dialect commun sql (#267)

Chao-Ma5566 · web-flow · commit 277062da28b4 · 2024-03-06T15:09:20.000+01:00
* fix: init branch

* feat: add test push with nil value descriptor

* feat: test push with nil value descriptor pass

* refactor: update error message

* fix: update test convert value error message

* refactor: change check null position

* feat(table): add test for sql server commun sql

* feat: add get column names func for struct table

* fix(pull): delete get column names func

* fix(pull): replace create select by select selectlimit

* feat: separated row reader func to test select sql generated correctly

* docs: update change log

* feat: add SQL query test for each dialect
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -14,6 +14,11 @@ Types of changes
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
+## [2.7.1]
+
+- `Fixed` panic during push on Oracle database with a `null` column value.
+- `Fixed` issue with SQL queries involving limit orders in SQL Server database.
+
 ## [2.7.0]
 
 - `Added` columns information and export type using the `lino table extract` command, columns and keys organized according to the database order.
diff --git a/internal/infra/commonsql/dialect_sqlserver.go b/internal/infra/commonsql/dialect_sqlserver.go
@@ -31,7 +31,7 @@ func (sd SQLServerDialect) Placeholder(position int) string {
 
 // Limit method is adjusted to be compatible with SQL Server
 func (sd SQLServerDialect) Limit(limit uint) string {
-	return fmt.Sprintf(" TOP %d", limit)
+	return fmt.Sprintf("TOP %d", limit)
 }
 
 // From clause
diff --git a/internal/infra/commonsql/dialect_test.go b/internal/infra/commonsql/dialect_test.go
@@ -0,0 +1,52 @@
+package commonsql
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSQLServerDialect_Select(t *testing.T) {
+	dialect := SQLServerDialect{}
+
+	tableName := "MyTable"
+	schemaName := "dbo"
+	whereClause := "column1 = 1"
+	distinct := true
+	columns := []string{"column1", "column2"}
+	expectedResult := "SELECT DISTINCT column1, column2 FROM dbo.MyTable WHERE column1 = 1"
+
+	result := dialect.Select(tableName, schemaName, whereClause, distinct, columns...)
+
+	assert.Equal(t, expectedResult, result)
+}
+
+func TestSQLServerDialect_SelectLimit(t *testing.T) {
+	dialect := SQLServerDialect{}
+
+	tableName := "MyTable"
+	schemaName := "dbo"
+	whereClause := "column1 = 1"
+	distinct := false
+	limit := uint(10)
+	expectedResult := "SELECT TOP 10 * FROM dbo.MyTable WHERE column1 = 1"
+
+	result := dialect.SelectLimit(tableName, schemaName, whereClause, distinct, limit)
+
+	assert.Equal(t, expectedResult, result)
+}
+
+func TestSQLServerDialect_CreateSelect(t *testing.T) {
+	dialect := SQLServerDialect{}
+
+	sel := "SELECT"
+	where := "WHERE column1 = 1"
+	limit := "TOP 10"
+	columns := "column1, column2"
+	from := "FROM dbo.MyTable"
+	expectedResult := "SELECT TOP 10 column1, column2 FROM dbo.MyTable WHERE column1 = 1"
+
+	result := dialect.CreateSelect(sel, where, limit, columns, from)
+
+	assert.Equal(t, expectedResult, result)
+}
diff --git a/internal/infra/pull/datasource_db2_test.go b/internal/infra/pull/datasource_db2_test.go
@@ -4,6 +4,7 @@ import (
 	"testing"
 
 	"github.com/DATA-DOG/go-sqlmock"
+	"github.com/cgi-fr/lino/internal/infra/commonsql"
 	infra "github.com/cgi-fr/lino/internal/infra/pull"
 	"github.com/cgi-fr/lino/pkg/pull"
 
@@ -18,6 +19,13 @@ func TestCreateSelectDb2(t *testing.T) {
 	assert.Nil(t, err)
 	defer db.Close()
 
+	// Check SQL query is correctly created
+	ds := infra.NewSQLDataSource("pg://server/name", "", nil, db, commonsql.Db2Dialect{})
+	_, sql := ds.GetSelectSQLAndValues(aTable, aFilter)
+	expectSQL := "SELECT * FROM CUSTOMERS WHERE  1=1   FETCH FIRST 5 ROWS ONLY"
+	assert.Equal(t, expectSQL, sql)
+
+	// Check SQL query can correctly excute in DB2
 	mock.ExpectQuery("SELECT  *  FROM CUSTOMERS  WHERE  1=1   FETCH FIRST 5 ROWS ONLY").WillReturnRows()
 
 	db2Factory := infra.NewDb2DataSourceFactory()
@@ -31,3 +39,37 @@ func TestCreateSelectDb2(t *testing.T) {
 	_, err = db2DS.RowReader(aTable, aFilter)
 	assert.Nil(t, err)
 }
+
+func TestCreateSelectDb2WithColumns(t *testing.T) {
+	columns := []pull.Column{
+		{Name: "ID"},
+		{Name: "Name"},
+		{Name: "Age"},
+	}
+	aTable := pull.Table{Name: "CUSTOMERS", Columns: columns}
+	aFilter := pull.Filter{Limit: 5}
+
+	db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual))
+	assert.Nil(t, err)
+	defer db.Close()
+
+	// Check SQL query is correctly created
+	ds := infra.NewSQLDataSource("pg://server/name", "", nil, db, commonsql.Db2Dialect{})
+	_, sql := ds.GetSelectSQLAndValues(aTable, aFilter)
+	expectSQL := "SELECT  ID,  Name,  Age FROM CUSTOMERS WHERE  1=1   FETCH FIRST 5 ROWS ONLY"
+	assert.Equal(t, expectSQL, sql)
+
+	// Check SQL query can correctly excute in DB2
+	mock.ExpectQuery(sql).WillReturnRows()
+
+	msFactory := infra.NewDb2DataSourceFactory()
+
+	msDS := msFactory.New("pg://server/name", "")
+
+	err = msDS.(*infra.SQLDataSource).OpenWithDB(db)
+
+	assert.Nil(t, err)
+
+	_, err = msDS.RowReader(aTable, aFilter)
+	assert.Nil(t, err)
+}
diff --git a/internal/infra/pull/datasource_mariadb_test.go b/internal/infra/pull/datasource_mariadb_test.go
@@ -4,6 +4,7 @@ import (
 	"testing"
 
 	"github.com/DATA-DOG/go-sqlmock"
+	"github.com/cgi-fr/lino/internal/infra/commonsql"
 	infra "github.com/cgi-fr/lino/internal/infra/pull"
 	"github.com/cgi-fr/lino/pkg/pull"
 
@@ -18,7 +19,48 @@ func TestCreateSelectMariadb(t *testing.T) {
 	assert.Nil(t, err)
 	defer db.Close()
 
-	mock.ExpectQuery("SELECT  *  FROM CUSTOMERS  WHERE  1=1   LIMIT 5").WillReturnRows()
+	// Check SQL query is correctly created
+	ds := infra.NewSQLDataSource("pg://server/name", "", nil, db, commonsql.MariadbDialect{})
+	_, sql := ds.GetSelectSQLAndValues(aTable, aFilter)
+	expectSQL := "SELECT * FROM CUSTOMERS WHERE  1=1   LIMIT 5"
+	assert.Equal(t, expectSQL, sql)
+
+	// Check SQL query can correctly excute in MariaDB
+	mock.ExpectQuery(sql).WillReturnRows()
+
+	pgFactory := infra.NewMariadbDataSourceFactory()
+
+	pgDS := pgFactory.New("pg://server/name", "")
+
+	err = pgDS.(*infra.SQLDataSource).OpenWithDB(db)
+
+	assert.Nil(t, err)
+
+	_, err = pgDS.RowReader(aTable, aFilter)
+	assert.Nil(t, err)
+}
+
+func TestCreateSelectMariadbWithColumns(t *testing.T) {
+	columns := []pull.Column{
+		{Name: "ID"},
+		{Name: "Name"},
+		{Name: "Age"},
+	}
+	aTable := pull.Table{Name: "CUSTOMERS", Columns: columns}
+	aFilter := pull.Filter{Limit: 5}
+
+	db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual))
+	assert.Nil(t, err)
+	defer db.Close()
+
+	// Check SQL query is correctly created
+	ds := infra.NewSQLDataSource("pg://server/name", "", nil, db, commonsql.MariadbDialect{})
+	_, sql := ds.GetSelectSQLAndValues(aTable, aFilter)
+	expectSQL := "SELECT  ID,  Name,  Age FROM CUSTOMERS WHERE  1=1   LIMIT 5"
+	assert.Equal(t, expectSQL, sql)
+
+	// Check SQL query can correctly excute in MariaDB
+	mock.ExpectQuery(sql).WillReturnRows()
 
 	pgFactory := infra.NewMariadbDataSourceFactory()
 
diff --git a/internal/infra/pull/datasource_oracle_test.go b/internal/infra/pull/datasource_oracle_test.go
@@ -4,6 +4,7 @@ import (
 	"testing"
 
 	"github.com/DATA-DOG/go-sqlmock"
+	"github.com/cgi-fr/lino/internal/infra/commonsql"
 	infra "github.com/cgi-fr/lino/internal/infra/pull"
 	"github.com/cgi-fr/lino/pkg/pull"
 
@@ -18,7 +19,48 @@ func TestCreateSelectOracle(t *testing.T) {
 	assert.Nil(t, err)
 	defer db.Close()
 
-	mock.ExpectQuery("SELECT * FROM CUSTOMERS WHERE 1=1 AND rownum <= 5").WillReturnRows()
+	// Check SQL query is correctly created
+	ds := infra.NewSQLDataSource("pg://server/name", "", nil, db, commonsql.OracleDialect{})
+	_, sql := ds.GetSelectSQLAndValues(aTable, aFilter)
+	expectSQL := "SELECT * FROM CUSTOMERS WHERE  1=1   AND rownum <= 5"
+	assert.Equal(t, expectSQL, sql)
+
+	// Check SQL query can correctly excute in Oracle DB
+	mock.ExpectQuery(sql).WillReturnRows()
+
+	pgFactory := infra.NewOracleDataSourceFactory()
+
+	pgDS := pgFactory.New("pg://server/name", "")
+
+	err = pgDS.(*infra.SQLDataSource).OpenWithDB(db)
+
+	assert.Nil(t, err)
+
+	_, err = pgDS.RowReader(aTable, aFilter)
+	assert.Nil(t, err)
+}
+
+func TestCreateSelectOracleWithColumns(t *testing.T) {
+	columns := []pull.Column{
+		{Name: "ID"},
+		{Name: "Name"},
+		{Name: "Age"},
+	}
+	aTable := pull.Table{Name: "CUSTOMERS", Columns: columns}
+	aFilter := pull.Filter{Limit: 5}
+
+	db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual))
+	assert.Nil(t, err)
+	defer db.Close()
+
+	// Check SQL query is correctly created
+	ds := infra.NewSQLDataSource("pg://server/name", "", nil, db, commonsql.OracleDialect{})
+	_, sql := ds.GetSelectSQLAndValues(aTable, aFilter)
+	expectSQL := "SELECT  ID,  Name,  Age FROM CUSTOMERS WHERE  1=1   AND rownum <= 5"
+	assert.Equal(t, expectSQL, sql)
+
+	// Check SQL query can correctly excute in Oracle DB
+	mock.ExpectQuery(sql).WillReturnRows()
 
 	pgFactory := infra.NewOracleDataSourceFactory()
 
diff --git a/internal/infra/pull/datasource_postgres_test.go b/internal/infra/pull/datasource_postgres_test.go
@@ -4,6 +4,7 @@ import (
 	"testing"
 
 	"github.com/DATA-DOG/go-sqlmock"
+	"github.com/cgi-fr/lino/internal/infra/commonsql"
 	infra "github.com/cgi-fr/lino/internal/infra/pull"
 	"github.com/cgi-fr/lino/pkg/pull"
 
@@ -18,7 +19,14 @@ func TestCreateSelectPostgres(t *testing.T) {
 	assert.Nil(t, err)
 	defer db.Close()
 
-	mock.ExpectQuery("SELECT * FROM CUSTOMERS WHERE 1=1 LIMIT 5").WillReturnRows()
+	// Check SQL query is correctly created
+	ds := infra.NewSQLDataSource("pg://server/name", "", nil, db, commonsql.PostgresDialect{})
+	_, sql := ds.GetSelectSQLAndValues(aTable, aFilter)
+	expectSQL := "SELECT * FROM CUSTOMERS WHERE  1=1  LIMIT 5"
+	assert.Equal(t, expectSQL, sql)
+
+	// Check SQL query can correctly excute in Postgres
+	mock.ExpectQuery(sql).WillReturnRows()
 
 	pgFactory := infra.NewPostgresDataSourceFactory()
 
@@ -36,25 +44,26 @@ func TestCreateSelectPostgresWithColumns(t *testing.T) {
 	aTable := pull.Table{Name: "CUSTOMERS", Columns: []pull.Column{{Name: "Name"}}}
 	aFilter := pull.Filter{Limit: 5}
 
-	// Créer une base de données simulée avec un mock
 	db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual))
 	assert.Nil(t, err)
 	defer db.Close()
 
-	// Définir l'expectation de la requête SELECT avec des colonnes spécifiques
-	mock.ExpectQuery("SELECT Name FROM CUSTOMERS WHERE 1=1 LIMIT 5").WillReturnRows()
+	// Check SQL query is correctly created
+	ds := infra.NewSQLDataSource("pg://server/name", "", nil, db, commonsql.PostgresDialect{})
+	_, sql := ds.GetSelectSQLAndValues(aTable, aFilter)
+	expectSQL := "SELECT  Name FROM CUSTOMERS WHERE  1=1  LIMIT 5"
+	assert.Equal(t, expectSQL, sql)
+
+	// Check SQL query can correctly excute in Postgres
+	mock.ExpectQuery(sql).WillReturnRows()
 
-	// Créer une instance de la fabrique de source de données PostgreSQL
 	pgFactory := infra.NewPostgresDataSourceFactory()
 
-	// Créer une source de données PostgreSQL avec une URL factice
 	pgDS := pgFactory.New("pg://server/name", "")
 
-	// Ouvrir la source de données avec la base de données simulée
 	err = pgDS.(*infra.SQLDataSource).OpenWithDB(db)
 	assert.Nil(t, err)
 
-	// Appeler RowReader avec la table et le filtre
 	_, err = pgDS.RowReader(aTable, aFilter)
 	assert.Nil(t, err)
 }
diff --git a/internal/infra/pull/datasource_sqlserver_test.go b/internal/infra/pull/datasource_sqlserver_test.go
@@ -4,6 +4,7 @@ import (
 	"testing"
 
 	"github.com/DATA-DOG/go-sqlmock"
+	"github.com/cgi-fr/lino/internal/infra/commonsql"
 	infra "github.com/cgi-fr/lino/internal/infra/pull"
 	"github.com/cgi-fr/lino/pkg/pull"
 
@@ -18,7 +19,48 @@ func TestCreateSelectSQLServer(t *testing.T) {
 	assert.Nil(t, err)
 	defer db.Close()
 
-	mock.ExpectQuery("SELECT  TOP 5 *  FROM CUSTOMERS  WHERE  1=1").WillReturnRows()
+	// Check SQL query is correctly created
+	ds := infra.NewSQLDataSource("pg://server/name", "", nil, db, commonsql.SQLServerDialect{})
+	_, sql := ds.GetSelectSQLAndValues(aTable, aFilter)
+	expectSQL := "SELECT TOP 5 * FROM CUSTOMERS WHERE  1=1 "
+	assert.Equal(t, expectSQL, sql)
+
+	// Check SQL query can correctly excute in SQL Server
+	mock.ExpectQuery(sql).WillReturnRows()
+
+	msFactory := infra.NewSQLServerDataSourceFactory()
+
+	msDS := msFactory.New("pg://server/name", "")
+
+	err = msDS.(*infra.SQLDataSource).OpenWithDB(db)
+
+	assert.Nil(t, err)
+
+	_, err = msDS.RowReader(aTable, aFilter)
+	assert.Nil(t, err)
+}
+
+func TestCreateSelectSQLServerWithColumns(t *testing.T) {
+	columns := []pull.Column{
+		{Name: "ID"},
+		{Name: "Name"},
+		{Name: "Age"},
+	}
+	aTable := pull.Table{Name: "CUSTOMERS", Columns: columns}
+	aFilter := pull.Filter{Limit: 5}
+
+	db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual))
+	assert.Nil(t, err)
+	defer db.Close()
+
+	// Check SQL query is correctly created
+	ds := infra.NewSQLDataSource("pg://server/name", "", nil, db, commonsql.SQLServerDialect{})
+	_, sql := ds.GetSelectSQLAndValues(aTable, aFilter)
+	expectSQL := "SELECT TOP 5  ID,  Name,  Age FROM CUSTOMERS WHERE  1=1 "
+	assert.Equal(t, expectSQL, sql)
+
+	// Check SQL query can correctly excute in SQL Server
+	mock.ExpectQuery(sql).WillReturnRows()
 
 	msFactory := infra.NewSQLServerDataSourceFactory()
 
diff --git a/internal/infra/pull/sql_datasource.go b/internal/infra/pull/sql_datasource.go
diff --git a/internal/infra/push/datadestination_oracle.go b/internal/infra/push/datadestination_oracle.go
diff --git a/internal/infra/push/translator_file_test.go b/internal/infra/push/translator_file_test.go

Original file line number	Diff line number	Diff line change
`@@ -31,7 +31,7 @@ func (sd SQLServerDialect) Placeholder(position int) string {`
`31`	`31`
`32`	`32`	`// Limit method is adjusted to be compatible with SQL Server`
`33`	`33`	`func (sd SQLServerDialect) Limit(limit uint) string {`
`34`		`- return fmt.Sprintf(" TOP %d", limit)`
	`34`	`+ return fmt.Sprintf("TOP %d", limit)`
`35`	`35`	`}`
`36`	`36`
`37`	`37`	`// From clause`