Skip to content

Commit 57cffdb

Browse files
adrienauryadrienaury
authored
feat: export presence (#328)
* feat(presence): add venom test * feat(presence): implement flag in row export * feat(presence): fix venom test * feat(presence): wip! do not extract value * feat(presence): wip! do not extract value * feat(presence): do not extract value * feat(export): fix presence for oracle
1 parent 23f2239 commit 57cffdb

File tree

12 files changed

+162
-57
lines changed

12 files changed

+162
-57
lines changed

‎CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ Types of changes
1919
- `Added` possibility to filter columns via `select` property in ingress descriptors
2020
- `Added` commands `set-parent-select` and `set-child-select` to `lino id`
2121
- `Added` `import: no` option for columns in `tables.yaml` configuration
22+
- `Added` `export: presence` option for columns in `tables.yaml` configuration
2223

2324
## [3.0.2]
2425

‎internal/infra/analyse/sql_datasource.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,7 @@ func (ds *SQLDataSource) Open() error {
134134
// Get WHERE Clause query
135135
sqlWhere, values := commonsql.GetWhereSQLAndValues(map[string]any{}, ds.where, ds.dialect)
136136

137-
sql := ds.dialect.Select(ds.table, ds.schema, sqlWhere, false, ds.column)
137+
sql := ds.dialect.Select(ds.table, ds.schema, sqlWhere, false, commonsql.ColumnExportDefinition{Name: ds.column})
138138

139139
// If log level is more than debug level, this function will log all SQL Query
140140
commonsql.LogSQLQuery(sql, values, ds.dialect)

‎internal/infra/commonsql/dialect.go

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,19 @@ import (
2525
"github.com/rs/zerolog/log"
2626
)
2727

28+
type ColumnExportDefinition struct {
29+
Name string
30+
OnlyPresence bool
31+
}
32+
33+
func Names(columns []ColumnExportDefinition) []string {
34+
us := make([]string, len(columns))
35+
for i := range columns {
36+
us[i] = columns[i].Name
37+
}
38+
return us
39+
}
40+
2841
type Dialect interface {
2942
// Placeholder format variable in query
3043
Placeholder(int) string
@@ -35,9 +48,9 @@ type Dialect interface {
3548
// Where clause
3649
Where(string) string
3750
// Select clause
38-
Select(tableName string, schemaName string, where string, distinct bool, columns ...string) string
51+
Select(tableName string, schemaName string, where string, distinct bool, columns ...ColumnExportDefinition) string
3952
// SelectLimit clause
40-
SelectLimit(tableName string, schemaName string, where string, distinct bool, limit uint, columns ...string) string
53+
SelectLimit(tableName string, schemaName string, where string, distinct bool, limit uint, columns ...ColumnExportDefinition) string
4154
// Quote identifier
4255
Quote(id string) string
4356

‎internal/infra/commonsql/dialect_db2.go

Lines changed: 22 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ func (db2 Db2Dialect) Where(where string) string {
5353
}
5454

5555
// Select clause
56-
func (db2 Db2Dialect) Select(tableName string, schemaName string, where string, distinct bool, columns ...string) string {
56+
func (db2 Db2Dialect) Select(tableName string, schemaName string, where string, distinct bool, columns ...ColumnExportDefinition) string {
5757
var query strings.Builder
5858

5959
query.WriteString("SELECT ")
@@ -62,11 +62,15 @@ func (db2 Db2Dialect) Select(tableName string, schemaName string, where string,
6262
query.WriteString("DISTINCT ")
6363
}
6464

65-
if len(columns) > 0 {
66-
for i := range columns {
67-
columns[i] = db2.Quote(columns[i])
65+
if names := Names(columns); len(names) > 0 {
66+
for i := range names {
67+
if columns[i].OnlyPresence {
68+
names[i] = db2.selectPresence(names[i])
69+
} else {
70+
names[i] = db2.Quote(names[i])
71+
}
6872
}
69-
query.WriteString(strings.Join(columns, ", "))
73+
query.WriteString(strings.Join(names, ", "))
7074
} else {
7175
query.WriteRune('*')
7276
}
@@ -80,7 +84,7 @@ func (db2 Db2Dialect) Select(tableName string, schemaName string, where string,
8084
}
8185

8286
// SelectLimit clause
83-
func (db2 Db2Dialect) SelectLimit(tableName string, schemaName string, where string, distinct bool, limit uint, columns ...string) string {
87+
func (db2 Db2Dialect) SelectLimit(tableName string, schemaName string, where string, distinct bool, limit uint, columns ...ColumnExportDefinition) string {
8488
var query strings.Builder
8589

8690
query.WriteString("SELECT ")
@@ -89,11 +93,15 @@ func (db2 Db2Dialect) SelectLimit(tableName string, schemaName string, where str
8993
query.WriteString("DISTINCT ")
9094
}
9195

92-
if len(columns) > 0 {
93-
for i := range columns {
94-
columns[i] = db2.Quote(columns[i])
96+
if names := Names(columns); len(names) > 0 {
97+
for i := range names {
98+
if columns[i].OnlyPresence {
99+
names[i] = db2.selectPresence(names[i])
100+
} else {
101+
names[i] = db2.Quote(names[i])
102+
}
95103
}
96-
query.WriteString(strings.Join(columns, ", "))
104+
query.WriteString(strings.Join(names, ", "))
97105
} else {
98106
query.WriteRune('*')
99107
}
@@ -123,3 +131,7 @@ func (db2 Db2Dialect) Quote(id string) string {
123131
func (db2 Db2Dialect) CreateSelect(sel string, where string, limit string, columns string, from string) string {
124132
return fmt.Sprintf("%s %s %s %s %s", sel, columns, from, where, limit)
125133
}
134+
135+
func (db2 Db2Dialect) selectPresence(column string) string {
136+
return fmt.Sprintf("CASE WHEN %s IS NOT NULL THEN 1 ELSE NULL END AS %s", db2.Quote(column), db2.Quote(column))
137+
}

‎internal/infra/commonsql/dialect_mariadb.go

Lines changed: 22 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ func (pd MariadbDialect) Where(where string) string {
5353
}
5454

5555
// Select clause
56-
func (pd MariadbDialect) Select(tableName string, schemaName string, where string, distinct bool, columns ...string) string {
56+
func (pd MariadbDialect) Select(tableName string, schemaName string, where string, distinct bool, columns ...ColumnExportDefinition) string {
5757
var query strings.Builder
5858

5959
query.WriteString("SELECT ")
@@ -62,11 +62,15 @@ func (pd MariadbDialect) Select(tableName string, schemaName string, where strin
6262
query.WriteString("DISTINCT ")
6363
}
6464

65-
if len(columns) > 0 {
66-
for i := range columns {
67-
columns[i] = pd.Quote(columns[i])
65+
if names := Names(columns); len(names) > 0 {
66+
for i := range names {
67+
if columns[i].OnlyPresence {
68+
names[i] = pd.selectPresence(names[i])
69+
} else {
70+
names[i] = pd.Quote(names[i])
71+
}
6872
}
69-
query.WriteString(strings.Join(columns, ", "))
73+
query.WriteString(strings.Join(names, ", "))
7074
} else {
7175
query.WriteRune('*')
7276
}
@@ -80,7 +84,7 @@ func (pd MariadbDialect) Select(tableName string, schemaName string, where strin
8084
}
8185

8286
// SelectLimit clause
83-
func (pd MariadbDialect) SelectLimit(tableName string, schemaName string, where string, distinct bool, limit uint, columns ...string) string {
87+
func (pd MariadbDialect) SelectLimit(tableName string, schemaName string, where string, distinct bool, limit uint, columns ...ColumnExportDefinition) string {
8488
var query strings.Builder
8589

8690
query.WriteString("SELECT ")
@@ -89,11 +93,15 @@ func (pd MariadbDialect) SelectLimit(tableName string, schemaName string, where
8993
query.WriteString("DISTINCT ")
9094
}
9195

92-
if len(columns) > 0 {
93-
for i := range columns {
94-
columns[i] = pd.Quote(columns[i])
96+
if names := Names(columns); len(names) > 0 {
97+
for i := range names {
98+
if columns[i].OnlyPresence {
99+
names[i] = pd.selectPresence(names[i])
100+
} else {
101+
names[i] = pd.Quote(names[i])
102+
}
95103
}
96-
query.WriteString(strings.Join(columns, ", "))
104+
query.WriteString(strings.Join(names, ", "))
97105
} else {
98106
query.WriteRune('*')
99107
}
@@ -123,3 +131,7 @@ func (sd MariadbDialect) Quote(id string) string {
123131
func (sd MariadbDialect) CreateSelect(sel string, where string, limit string, columns string, from string) string {
124132
return fmt.Sprintf("%s %s %s %s %s", sel, columns, from, where, limit)
125133
}
134+
135+
func (sd MariadbDialect) selectPresence(column string) string {
136+
return fmt.Sprintf("CASE WHEN %s IS NOT NULL THEN 'TRUE' ELSE NULL END AS %s", sd.Quote(column), sd.Quote(column))
137+
}

‎internal/infra/commonsql/dialect_oracle.go

Lines changed: 22 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ func (od OracleDialect) Where(where string) string {
5353
}
5454

5555
// Select clause
56-
func (od OracleDialect) Select(tableName string, schemaName string, where string, distinct bool, columns ...string) string {
56+
func (od OracleDialect) Select(tableName string, schemaName string, where string, distinct bool, columns ...ColumnExportDefinition) string {
5757
var query strings.Builder
5858

5959
query.WriteString("SELECT ")
@@ -62,11 +62,15 @@ func (od OracleDialect) Select(tableName string, schemaName string, where string
6262
query.WriteString("DISTINCT ")
6363
}
6464

65-
if len(columns) > 0 {
66-
for i := range columns {
67-
columns[i] = od.Quote(columns[i])
65+
if names := Names(columns); len(names) > 0 {
66+
for i := range names {
67+
if columns[i].OnlyPresence {
68+
names[i] = od.selectPresence(names[i])
69+
} else {
70+
names[i] = od.Quote(names[i])
71+
}
6872
}
69-
query.WriteString(strings.Join(columns, ", "))
73+
query.WriteString(strings.Join(names, ", "))
7074
} else {
7175
query.WriteRune('*')
7276
}
@@ -80,7 +84,7 @@ func (od OracleDialect) Select(tableName string, schemaName string, where string
8084
}
8185

8286
// SelectLimit clause
83-
func (od OracleDialect) SelectLimit(tableName string, schemaName string, where string, distinct bool, limit uint, columns ...string) string {
87+
func (od OracleDialect) SelectLimit(tableName string, schemaName string, where string, distinct bool, limit uint, columns ...ColumnExportDefinition) string {
8488
var query strings.Builder
8589

8690
query.WriteString("SELECT ")
@@ -89,11 +93,15 @@ func (od OracleDialect) SelectLimit(tableName string, schemaName string, where s
8993
query.WriteString("DISTINCT ")
9094
}
9195

92-
if len(columns) > 0 {
93-
for i := range columns {
94-
columns[i] = od.Quote(columns[i])
96+
if names := Names(columns); len(names) > 0 {
97+
for i := range names {
98+
if columns[i].OnlyPresence {
99+
names[i] = od.selectPresence(names[i])
100+
} else {
101+
names[i] = od.Quote(names[i])
102+
}
95103
}
96-
query.WriteString(strings.Join(columns, ", "))
104+
query.WriteString(strings.Join(names, ", "))
97105
} else {
98106
query.WriteRune('*')
99107
}
@@ -123,3 +131,7 @@ func (od OracleDialect) Quote(id string) string {
123131
func (od OracleDialect) CreateSelect(sel string, where string, limit string, columns string, from string) string {
124132
return fmt.Sprintf("%s %s %s %s %s", sel, columns, from, where, limit)
125133
}
134+
135+
func (od OracleDialect) selectPresence(column string) string {
136+
return fmt.Sprintf("CASE WHEN %s IS NOT NULL THEN 'TRUE' ELSE NULL END AS %s", od.Quote(column), od.Quote(column))
137+
}

‎internal/infra/commonsql/dialect_postgres.go

Lines changed: 21 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ func (pgd PostgresDialect) Where(where string) string {
5353
}
5454

5555
// Select clause
56-
func (pgd PostgresDialect) Select(tableName string, schemaName string, where string, distinct bool, columns ...string) string {
56+
func (pgd PostgresDialect) Select(tableName string, schemaName string, where string, distinct bool, columns ...ColumnExportDefinition) string {
5757
var query strings.Builder
5858

5959
query.WriteString("SELECT ")
@@ -62,11 +62,15 @@ func (pgd PostgresDialect) Select(tableName string, schemaName string, where str
6262
query.WriteString("DISTINCT ")
6363
}
6464

65-
if len(columns) > 0 {
65+
if names := Names(columns); len(names) > 0 {
6666
for i := range columns {
67-
columns[i] = pgd.Quote(columns[i])
67+
if columns[i].OnlyPresence {
68+
names[i] = pgd.selectPresence(names[i])
69+
} else {
70+
names[i] = pgd.Quote(names[i])
71+
}
6872
}
69-
query.WriteString(strings.Join(columns, ", "))
73+
query.WriteString(strings.Join(names, ", "))
7074
} else {
7175
query.WriteRune('*')
7276
}
@@ -80,7 +84,7 @@ func (pgd PostgresDialect) Select(tableName string, schemaName string, where str
8084
}
8185

8286
// SelectLimit clause
83-
func (pgd PostgresDialect) SelectLimit(tableName string, schemaName string, where string, distinct bool, limit uint, columns ...string) string {
87+
func (pgd PostgresDialect) SelectLimit(tableName string, schemaName string, where string, distinct bool, limit uint, columns ...ColumnExportDefinition) string {
8488
var query strings.Builder
8589

8690
query.WriteString("SELECT ")
@@ -89,11 +93,15 @@ func (pgd PostgresDialect) SelectLimit(tableName string, schemaName string, wher
8993
query.WriteString("DISTINCT ")
9094
}
9195

92-
if len(columns) > 0 {
93-
for i := range columns {
94-
columns[i] = pgd.Quote(columns[i])
96+
if names := Names(columns); len(names) > 0 {
97+
for i := range names {
98+
if columns[i].OnlyPresence {
99+
names[i] = pgd.selectPresence(names[i])
100+
} else {
101+
names[i] = pgd.Quote(names[i])
102+
}
95103
}
96-
query.WriteString(strings.Join(columns, ", "))
104+
query.WriteString(strings.Join(names, ", "))
97105
} else {
98106
query.WriteRune('*')
99107
}
@@ -123,3 +131,7 @@ func (pgd PostgresDialect) Quote(id string) string {
123131
func (pgd PostgresDialect) CreateSelect(sel string, where string, limit string, columns string, from string) string {
124132
return fmt.Sprintf("%s %s %s %s %s", sel, columns, from, where, limit)
125133
}
134+
135+
func (pgd PostgresDialect) selectPresence(column string) string {
136+
return fmt.Sprintf("CASE WHEN (%s IS NOT NULL) THEN TRUE ELSE NULL END AS %s", pgd.Quote(column), pgd.Quote(column))
137+
}

‎internal/infra/commonsql/dialect_sqlserver.go

Lines changed: 20 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ func (sd SQLServerDialect) Where(where string) string {
5454
}
5555

5656
// Select clause
57-
func (sd SQLServerDialect) Select(tableName string, schemaName string, where string, distinct bool, columns ...string) string {
57+
func (sd SQLServerDialect) Select(tableName string, schemaName string, where string, distinct bool, columns ...ColumnExportDefinition) string {
5858
var query strings.Builder
5959

6060
query.WriteString("SELECT ")
@@ -63,11 +63,15 @@ func (sd SQLServerDialect) Select(tableName string, schemaName string, where str
6363
query.WriteString("DISTINCT ")
6464
}
6565

66-
if len(columns) > 0 {
66+
if names := Names(columns); len(names) > 0 {
6767
for i := range columns {
68-
columns[i] = sd.Quote(columns[i])
68+
if columns[i].OnlyPresence {
69+
names[i] = sd.selectPresence(names[i])
70+
} else {
71+
names[i] = sd.Quote(names[i])
72+
}
6973
}
70-
query.WriteString(strings.Join(columns, ", "))
74+
query.WriteString(strings.Join(names, ", "))
7175
} else {
7276
query.WriteRune('*')
7377
}
@@ -81,7 +85,7 @@ func (sd SQLServerDialect) Select(tableName string, schemaName string, where str
8185
}
8286

8387
// SelectLimit clause
84-
func (sd SQLServerDialect) SelectLimit(tableName string, schemaName string, where string, distinct bool, limit uint, columns ...string) string {
88+
func (sd SQLServerDialect) SelectLimit(tableName string, schemaName string, where string, distinct bool, limit uint, columns ...ColumnExportDefinition) string {
8589
var query strings.Builder
8690

8791
query.WriteString("SELECT ")
@@ -93,11 +97,15 @@ func (sd SQLServerDialect) SelectLimit(tableName string, schemaName string, wher
9397
query.WriteString(sd.Limit(limit))
9498
query.WriteRune(' ')
9599

96-
if len(columns) > 0 {
100+
if names := Names(columns); len(names) > 0 {
97101
for i := range columns {
98-
columns[i] = sd.Quote(columns[i])
102+
if columns[i].OnlyPresence {
103+
names[i] = sd.selectPresence(names[i])
104+
} else {
105+
names[i] = sd.Quote(names[i])
106+
}
99107
}
100-
query.WriteString(strings.Join(columns, ", "))
108+
query.WriteString(strings.Join(names, ", "))
101109
} else {
102110
query.WriteRune('*')
103111
}
@@ -125,3 +133,7 @@ func (sd SQLServerDialect) Quote(id string) string {
125133
func (sd SQLServerDialect) CreateSelect(sel string, where string, limit string, columns string, from string) string {
126134
return fmt.Sprintf("%s %s %s %s %s", sel, limit, columns, from, where)
127135
}
136+
137+
func (sd SQLServerDialect) selectPresence(column string) string {
138+
return fmt.Sprintf("CASE WHEN %s IS NOT NULL THEN 1 ELSE NULL END AS %s", sd.Quote(column), sd.Quote(column))
139+
}

0 commit comments

Comments
 (0)