Skip to content

Commit e12898a

Browse files
yad-unextyad-unext
committed
add a simple spanner sql parser for migration
1 parent 2788339 commit e12898a

File tree

8 files changed

+314
-0
lines changed

8 files changed

+314
-0
lines changed

‎.golangci.yml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,3 +28,6 @@ issues:
2828
exclude:
2929
# gosec: Duplicated errcheck checks
3030
- G104
31+
- bad syntax for struct tag pair
32+
- struct literal uses unkeyed fields
33+
- bad syntax for struct tag key

‎database/spanner/ddl/.got/snapshots/TestBasic/ddl-string.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
"A B;\nA;\nA (B C);\nA \"B\";\nA <B , C>;\nA B , A (B (C D)) E;\n"

‎database/spanner/ddl/.got/snapshots/TestBasic/ddl.json

Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,164 @@
1+
{
2+
"Statements": [
3+
{
4+
"Expression": {
5+
"Values": [
6+
{
7+
"Component": [
8+
"A",
9+
"B"
10+
],
11+
"Braced": null,
12+
"Bracketed": null
13+
}
14+
]
15+
},
16+
"End": ";"
17+
},
18+
{
19+
"Expression": {
20+
"Values": [
21+
{
22+
"Component": [
23+
"A"
24+
],
25+
"Braced": null,
26+
"Bracketed": null
27+
}
28+
]
29+
},
30+
"End": ";"
31+
},
32+
{
33+
"Expression": {
34+
"Values": [
35+
{
36+
"Component": [
37+
"A"
38+
],
39+
"Braced": null,
40+
"Bracketed": null
41+
},
42+
{
43+
"Component": null,
44+
"Braced": {
45+
"Values": [
46+
{
47+
"Component": [
48+
"B",
49+
"C"
50+
],
51+
"Braced": null,
52+
"Bracketed": null
53+
}
54+
]
55+
},
56+
"Bracketed": null
57+
}
58+
]
59+
},
60+
"End": ";"
61+
},
62+
{
63+
"Expression": {
64+
"Values": [
65+
{
66+
"Component": [
67+
"A",
68+
"\"B\""
69+
],
70+
"Braced": null,
71+
"Bracketed": null
72+
}
73+
]
74+
},
75+
"End": ";"
76+
},
77+
{
78+
"Expression": {
79+
"Values": [
80+
{
81+
"Component": [
82+
"A"
83+
],
84+
"Braced": null,
85+
"Bracketed": null
86+
},
87+
{
88+
"Component": null,
89+
"Braced": null,
90+
"Bracketed": {
91+
"Values": [
92+
{
93+
"Component": [
94+
"B",
95+
",",
96+
"C"
97+
],
98+
"Braced": null,
99+
"Bracketed": null
100+
}
101+
]
102+
}
103+
}
104+
]
105+
},
106+
"End": ";"
107+
},
108+
{
109+
"Expression": {
110+
"Values": [
111+
{
112+
"Component": [
113+
"A",
114+
"B",
115+
",",
116+
"A"
117+
],
118+
"Braced": null,
119+
"Bracketed": null
120+
},
121+
{
122+
"Component": null,
123+
"Braced": {
124+
"Values": [
125+
{
126+
"Component": [
127+
"B"
128+
],
129+
"Braced": null,
130+
"Bracketed": null
131+
},
132+
{
133+
"Component": null,
134+
"Braced": {
135+
"Values": [
136+
{
137+
"Component": [
138+
"C",
139+
"D"
140+
],
141+
"Braced": null,
142+
"Bracketed": null
143+
}
144+
]
145+
},
146+
"Bracketed": null
147+
}
148+
]
149+
},
150+
"Bracketed": null
151+
},
152+
{
153+
"Component": [
154+
"E"
155+
],
156+
"Braced": null,
157+
"Bracketed": null
158+
}
159+
]
160+
},
161+
"End": ";"
162+
}
163+
]
164+
}

‎database/spanner/ddl/ddl_test.go

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
package ddl
2+
3+
import (
4+
"testing"
5+
6+
"github.com/ysmood/got"
7+
)
8+
9+
func TestBasic(t *testing.T) {
10+
g := got.T(t)
11+
12+
ddl, err := parser.ParseString("", `
13+
-- Comment
14+
A B;
15+
A;
16+
A (B C); /*
17+
Multi-line
18+
comment
19+
*/
20+
A "B";
21+
A <B, C>;
22+
A -- Comment
23+
B,
24+
-- Comment
25+
A (B (C D)) E;
26+
`)
27+
g.E(err)
28+
29+
g.Snapshot("ddl", ddl)
30+
31+
g.Snapshot("ddl-string", ddl.String())
32+
}

‎database/spanner/ddl/format.go

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
package ddl
2+
3+
import (
4+
"fmt"
5+
"strings"
6+
)
7+
8+
func ToMigrationStatements(path, ddl string) ([]string, error) {
9+
_, err := parser.ParseString(path, ddl)
10+
if err != nil {
11+
return nil, fmt.Errorf("failed to parse DDL: %w", err)
12+
}
13+
14+
return nil, nil
15+
}
16+
17+
func (d *SpannerDDL) String() string {
18+
out := ""
19+
20+
for _, s := range d.Statements {
21+
out += s.String() + "\n"
22+
}
23+
24+
return out
25+
}
26+
27+
func (s *Statement) String() string {
28+
return s.Expression.String() + s.End
29+
}
30+
31+
func (e *Expression) String() string {
32+
list := []string{}
33+
34+
for _, v := range e.Values {
35+
list = append(list, v.String())
36+
}
37+
38+
return strings.Join(list, " ")
39+
}
40+
41+
func (v *Value) String() string {
42+
out := ""
43+
44+
if v.Braced != nil {
45+
out += "(" + v.Braced.String() + ")"
46+
} else if v.Bracketed != nil {
47+
out += "<" + v.Bracketed.String() + ">"
48+
} else {
49+
out += strings.Join(v.Component, " ")
50+
}
51+
52+
return out
53+
}

‎database/spanner/ddl/parser.go

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
package ddl
2+
3+
import (
4+
"github.com/alecthomas/participle/v2"
5+
"github.com/alecthomas/participle/v2/lexer"
6+
)
7+
8+
// SpannerDDL only parses the components that are relevant for the migration tool.
9+
// Main objectives:
10+
// - Remove comments: spanner does allow comments in DDL.
11+
// - Split DDL into statements: spanner requires the migration request to be a list of statements, not a single DDL file.
12+
// Also each request should not exceed 10 statements.
13+
type SpannerDDL struct {
14+
Statements []*Statement `@@*`
15+
}
16+
17+
type Statement struct {
18+
Expression *Expression `@@`
19+
End string `@";"`
20+
}
21+
22+
type Expression struct {
23+
Values []*Value `@@+`
24+
}
25+
26+
type Value struct {
27+
Component []string ` @(Ident | String | Comma)+`
28+
Braced *Expression `| "(" @@ ")"`
29+
Bracketed *Expression `| "<" @@ ">"`
30+
}
31+
32+
var parser = participle.MustBuild[SpannerDDL](
33+
participle.Lexer(lexer.MustSimple([]lexer.SimpleRule{
34+
{"SingleLineComment", `--[^\n]*`},
35+
{"MultiLineComment", `(?s)/\*.*?\*/`},
36+
{"Whitespace", `\s+`},
37+
38+
{"StatementEnd", `;`},
39+
{"Group", `[()<>]`},
40+
41+
{"Ident", `[a-zA-Z\d_]+`},
42+
{"String", `"(?:\\.|[^"])*"`},
43+
{"Comma", `,`},
44+
})),
45+
participle.Elide("SingleLineComment", "MultiLineComment", "Whitespace"),
46+
)

‎go.mod

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ require (
77
cloud.google.com/go/storage v1.38.0
88
github.com/Azure/go-autorest/autorest/adal v0.9.16
99
github.com/ClickHouse/clickhouse-go v1.4.3
10+
github.com/alecthomas/participle/v2 v2.1.4
1011
github.com/aws/aws-sdk-go v1.49.6
1112
github.com/cenkalti/backoff/v4 v4.1.2
1213
github.com/cockroachdb/cockroach-go/v2 v2.1.1
@@ -33,6 +34,7 @@ require (
3334
github.com/snowflakedb/gosnowflake v1.6.19
3435
github.com/stretchr/testify v1.9.0
3536
github.com/xanzy/go-gitlab v0.15.0
37+
github.com/ysmood/got v0.41.0
3638
go.mongodb.org/mongo-driver v1.7.5
3739
go.uber.org/atomic v1.7.0
3840
golang.org/x/oauth2 v0.18.0
@@ -56,6 +58,7 @@ require (
5658
github.com/modern-go/reflect2 v1.0.2 // indirect
5759
github.com/rogpeppe/go-internal v1.12.0 // indirect
5860
github.com/stretchr/objx v0.5.2 // indirect
61+
github.com/ysmood/gop v0.2.0 // indirect
5962
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect
6063
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect
6164
go.opentelemetry.io/otel v1.29.0 // indirect

‎go.sum

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,12 @@ github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030I
6969
github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
7070
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
7171
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
72+
github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0=
73+
github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k=
74+
github.com/alecthomas/participle/v2 v2.1.4 h1:W/H79S8Sat/krZ3el6sQMvMaahJ+XcM9WSI2naI7w2U=
75+
github.com/alecthomas/participle/v2 v2.1.4/go.mod h1:8tqVbpTX20Ru4NfYQgZf4mP18eXPTBViyMWiArNEgGI=
76+
github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc=
77+
github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
7278
github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
7379
github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
7480
github.com/apache/arrow/go/v10 v10.0.1 h1:n9dERvixoC/1JjDmBcs9FPaEryoANa2sCgVFo6ez9cI=
@@ -321,6 +327,8 @@ github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9
321327
github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
322328
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
323329
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
330+
github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
331+
github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
324332
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
325333
github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo=
326334
github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk=
@@ -581,6 +589,10 @@ github.com/xdg-go/stringprep v1.0.3 h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCO
581589
github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
582590
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
583591
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
592+
github.com/ysmood/gop v0.2.0 h1:+tFrG0TWPxT6p9ZaZs+VY+opCvHU8/3Fk6BaNv6kqKg=
593+
github.com/ysmood/gop v0.2.0/go.mod h1:rr5z2z27oGEbyB787hpEcx4ab8cCiPnKxn0SUHt6xzk=
594+
github.com/ysmood/got v0.41.0 h1:XiFH311ltTSGyxjeKcNvy7dzbJjjTzn6DBgK313JHBs=
595+
github.com/ysmood/got v0.41.0/go.mod h1:W7DdpuX6skL3NszLmAsC5hT7JAhuLZhByVzHTq874Qg=
584596
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
585597
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
586598
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=

0 commit comments

Comments
 (0)