Skip to content

sstable: add code generator for sstable properties #5099

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

sstable: add code generator for sstable properties #5099

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 7 additions & 3 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,9 @@ require (
github.com/stretchr/testify v1.9.0
golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df
golang.org/x/perf v0.0.0-20230113213139-801c7ef9e5c5
golang.org/x/sync v0.7.0
golang.org/x/sys v0.18.0
golang.org/x/sync v0.16.0
golang.org/x/sys v0.34.0
golang.org/x/tools v0.35.0
)

require (
Expand All @@ -48,9 +49,12 @@ require (
github.com/prometheus/procfs v0.10.1 // indirect
github.com/rogpeppe/go-internal v1.9.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
golang.org/x/mod v0.26.0 // indirect
golang.org/x/text v0.14.0 // indirect
google.golang.org/protobuf v1.33.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

go 1.23
go 1.23.0

toolchain go1.23.6
16 changes: 10 additions & 6 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,8 +111,8 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/safehtml v0.0.2/go.mod h1:L4KWwDsUJdECRAEpZoBn3O64bQaywRscowZjJAzjHnU=
github.com/googleapis/gax-go v0.0.0-20161107002406-da06d194a00e/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
Expand Down Expand Up @@ -253,6 +253,8 @@ golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCc
golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.26.0 h1:EGMPT//Ezu+ylkCijjPc+f4Aih7sZvaAr+O3EHBxvZg=
golang.org/x/mod v0.26.0/go.mod h1:/j6NAhSk8iQ723BGAUyoAcn7SlD7s15Dp9Nd/SfeaFQ=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
Expand All @@ -273,8 +275,8 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw=
golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
Expand All @@ -284,8 +286,8 @@ golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210304124612-50617c2ba197/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA=
golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
Expand All @@ -304,6 +306,8 @@ golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtn
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.35.0 h1:mBffYraMEf7aa0sB+NuKnuCy8qI/9Bughn8dC2Gu5r0=
golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
Expand Down
273 changes: 273 additions & 0 deletions sstable/internal/genprops/gen_props.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,273 @@
// Copyright 2025 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.

package main

import (
"bytes"
"go/ast"
"go/format"
"go/token"
"log"
"os"
"path/filepath"
"reflect"
"slices"
"strings"
"text/template"

"golang.org/x/tools/go/packages"
)

// Field represents a struct field tagged with `prop:"..."`.
type Field struct {
Name string // Go identifier
Tag string // tag value
Kind string // bool | uint32 | uint64 | string
EncodeEmpty bool // whether to encode empty a zero value
Intern bool // whether to intern the string value
}

// Template for sstable/properties_gen.go.
const tmpl = `// Code generated by genprops; DO NOT EDIT.
package sstable

import (
"bytes"
"encoding/binary"
"fmt"
"iter"
"maps"
"slices"
"strings"

"github.com/cockroachdb/pebble/internal/intern"
)

// load populates *Properties from an iterator and records which fields were
// present using the bit‑vector.
func (p *Properties) load(i iter.Seq2[[]byte, []byte]) error {
p.Loaded = 0
for k, v := range i {
switch string(k) {
{{- range .Fields }}
case "{{ .Tag }}":
p.Loaded |= 1 << _bit_{{ .Name }}
{{- if eq .Kind "bool" }}
p.{{ .Name }} = len(v) == 1 && v[0] == '1'
{{- else if eq .Kind "uint32" }}
p.{{ .Name }} = binary.LittleEndian.Uint32(v)
{{- else if eq .Kind "uint64" }}
n, _ := binary.Uvarint(v)
p.{{ .Name }} = n
{{- else if eq .Kind "string" }}
{{- if .Intern }}
p.{{ .Name }} = intern.Bytes(v)
{{- else }}
p.{{ .Name }} = string(v)
{{- end }}
{{- end }}
{{- end }}
default:
if _, denied := ignoredInternalProperties[string(k)]; !denied {
if p.UserProperties == nil {
p.UserProperties = make(map[string]string)
}
p.UserProperties[intern.Bytes(k)] = string(v)
}
}
}
return nil
}

// encodeAll returns a map of property keys and encoded values.
func (p *Properties) encodeAll() map[string][]byte {
m := make(map[string][]byte, _numPropBits+len(p.UserProperties))
var allocBuf []byte
alloc := func(n int) []byte {
if len(allocBuf) < n {
allocBuf = make([]byte, n + 512)
}
res := allocBuf[:n]
allocBuf = allocBuf[n:]
return res
}

{{- range .Fields }}
{{- if .EncodeEmpty }}
if true {
{{- else }}
if p.{{ .Name }} != {{ zeroVal .Kind }} {
{{- end }}
{{- if eq .Kind "bool" }}
val := alloc(1)
val[0] = '0'
if p.{{ .Name }} {
val[0] = '1'
}
{{- else if eq .Kind "uint32" }}
val := alloc(4)
binary.LittleEndian.PutUint32(val, p.{{ .Name }})
{{- else if eq .Kind "uint64" }}
val := alloc(10)
n := binary.PutUvarint(val, p.{{ .Name }})
val = val[:n]
{{- else if eq .Kind "string" }}
val := alloc(len(p.{{ .Name }}))
copy(val, p.{{ .Name }})
{{- end }}
m["{{ .Tag }}"] = val
}
{{- end }}
return m
}

// isLoaded returns true if the bit corresponding to field bit is set.
func (p *Properties) isLoaded(bit int) bool { return p.Loaded&(1<<bit) != 0 }

// String writes a human‑readable representation of Properties, matching the
// previous reflection‑based output.
func (p *Properties) String() string {
var buf bytes.Buffer
{{- range .Fields }}
if p.{{ .Name }} != {{ zeroVal .Kind }} || p.isLoaded(_bit_{{ .Name }}) {
fmt.Fprintf(&buf, "%s: %v\n", "{{ .Tag }}", p.{{ .Name }})
}
{{- end }}
if len(p.UserProperties) > 0 {
// Print the user properties in alphabetical order.
for _, k := range slices.Sorted(maps.Keys(p.UserProperties)) {
v := p.UserProperties[k]
if strings.IndexFunc(v, func(r rune) bool { return r < ' ' || r > '~' }) != -1 {
fmt.Fprintf(&buf, "%s: hex:%x\n", k, v)
} else {
fmt.Fprintf(&buf, "%s: %s\n", k, v)
}
}
}
return buf.String()
}

// Bit positions for property field.
const (
{{- range $i, $f := .Fields }}
_bit_{{$f.Name}} = {{$i}}
{{- end }}
_numPropBits = {{ len .Fields }}
)
` // end template

// zeroVal returns a literal zero value string for the given kind.
func zeroVal(kind string) string {
switch kind {
case "bool":
return "false"
case "uint32", "uint64":
return "0"
case "string":
return `""`
default:
return "nil"
}
}

func main() {
cfg := &packages.Config{
Mode: packages.NeedTypes | packages.NeedSyntax | packages.NeedDeps |
packages.NeedTypesInfo | packages.NeedFiles | packages.NeedName,
}

pkgs, err := packages.Load(cfg, "github.com/cockroachdb/pebble/sstable")
if err != nil {
log.Fatalf("loading packages: %v", err)
}

var fields []Field
for _, pkg := range pkgs {
for _, file := range pkg.Syntax {
for _, decl := range file.Decls {
gd, ok := decl.(*ast.GenDecl)
if !ok || gd.Tok != token.TYPE {
continue
}
for _, spec := range gd.Specs {
ts := spec.(*ast.TypeSpec)
st, ok := ts.Type.(*ast.StructType)
if !ok {
continue
}
qname := pkg.Types.Path() + "." + ts.Name.Name
if qname != "github.com/cockroachdb/pebble/sstable.Properties" &&
qname != "github.com/cockroachdb/pebble/sstable.CommonProperties" {
continue
}
for _, f := range st.Fields.List {
if f.Tag == nil || len(f.Names) == 0 {
continue
}
tags := reflect.StructTag(strings.Trim(f.Tag.Value, "`"))
tag := tags.Get("prop")
if tag == "" {
continue
}
options := strings.Split(tags.Get("options"), ",")
name := f.Names[0].Name
typ := pkg.TypesInfo.Types[f.Type].Type.String()
var kind string
switch typ {
case "bool":
kind = "bool"
case "uint32":
kind = "uint32"
case "uint64":
kind = "uint64"
case "string":
kind = "string"
default:
log.Fatalf("unsupported property type %s", typ)
}
// We always encode some properties, even if they are zero.
encodeEmpty := slices.Contains(options, "encodeempty")
intern := slices.Contains(options, "intern")
fields = append(fields, Field{Name: name, Tag: tag, Kind: kind, EncodeEmpty: encodeEmpty, Intern: intern})
}
}
}
}
}

if len(fields) > 64 {
log.Fatalf("too many prop fields (%d), exceeds 64‑bit bitfield", len(fields))
}
var sstableDir string
for _, pkg := range pkgs {
if pkg.PkgPath == "github.com/cockroachdb/pebble/sstable" {
sstableDir = filepath.Dir(pkg.GoFiles[0])
break
}
}
if sstableDir == "" {
log.Fatalf("sstable package not found")
}
outputFile := filepath.Join(sstableDir, "properties_gen.go")

t := template.Must(template.New("gen").Funcs(template.FuncMap{
"zeroVal": zeroVal,
}).Parse(tmpl))

var buf bytes.Buffer
if err := t.Execute(&buf, map[string]any{"Fields": fields}); err != nil {
log.Fatalf("executing template: %v", err)
}

src, err := format.Source(buf.Bytes())
if err != nil {
_ = os.WriteFile(outputFile, buf.Bytes(), 0o644)
log.Fatalf("formatting source: %v\n", err)
}

if err := os.WriteFile(outputFile, src, 0o644); err != nil {
log.Fatalf("writing generated file: %v", err)
}
}
Loading