added slice fields support

slice fields length configuration via struct tags;
generated code optimizations;
code improvements;
readme updated;

Author: amanofbits

README.md

@@ -1,11 +1,45 @@
 # Simser
-**Sim**ple **ser**ialization code generator for GO structs with unexported fields
+**Sim**ple **ser**ialization code generator for GO structs with unexported fields.  
+
+Like `binary.Read`, but a bit different and without reflection.
+
+## Features
+
+### Core
+- go module mode only
+- simple sequential [de]serialization of simple structs
+- supports non-exported fields, as well as exported
+- basic (`int32`, `float64`, etc.), named (`type My uint64`, etc.) field types, and arrays of them (struct type fields,
+array of arrays and slice of arrays are not supported right now)
+- no reflection in generated code, it is simple and fast
+- possibility to select type[s] to serialize via `-types` CLI flag
+- possibility to select output function names (and exportability, as implied by Go)
+
+### Advanced
+- slice field length can be set to any expression that returns `int`, by using tags. Currently [de]serialized instance can be referred to as "`o`", within expression.  
+E.g. `simser:"len=o.PreviousIntegerField-5"`.  
+Or `simser:"len=otherFunc()"`  
+Remember that only fields that get read _before_ the slice field will have meaningful values (unless some tricks were used)
+
+## Usage
+
+#### Basic
+
+`//go:generate go run github.com/am4n0w4r/simser -types=Header,body`
+`//go:generate go run github.com/am4n0w4r/simser -types=all`
+
+- `-types` (required): can be a comma-separated list of types you want to process, or reserved keyword `all` for processing all top-level `struct`s, found in file.  
+`all` usage and requiredness of the argument can change in the future.
+
+#### Custom
+
+`//go:generate go run github.com/am4n0w4r/simser -types=Header -read-fn-name=customReadFnName -write-fn-name=CustomWriteFnName`
+
+- `-read-fn-name` (optional): custom name for deserializing function. Is set per-file.
+- `-write-fn-name` (optional): custom name for deserializing function. Is set per-file.
 
-This is a generator that should help with simple sequential reading/writing of structs to/from binary file.  
-`binary.Read` actually does the same thing and, probably, does this well, but it has a drawback - it doesn't work with non-exported fields.  
-Also, `binary` stuff is a runtime handler, so it has some overhead, while code, generated by this module tries to be simple and fast.
 
 ## Project state
 
-A bit messy, probably unoptimal, but simple and working. It was developed quickly from scratch, to serve a purpose, so the code itself is rather not perfect, but generated code should be at least ok.  
-It's the first time I saw go's ast, so it was a lot of try-and-fail behind the scenes. Feel free to file issues.
+A bit messy, not very optimal, but simple and working. It was developed quickly from scratch, to serve a particular practical purpose, so the code itself is rather not perfect, but generated code should be good and do the job.  
+It's the first time I worked with go's ast, so it was a lot of try-and-fail behind the scenes. Feel free to file issues.

internal/domain/domain.go

@@ -20,6 +20,7 @@ import (
 	"errors"
 	"fmt"
 	"go/types"
+	"strconv"
 	"strings"
 )
 
@@ -31,8 +32,8 @@ type InputStruct struct {
 	fields []StructField
 }
 
-func NewInputStruct(name string, typ *types.Struct) InputStruct {
-	return InputStruct{
+func NewInputStruct(name string, typ *types.Struct) *InputStruct {
+	return &InputStruct{
 		name: name,
 		typ:  typ,
 	}
@@ -50,45 +51,69 @@ func (s *InputStruct) SetFields(f []StructField) { s.fields = f }
 type StructField struct {
 	name string
 	typ  FieldType
+	tag  map[string]string
 }
 
-func NewStructField(name string, typ FieldType) StructField {
+func NewStructField(name string, typ FieldType, tag map[string]string) StructField {
 	return StructField{
 		name: name,
 		typ:  typ,
+		tag:  tag,
 	}
 }
 
 func (f StructField) Name() string    { return f.name }
 func (f StructField) Type() FieldType { return f.typ }
 
-//
+// Type
 
 type FieldType interface {
 	Name() string
-	Size() int // Total size of the field
-	// Don't confuse with IsInt. Checks if type is intX OR uintX
+	// Total size of the field.
+	// If Size() < 0, then size is not fixed, and SizeExpr() should be used.
+	Size() int
+	// Expression that determines the size of the field.
+	// Should always be constructed in a way to return int type.
+	SizeExpr() string
+	// Don't confuse with IsInt. Checks if type is intX OR uintX OR byte
 	IsInteger() bool
+	// Should return true if the type is an array or slice
+	IsSequence() bool
+}
+
+// Field type that is a sequence, i.e. array or slice, which have a number of certain elements
+type SequenceFieldType interface { // No need to make this a more generic collection, as [de]serialization is sequential
+	ElType() FieldType
+	LenExpr() string
 }
 
+// Simple
+
 type SimpleFieldType struct {
 	name       string
 	size       int
 	underlying *SimpleFieldType
 }
 
 func NewSimpleFieldType(name string, size int, underlying *SimpleFieldType) *SimpleFieldType {
+	if size < 1 {
+		panic(fmt.Sprintf("simple type size < 1. This should be caught earlier. Please, file a bug to the repo. Type %s, size %d",
+			name, size))
+	}
 	return &SimpleFieldType{
 		name:       name,
 		size:       size,
 		underlying: underlying,
 	}
 }
 
-func (t SimpleFieldType) Name() string                 { return t.name }
+func (t SimpleFieldType) Name() string { return t.name }
+
 func (t SimpleFieldType) Size() int                    { return t.size }
+func (t SimpleFieldType) SizeExpr() string             { return strconv.Itoa(t.size) }
 func (t SimpleFieldType) BitSize() int                 { return t.size * 8 }
 func (t SimpleFieldType) Underlying() *SimpleFieldType { return t.underlying }
+func (t SimpleFieldType) IsSequence() bool             { return false }
 
 func (bt SimpleFieldType) IsInteger() bool {
 	tmp := &bt
@@ -98,20 +123,65 @@ func (bt SimpleFieldType) IsInteger() bool {
 	return strings.HasPrefix(tmp.name, "int") || strings.HasPrefix(tmp.name, "uint") || tmp.name == "byte"
 }
 
+// Array
+
 type ArrayFieldType struct {
 	length int
 	elType FieldType
 }
 
 func NewArrayFieldType(length int, el FieldType) *ArrayFieldType {
+	if length < 0 {
+		panic(fmt.Sprintf("array length < 1. This should be caught earlier. Please, file a bug to the repo. Length %d",
+			length))
+	}
 	return &ArrayFieldType{
 		length: length,
 		elType: el,
 	}
 }
 
-func (at ArrayFieldType) Name() string      { return fmt.Sprintf("[]%s", at.elType.Name()) }
-func (at ArrayFieldType) Size() int         { return at.elType.Size() * at.length }
-func (at ArrayFieldType) Length() int       { return at.length }
-func (at ArrayFieldType) ElType() FieldType { return at.elType }
-func (at ArrayFieldType) IsInteger() bool   { return false }
+func (t ArrayFieldType) Name() string { return fmt.Sprintf("[]%s", t.elType.Name()) }
+
+func (t ArrayFieldType) Size() int {
+	if !IsFixedSize(t.elType.Size()) {
+		return -1
+	}
+	return t.elType.Size() * t.length
+}
+func (t ArrayFieldType) SizeExpr() string {
+	if IsFixedSize(t) {
+		return strconv.Itoa(t.elType.Size() * t.length)
+	}
+	return fmt.Sprintf("(%s) * %d", t.elType.SizeExpr(), t.length)
+}
+func (t ArrayFieldType) Length() int       { return t.length }
+func (t ArrayFieldType) LenExpr() string   { return strconv.Itoa(t.length) }
+func (t ArrayFieldType) ElType() FieldType { return t.elType }
+func (t ArrayFieldType) IsInteger() bool   { return false }
+func (t ArrayFieldType) IsSequence() bool  { return true }
+
+// Slice
+
+type SliceFieldType struct {
+	lenExpr string // expression used to calculate length.
+	elType  FieldType
+}
+
+func NewSliceFieldType(lenExpr string, el FieldType) *SliceFieldType {
+	return &SliceFieldType{
+		lenExpr: strings.TrimSpace(lenExpr),
+		elType:  el,
+	}
+}
+
+func (t SliceFieldType) Name() string { return fmt.Sprintf("[]%s", t.elType.Name()) }
+
+func (t SliceFieldType) Size() int { return -1 }
+func (t SliceFieldType) SizeExpr() string {
+	return fmt.Sprintf("%s * %s", ParenthesizeIntExpr(t.elType.SizeExpr()), ParenthesizeIntExpr(t.lenExpr))
+}
+func (t SliceFieldType) LenExpr() string   { return t.lenExpr }
+func (t SliceFieldType) ElType() FieldType { return t.elType }
+func (t SliceFieldType) IsInteger() bool   { return false }
+func (t SliceFieldType) IsSequence() bool  { return true }

internal/domain/functions.go

@@ -0,0 +1,67 @@
+// Copyright 2023 am4n0w4r
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package domain
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+)
+
+/******************************************************************/
+/* This file should be kept as small as possible.                 */
+/* Ony functions that are used in domain methods and ALSO outside */
+/******************************************************************/
+
+// Adds parentheses to Go's int expression if it is
+// a) not a simple int value,
+// b) not already cast to int.
+func ParenthesizeIntExpr(expr string) string {
+	if _, err := strconv.Atoi(expr); err == nil {
+		return expr
+	}
+	if !strings.HasPrefix(expr, "int") || !strings.HasSuffix(expr, ")") {
+		return fmt.Sprintf("(%s)", expr)
+	}
+	pos := 3
+	for expr[pos] == ' ' {
+		pos++
+	}
+	if expr[pos] != '(' {
+		return fmt.Sprintf("(%s)", expr)
+	}
+	return expr
+}
+
+// Returns true if total argument's size can be interpreted as known at declaration time.
+// E.g. (primitives, arrays). NOT slices.
+func IsFixedSize[
+	T StructField | SimpleFieldType | ArrayFieldType | SliceFieldType | int](a T) bool {
+
+	switch arg := any(a).(type) {
+	case StructField:
+		return arg.Type().Size() >= 0
+	case SimpleFieldType:
+		return arg.Size() >= 0
+	case ArrayFieldType:
+		return arg.Size() >= 0
+	case SliceFieldType:
+		return arg.Size() >= 0
+	case int:
+		return arg >= 0
+	default:
+		panic(fmt.Errorf("unknown type %T", a))
+	}
+}

internal/generator/generator.go

@@ -23,25 +23,54 @@ func GenStructCode(s domain.InputStruct, out *Output, readFnName, writeFnName st
 		return nil
 	}
 
-	totalBufLen := 0
-	for i := 0; i < s.FieldCount(); i++ {
-		totalBufLen += s.Field(i).Type().Size()
-	}
+	sizeGroups := getFieldSizeGroups(s)
 
-	// Generate func (o 'typename')LoadFrom(io.Reader) (*typename, error)
+	// Generate func (o 'typename')LoadFrom(io.Reader) (*'typename', error)
 	{
 		out.AppendImport("io")
 
 		out.AppendF("func (o *%s) %s(r io.Reader) (n int, err error) {\n", s.Name(), readFnName)
-		out.AppendF("b := make([]byte, %d)\n", totalBufLen)
-		out.AppendF("p := 0\n")
+		out.AppendF("p, nRead := 0, 0\n")
+
+		out.AppendF("toRead := ")
+		if domain.IsFixedSize(sizeGroups[0]) {
+			out.AppendF("%d\n", sizeGroups[0])
+		} else {
+			out.AppendF("%s\n", s.Field(0).Type().SizeExpr())
+		}
+
+		for i := 0; i < s.FieldCount(); i++ {
+			if s.Field(i).Type().IsSequence() {
+				out.AppendF("sLen, sElSize := 0, 0\n")
+				break
+			}
+		}
+
 		out.LF()
-		out.Append(tpl_ReadNbytesIntoBuf("b", uint(totalBufLen))).LF()
+		out.AppendF("b := make([]byte, toRead)\n")
+		out.Append(tpl_ReadBytesIntoBuf("b")).LF()
 		out.LF()
 
 		for i := 0; i < s.FieldCount(); i++ {
 			field := s.Field(i)
-			out.AppendF("// %s", field.Name()).LF()
+			out.AppendF("\n// %s\n", field.Name())
+			if i != 0 {
+				if size, ok := sizeGroups[i]; ok {
+					if !domain.IsFixedSize(size) {
+						if field.Type().IsSequence() {
+							seqType := field.Type().(domain.SequenceFieldType)
+							out.AppendF("sLen, sElSize = %s, %s\n", seqType.LenExpr(), seqType.ElType().SizeExpr())
+						}
+						out.Append("p, toRead = 0, sLen * sElSize\n")
+					} else {
+						out.AppendF("p, toRead = 0, %d\n", size)
+					}
+					out.AppendF("if toRead > cap(b) {\n")
+					out.AppendF("b = make([]byte, toRead)\n")
+					out.Append("}\n")
+					out.Append(tpl_ReadBytesIntoBuf("b")).LF()
+				}
+			}
 			s, err := tpl_ReadField(field, "b")
 			if err != nil {
 				return err
@@ -60,13 +89,25 @@ func GenStructCode(s domain.InputStruct, out *Output, readFnName, writeFnName st
 	{
 		out.AppendImport("io")
 
-		out.AppendF("func (s *%s) %s(w io.Writer) (n int, err error) {\n", s.Name(), writeFnName)
-		out.AppendF("b := make([]byte, 0, %d)\n", totalBufLen)
+		out.AppendF("func (o *%s) %s(w io.Writer) (n int, err error) {\n", s.Name(), writeFnName)
+
+		out.AppendF("b := make([]byte, 0, ")
+		sb := fstringBuilder{}
+		constSize := 0
+		for i, size := range sizeGroups {
+			if !domain.IsFixedSize(size) {
+				expr := s.Field(i).Type().SizeExpr()
+				sb.WriteFString("+ %s", domain.ParenthesizeIntExpr(expr))
+				continue
+			}
+			constSize += size
+		}
+		out.AppendF("%d %s)\n", constSize, sb.String())
 		out.LF()
 
 		for i := 0; i < s.FieldCount(); i++ {
 			field := s.Field(i)
-			out.AppendF("// %s", field.Name()).LF()
+			out.AppendF("\n// %s", field.Name()).LF()
 			s, err := tpl_WriteField(field, "b")
 			if err != nil {
 				return err