Create target-jsonl-blob (#1)

* it's working

* cleanup

* add workflows

* fix release input values

* unmarshall into singerMessage

* support more fields in key template

* refactor

Author: edgarrmondragon

.github/workflows/release.yml

@@ -0,0 +1,37 @@
+on:
+  release:
+    types: [created]
+jobs:
+  releases-matrix:
+    name: Release Go Binaries
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+        - goos: linux
+          goarch: "386"
+        - goos: linux
+          goarch: "amd64"
+        - goos: linux
+          goarch: "arm64"
+        - goos: darwin
+          goarch: "amd64"
+        - goos: darwin
+          goarch: "arm64"
+        - goos: windows
+          goarch: "386"
+        - goos: windows
+          goarch: "amd64"
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v2
+    - name: Release binaries
+      uses: wangyoucao577/go-release-action@v1.25
+      with:
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+        goos: ${{ matrix.goos }}
+        goarch: ${{ matrix.goarch }}
+        goversion: "1.17"
+        project_path: ./
+        binary_name: target-jsonl-blob
+        extra_files: ./README.md

.github/workflows/test.yml

@@ -0,0 +1,21 @@
+name: Test
+on:
+  pull_request: {}
+  push:
+    branches: [main]
+jobs:
+  test:
+    strategy:
+      matrix:
+        go-version: [1.16.x, 1.17.x]
+        os: [ubuntu-latest, macos-latest, windows-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go-version }}
+    - name: Checkout code
+      uses: actions/checkout@v2
+    - name: Test
+      run: go test ./...

.gitignore

@@ -1,4 +1,13 @@
+# Config files
+*.json
+*.yaml
+!*.example.*
+
+# Data files
+**/*.jsonl
+
 # Binaries for programs and plugins
+target-jsonl-blob
 *.exe
 *.exe~
 *.dll

LICENSE

@@ -1,3 +1,4 @@
+
                                  Apache License
                            Version 2.0, January 2004
                         http://www.apache.org/licenses/

README.md

@@ -1,18 +1,53 @@
 # target-jsonl-blob
 
-JSONL Singer target for local storage, S3, GCS and Azure Blob Storage.
+JSONL Singer target for local storage, S3 and Azure Blob Storage.
 
-## Immediate Roadmap
+## Configuration
 
-- Use Go and https://gocloud.dev/howto/blob
-- Build a working singer target that can write to the destinations mentioned above
-- Make the destination easily configurable with a "bucket" URL (e.g. `"s3://my-bucket?region=us-west-1"`)
-- Use a configurable template for the object keys with a rich set of available inputs (e.g. `prefix/{stream_name}/{day}/{hour}.jsonl`)
+| Setting | Required | Default | Description |
+|----------------|----------|-------------------------|-------------------------------|
+| `bucket` | Yes | - | Blob storage [bucket URL](#bucket-urls) |
+| `key_template` | No | `{{.StreamName}}.jsonl` | Template string for file keys |
+
+### Bucket URLs
+
+| Storage | Example URL                 |
+|---------|-----------------------------|
+| local   | `file:///path/to/directory` |
+| S3      | `s3://my-bucket`            |
+| Azure   | `azblob://my-container`     |
+
+### Available fields for `key_template`
+
+- `StreamName`
+- `Date` (YYYY-MM-DD)
+- `TimestampSeconds`
+- `Minute`
+- `Hour`
+- `Day`
+- `Month`
+- `Year`
+
+Example: `{{.StreamName}}/{{.Year}}/{{.Month}}/{{.Day}}/{{.Hour}}/{{.Minute}}/{{.StreamName}}.jsonl`
 
 ## Build from source
 
-_TODO_
+```shell
+go build -o target-jsonl-blob
+```
 
 ## Usage with Meltano
 
 _TODO_
+
+## Roadmap
+
+- Support GCS
+
+  Currently blocked by
+
+  ```
+  cloud.google.com/go/storage@v1.16.1/storage.go:1416:53: o.GetCustomerEncryption().GetKeySha256 undefined (type *"google.golang.org/genproto/googleapis/storage/v2".Object_CustomerEncryption has no field or method GetKeySha256)
+  ```
+
+- Build a lighter binary

cmd/root.go

@@ -0,0 +1,117 @@
+/*
+Copyright © 2022 Edgar Ramírez-Mondragón
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"log"
+	"os"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	"meltano.com/target-jsonl-blob/target"
+
+	"gocloud.dev/blob"
+	// _ "gocloud.dev/blob/gcsblob"
+	_ "gocloud.dev/blob/fileblob"
+	_ "gocloud.dev/blob/s3blob"
+)
+
+var (
+	configFile string
+	inputFile  string
+	C          target.Config
+)
+
+var rootCmd = &cobra.Command{
+	Use:   "target-jsonl-blob",
+	Short: "JSONLines Singer target for blob storages",
+	Run:   func(cmd *cobra.Command, args []string) {},
+}
+
+// Execute adds all child commands to the root command and sets flags appropriately.
+// This is called by main.main(). It only needs to happen once to the rootCmd.
+func Execute() {
+	err := rootCmd.Execute()
+	if err != nil {
+		os.Exit(1)
+	}
+
+	readConfig(configFile)
+
+	if C.Bucket == "" {
+		log.Fatalf("Bucket is required")
+	}
+
+	if C.KeyTemplate == "" {
+		C.KeyTemplate = target.DEFAULT_KEY_TEMPLATE
+	}
+
+	ctx := context.Background()
+	bucket, err := blob.OpenBucket(ctx, C.Bucket)
+	if err != nil {
+		log.Fatalf("Unable to open bucket, %v", err)
+	}
+	defer bucket.Close()
+
+	var lines io.Reader
+
+	if inputFile == "" {
+		lines = os.Stdin
+	} else {
+		lines, err = os.Open(inputFile)
+		if err != nil {
+			fmt.Print(err)
+		}
+	}
+
+	target.ProcessLines(lines, C, ctx, bucket)
+
+	if inputFile == "" {
+		target.ProcessLines(os.Stdin, C, ctx, bucket)
+	} else {
+		inputLines, err := os.Open(inputFile)
+		if err != nil {
+			fmt.Print(err)
+		}
+		target.ProcessLines(inputLines, C, ctx, bucket)
+	}
+
+}
+
+func init() {
+	rootCmd.PersistentFlags().StringVarP(&configFile, "config", "c", "", "Config file")
+	rootCmd.PersistentFlags().StringVarP(&inputFile, "input", "i", "", "Input file")
+	rootCmd.MarkPersistentFlagRequired("config")
+}
+
+// initConfig reads in config file and ENV variables if set.
+func readConfig(file string) {
+	viper.SetConfigFile(file)
+
+	if err := viper.ReadInConfig(); err == nil {
+		log.Printf("Using config file %s", viper.ConfigFileUsed())
+	} else {
+		fmt.Fprintln(os.Stderr, "Config file is not valid")
+		os.Exit(1)
+	}
+
+	if err := viper.Unmarshal(&C); err != nil {
+		log.Fatalf("Unable to decode into struct, %v", err)
+	}
+}

config.example.json

@@ -0,0 +1,4 @@
+{
+  "bucket": "file://./output/my-bucket",
+  "key_template": "{{.StreamName}}.jsonl"
+}

config.example.yaml

@@ -0,0 +1,2 @@
+bucket: file://./output/my-bucket
+key_template: "{{.StreamName}}.jsonl"

configS3.example.yaml

@@ -0,0 +1,2 @@
+bucket: s3://my-bucket?endpoint=http://127.0.0.1:9000&disableSSL=true&s3ForcePathStyle=true
+key_template: "{{.StreamName}}/{{.Year}}/{{.Month}}/{{.Day}}/{{.Hour}}/{{.Minute}}/{{.StreamName}}.jsonl"

go.mod

@@ -0,0 +1,49 @@
+module meltano.com/target-jsonl-blob
+
+go 1.17
+
+require (
+	github.com/spf13/cobra v1.3.0
+	github.com/spf13/viper v1.10.1
+	gocloud.dev v0.24.0
+)
+
+require (
+	github.com/aws/aws-sdk-go v1.40.34 // indirect
+	github.com/aws/aws-sdk-go-v2 v1.9.0 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.7.0 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.4.0 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.5.0 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.2.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.4.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.7.0 // indirect
+	github.com/aws/smithy-go v1.8.0 // indirect
+	github.com/fsnotify/fsnotify v1.5.1 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/wire v0.5.0 // indirect
+	github.com/googleapis/gax-go/v2 v2.1.1 // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/magiconair/properties v1.8.5 // indirect
+	github.com/mitchellh/mapstructure v1.4.3 // indirect
+	github.com/pelletier/go-toml v1.9.4 // indirect
+	github.com/spf13/afero v1.6.0 // indirect
+	github.com/spf13/cast v1.4.1 // indirect
+	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/subosito/gotenv v1.2.0 // indirect
+	go.opencensus.io v0.23.0 // indirect
+	golang.org/x/net v0.0.0-20210825183410-e898025ed96a // indirect
+	golang.org/x/sys v0.0.0-20211210111614-af8b64212486 // indirect
+	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	google.golang.org/api v0.63.0 // indirect
+	google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa // indirect
+	google.golang.org/grpc v1.43.0 // indirect
+	google.golang.org/protobuf v1.27.1 // indirect
+	gopkg.in/ini.v1 v1.66.2 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+)