Added optional path to the codegen file for configuring the acceptance tests (#502)

Improved github action for acceptance tests to accept optionally path to
the `.codegen.json` for configuring the tests. This makes it possible to
run multiple type of tests separately within one code based.

**Details:**
The github action for acceptance tests
`databrickslabs/sandbox/acceptance@acceptance/v0.4.3` uses the first
`.codegen.json` file found in the project. For that reason it is
currently not possible to run different type of tests separately using
this action.

With the proposed changes, a path to the codegen can be optionally
provided, e.g.
```
- name: Run integration tests
  uses: databrickslabs/sandbox/acceptance@acceptance/v0.4.4
  with:
    vault_uri: ${{ secrets.VAULT_URI }}
    timeout: 2h
    codegen_path: tests/integration/.codegen.json

- name: Run e2e tests
  uses: databrickslabs/sandbox/acceptance@acceptance/v0.4.4
  with:
    vault_uri: ${{ secrets.VAULT_URI }}
    timeout: 2h
    codegen_path: tests/e2e/.codegen.json
```

tests/integration/.codegen.json:
```
{
  "version": {
    "src/databricks/labs/dqx/__about__.py": "__version__ = \"$VERSION\""
  },
  "toolchain": {
    "required": ["python3", "hatch"],
    "pre_setup": ["hatch env create"],
    "prepend_path": ".venv/bin",
    "acceptance_path": "tests/integration"
  }
}
```

tests/e2e/.codegen.json:
```
{
  "version": {
    "src/databricks/labs/dqx/__about__.py": "__version__ = \"$VERSION\""
  },
  "toolchain": {
    "required": ["python3", "hatch"],
    "pre_setup": ["hatch env create"],
    "prepend_path": ".venv/bin",
    "acceptance_path": "tests/e2e"
  }
}
```

The change is backward compatible so the old behaviour is retained:
```
- name: Run integration tests
  uses: databrickslabs/sandbox/acceptance@acceptance/v0.4.4
  with:
    vault_uri: ${{ secrets.VAULT_URI }}
    timeout: 2h
```

This will search for `.codegen.json` in the code and use the first
codegen found.

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

Author: mwojtyczka

acceptance/README.md

@@ -58,8 +58,82 @@ sequenceDiagram
 
 ## Usage
 
-Add to your `.github/workflows` folder:
+Add a file named `acceptance.yml` to the `.github/workflows` folder in your repository. 
+This file defines a GitHub Actions workflow for running acceptance tests on pull requests. 
+The workflow sets up the environment, installs dependencies, executes tests, and uploads artifacts for further analysis.
 
+Example `acceptance.yml` for Python projects:
+
+```yaml
+name: acceptance
+
+on:
+  pull_request:
+    types: [ opened, synchronize, ready_for_review ]
+  merge_group:
+    types: [ checks_requested ]
+
+permissions:
+  id-token: write
+  contents: read
+  pull-requests: write
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  acceptance:
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install Python
+        uses: actions/setup-python@v5
+        with:
+          cache: 'pip'
+          cache-dependency-path: '**/pyproject.toml'
+          python-version: '3.10'
+
+      - name: Install hatch
+        run: pip install hatch==1.9.4
+
+      - name: Run integration tests
+        uses: databrickslabs/sandbox/acceptance@acceptance/v0.4.4
+        with:
+          vault_uri: ${{ secrets.VAULT_URI }}
+          timeout: 2h
+          # optional path to codegen file containing configuration for the tests
+          # by default first `codegen.json` file found in the repository is used
+          # codegen_path: tests/integration/.codegen.json
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+```
+
+The project must include a `.codegen.json` file, which defines the toolchain configuration for the action. 
+This configuration specifies details such as versioning, required tools, and the paths to acceptance tests. 
+If a `codegen_path` field is not explicitly provided, the action will automatically search the project for the `.codegen.json` file and use the first one it locates.
+
+Example `.codegen.json`:
+```json
+{
+  "version": {
+    "src/databricks/labs/project_name/__about__.py": "__version__ = \"$VERSION\""
+  },
+  "toolchain": {
+    "required": ["python3", "hatch"],
+    "pre_setup": ["hatch env create"],
+    "prepend_path": ".venv/bin",
+    "acceptance_path": "tests/integration"
+  }
+}
+```
+
+Note: if `acceptance_path` is not provided in the `codegen.json`, the action will execute all tests in the project by default.
+
+Example for uploading artifacts to GitHub Actions:
 ```yaml
 name: acceptance
 

acceptance/action.yml

@@ -24,6 +24,9 @@ inputs:
     description: 'Create issues in the repository for failed tests'
     required: false
     default: false
+  codegen_path:
+    description: 'Relative path to the .codegen.json file to use for configuring the acceptance tests'
+    required: false  # by default the first .codegen.json found in the project is used
 outputs:
   sample:
     description: 'Sample output'

acceptance/ecosystem/python.go

@@ -91,7 +91,8 @@ func (py *pyContext) Start(script string, reply *localHookServer) chan error {
 }
 
 func (r pyTestRunner) prepare(ctx context.Context, redact redaction.Redaction, logfile string) (*pyContext, error) {
-	tc, err := toolchain.FromFileset(r.files)
+	codegenPath := env.Get(ctx, "codegen_path")
+	tc, err := toolchain.FromFileset(r.files, &codegenPath)
 	if err != nil {
 		return nil, fmt.Errorf("detect: %w", err)
 	}

acceptance/main.go

@@ -68,6 +68,9 @@ func (a *acceptance) trigger(ctx context.Context) (*notify.Notification, error)
 	defer stop()
 	// make sure that test logs leave their artifacts somewhere we can pickup
 	ctx = env.Set(ctx, ecosystem.LogDirEnv, artifactDir)
+	// set codegen path file used for configuring the acceptance tests
+	codegenPath := a.Action.GetInput("codegen_path")
+	ctx = env.Set(ctx, "codegen_path", codegenPath)
 	redact := loaded.Redaction()
 	report, err := a.runWithTimeout(ctx, redact, directory)
 	if err != nil {

go-libs/linkdev/repo.go

@@ -58,7 +58,7 @@ func (r Repo) Retest(ctx context.Context, upstreamFolder string) error {
 		return fmt.Errorf("fileset: %w", err)
 	}
 	// defer cleanup()
-	tc, err := toolchain.FromFileset(downstreamFiles)
+	tc, err := toolchain.FromFileset(downstreamFiles, nil)
 	if err != nil {
 		return fmt.Errorf("toolchain: %w", err)
 	}

go-libs/toolchain/toolchain.go

@@ -15,17 +15,33 @@ import (
 	"github.com/databrickslabs/sandbox/go-libs/process"
 )
 
-var ErrNotExist = fmt.Errorf("no .codegen.json found. %w", fs.ErrNotExist)
+func FromFileset(files fileset.FileSet, codegenPath *string) (*Toolchain, error) {
+	var raw []byte
+	var err error
 
-func FromFileset(files fileset.FileSet) (*Toolchain, error) {
-	configs := files.Filter(".codegen.json")
-	if len(configs) == 0 {
-		return nil, ErrNotExist
+	// Helper function to filter files and retrieve raw content
+	getFileContent := func(filter string) ([]byte, error) {
+		filteredFiles := files.Filter(filter)
+		if len(filteredFiles) == 0 {
+			return nil, fmt.Errorf("file not found in fileset: %s", filter)
+		}
+		return filteredFiles[0].Raw()
 	}
-	raw, err := configs[0].Raw()
-	if err != nil {
-		return nil, fmt.Errorf("read: %w", err)
+
+	// Check if codegenPath is provided and retrieve content
+	if codegenPath != nil && *codegenPath != "" {
+		raw, err = getFileContent(*codegenPath)
+		if err != nil {
+			return nil, fmt.Errorf("provided 'codegen_path' does not exist in the project: %w", fs.ErrNotExist)
+		}
+	} else {
+		raw, err = getFileContent(".codegen.json")
+		if err != nil {
+			return nil, fmt.Errorf("no .codegen.json found. %w", fs.ErrNotExist)
+		}
 	}
+
+	// Unmarshal JSON content into dotCodegen struct
 	var dc dotCodegen
 	err = json.Unmarshal(raw, &dc)
 	if err != nil {