Improve Type Hints + Increase Type Checking + Added Makefile

.gitignore

@@ -61,4 +61,7 @@ dbt_packages/
 # Python
 *.pyc
 
-*.db
+*.db
+*.db.*
+
+sample/dagster_project/storage/

.pre-commit-config.yaml

@@ -0,0 +1,39 @@
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.6.2  # Match your ruff version from pyproject.toml
+    hooks:
+      - id: ruff
+        args: [--fix]
+      - id: ruff-format
+
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.8.0
+    hooks:
+      - id: mypy
+        args: [
+          --cache-dir=.mypy_cache,  # Enable caching for faster runs
+          --ignore-missing-imports,  # Ignore missing imports from third-party packages
+        ]
+        additional_dependencies:
+          # Type stubs only - these are lightweight
+          - types-setuptools
+          - types-requests
+          - types-sqlalchemy
+          - types-PyYAML
+          - types-python-dateutil
+          - pyarrow-stubs>=17.0.0
+          # Core packages with minimal dependencies
+          - "dagster[mypy]>=1.7.8"  # Only mypy-related dependencies
+          - sqlmesh<1.0  # No alternative minimal package available
+          - pytest>=8.0.0  # Includes type hints
+          # Required for polars integration
+          - polars>=0.20.5  # No type stubs available yet
+          - dagster-duckdb-polars>=0.21.8  # Required for polars integration
+
+  - repo: https://github.com/astral-sh/uv-pre-commit
+    rev: 0.6.2  # Latest stable version
+    hooks:
+      - id: uv-lock
+        name: Lock dependencies
+        args: ["--python", "3.11"]
+        files: ^pyproject\.toml$ 

Makefile

@@ -0,0 +1,102 @@
+# System detection
+UNAME := "$(shell uname)"
+PYTHON_VENV_NAME := ".venv"
+
+# Python virtual environment settings
+VENV_NAME := .venv
+PYTHON := python
+
+# Check if we're on Windows
+ifeq ($(OS),Windows_NT)
+    PYTHON_CMD := $(CURDIR)/$(VENV_NAME)/Scripts/python
+    SQLMESH_CMD := $(CURDIR)/$(VENV_NAME)/Scripts/sqlmesh
+    UV_CMD := "$(subst \,/,$(USERPROFILE))/.local/bin/uv.exe"
+    ACTIVATE := source $(CURDIR)/$(VENV_NAME)/Scripts/activate
+    DEACTIVATE := source $(CURDIR)/$(VENV_NAME)/Scripts/deactivate
+else
+    PYTHON_CMD := $(CURDIR)/$(VENV_NAME)/bin/python
+    SQLMESH_CMD := $(CURDIR)/$(VENV_NAME)/bin/sqlmesh
+    UV_CMD := uv
+    ACTIVATE := source $(CURDIR)/$(VENV_NAME)/bin/activate
+    DEACTIVATE := deactivate
+endif
+
+# Python setup commands
+install-python:
+ifeq ($(UNAME),"Darwin")
+	brew install python@3.12
+else
+	@echo "Please install Python 3.12 manually for your operating system"
+	@echo "Visit: https://www.python.org/downloads/"
+	@exit 1
+endif
+
+check-uv:
+	@if [ "$(shell uname)" = "Darwin" ]; then \
+		which uv > /dev/null || (echo "Installing uv via Homebrew..." && brew install uv); \
+	else \
+		which uv > /dev/null || (echo "Installing uv via curl..." && curl -LsSf https://astral.sh/uv/install.sh | sh); \
+	fi
+
+init-python:
+	@if [ ! -d "$(PYTHON_VENV_NAME)" ]; then \
+		echo "Creating virtual environment with Python 3.12..."; \
+		uv venv --python 3.12 $(PYTHON_VENV_NAME); \
+	fi
+
+install-python-deps:
+	uv sync --all-extras
+
+upgrade-python-deps:
+	uv lock --upgrade
+	make install-python-deps
+
+# Node.js setup commands
+install-node:
+ifeq ($(UNAME),"Darwin")
+	brew install node@18
+else
+	@echo "Please install Node.js 18 manually for your operating system"
+	@echo "Visit: https://nodejs.org/dist/latest-v18.x/"
+	@exit 1
+endif
+
+check-pnpm:
+	@if [ "$(shell uname)" = "Darwin" ]; then \
+		which pnpm > /dev/null || (echo "Installing pnpm via npm..." && npm install -g pnpm); \
+	else \
+		which pnpm > /dev/null || (echo "Installing pnpm via npm..." && npm install -g pnpm); \
+	fi
+
+install-node-deps:
+	pnpm install
+
+upgrade-node-deps:
+	pnpm update
+
+# Base commands
+init: check-uv init-python install-python-deps check-pnpm install-node-deps
+
+clean:
+	find . \( -type d -name "__pycache__" -o -type f -name "*.pyc" -o -type d -name ".pytest_cache" -o -type d -name "*.egg-info" \) -print0 | xargs -0 rm -rf
+
+# Testing commands
+test:
+	$(PYTHON_CMD) -m pytest -vv --log-cli-level=INFO $(filter-out $@,$(MAKECMDGOALS))
+
+mypy:
+	$(PYTHON_CMD) -m mypy dagster_sqlmesh/ sample/
+
+# Sample project commands
+clean-dagster:
+	rm -rf sample/dagster_project/storage sample/dagster_project/logs sample/dagster_project/history
+
+dagster-dev: clean-dagster
+	DAGSTER_HOME=$(CURDIR)/sample/dagster_project $(PYTHON_CMD) -m dagster dev -h 0.0.0.0 -w sample/dagster_project/workspace.yaml
+
+dev: dagster-dev  # Alias for dagster-dev
+
+dagster-materialize:
+	$(PYTHON_CMD) -m dagster asset materialize -f sample/dagster_project/definitions.py --select '*'
+
+.PHONY: init init-python install-python check-uv install-python-deps upgrade-python-deps clean test mypy install-node check-pnpm install-node-deps upgrade-node-deps sample-dev dagster-dev dagster-materialize clean-dagster 

README.md

@@ -55,21 +55,33 @@ _Note: this is a python project but some of our dependent tools are in typescrip
 
 ### Installing
 
-To install everything you need for development just do the following:
+The project uses Make commands to simplify the development setup process. To get started:
 
 ```bash
-poetry install
-pnpm install
+make init
+```
+
+This will:
+- Set up a Python virtual environment with Python 3.12
+- Install all Python dependencies
+- Install Node.js dependencies via pnpm
+
+_Note: All Make commands automatically use the correct virtual environment - you don't need to activate it manually._
+
+To upgrade dependencies:
+```bash
+make upgrade-python-deps  # Upgrade Python dependencies
+make upgrade-node-deps   # Upgrade Node.js dependencies
 ```
 
 ### Running tests
 
 We have tests that should work entirely locally. You may see a `db.db` file appear in the root of the repository when these tests are run. It can be safely ignored or deleted.
 
-We run tests with `pytest` like so:
+To run tests:
 
 ```bash
-poetry run pytest
+make test
 ```
 
 ### Running the "sample" dagster project
@@ -79,13 +91,17 @@ accompanying sqlmesh project from `sample/sqlmesh_project` configured as an
 asset. To run the sample dagster project deployment with a UI:
 
 ```bash
-poetry run dagster dev -h 0.0.0.0 -f sample/dagster_project/definitions.py
+make dagster-dev 
+```
+or 
+```bash
+make dev
 ```
 
 If you'd like to materialize the dagster assets quickly on the CLI:
 
 ```bash
-dagster asset materialize -f sample/dagster_project/definitions.py --select '*'
+make dagster-materialize
 ```
 
 _Note: The sqlmesh project that is in the sample folder has a dependency on a

dagster_sqlmesh/asset.py

@@ -1,9 +1,10 @@
-import typing as t
 import logging
+import typing as t
 
 from dagster import (
-    multi_asset,
+    AssetsDefinition,
     RetryPolicy,
+    multi_asset,
 )
 
 from dagster_sqlmesh.controller import DagsterSQLMeshController
@@ -19,13 +20,13 @@ def sqlmesh_assets(
     *,
     environment: str,
     config: SQLMeshContextConfig,
-    name: t.Optional[str] = None,
-    dagster_sqlmesh_translator: t.Optional[SQLMeshDagsterTranslator] = None,
+    name: str | None = None,
+    dagster_sqlmesh_translator: SQLMeshDagsterTranslator | None = None,
     compute_kind: str = "sqlmesh",
-    op_tags: t.Optional[t.Mapping[str, t.Any]] = None,
-    required_resource_keys: t.Optional[t.Set[str]] = None,
-    retry_policy: t.Optional[RetryPolicy] = None,
-):
+    op_tags: t.Mapping[str, t.Any] | None = None,
+    required_resource_keys: set[str] | None = None,
+    retry_policy: RetryPolicy | None = None,
+) -> t.Callable[[t.Callable[..., t.Any]], AssetsDefinition]:
     controller = DagsterSQLMeshController.setup_with_config(config)
     if not dagster_sqlmesh_translator:
         dagster_sqlmesh_translator = SQLMeshDagsterTranslator()
@@ -40,4 +41,5 @@ def sqlmesh_assets(
         compute_kind=compute_kind,
         retry_policy=retry_policy,
         required_resource_keys=required_resource_keys,
+        can_subset=True,
     )

dagster_sqlmesh/config.py

@@ -1,16 +1,16 @@
-from typing import Optional, Dict, Any
 from dataclasses import dataclass
+from typing import Any
 
 from dagster import Config
-from sqlmesh.core.config import Config as MeshConfig
 from pydantic import Field
+from sqlmesh.core.config import Config as MeshConfig
 
 
 @dataclass
 class ConfigOverride:
-    config_as_dict: Dict
+    config_as_dict: dict
 
-    def dict(self):
+    def dict(self) -> dict:
         return self.config_as_dict
 
 
@@ -24,10 +24,10 @@ class SQLMeshContextConfig(Config):
 
     path: str
     gateway: str
-    config_override: Optional[Dict[str, Any]] = Field(default_factory=lambda: None)
+    config_override: dict[str, Any] | None = Field(default_factory=lambda: None)
 
     @property
-    def sqlmesh_config(self):
+    def sqlmesh_config(self) -> MeshConfig | None:
         if self.config_override:
             return MeshConfig.parse_obj(self.config_override)
         return None