Skip to content

Commit 2321abf

Browse files
KLongeKLonge
committed
feat(tests): add permanent SQLMesh project fixture and update tests
- Introduced `permanent_sqlmesh_project` fixture to provide a stable sample SQLMesh project for testing. - Updated `model_change_test_context` to utilize the new permanent project fixture for improved debugging. - Added a new row to `seed_data_1.csv` for enhanced test coverage. - Marked `test_given_model_chain_when_running_with_different_flags_then_behaves_as_expected` as a work-in-progress. - Commented out unused code in the test to focus on current implementation.
1 parent 64e118d commit 2321abf

File tree

4 files changed

+111
-79
lines changed

4 files changed

+111
-79
lines changed

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,3 +67,4 @@ sample/dagster_project/storage/
6767
sample/dagster_project/logs/
6868
sample/dagster_project/history/
6969
sample/dagster_project/schedules/
70+
tests/temp/

‎sample/sqlmesh_project/seeds/seed_data_1.csv

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,4 @@ id,item_id,event_date
66
5,1,2023-06-05
77
6,1,2024-07-06
88
7,1,2024-07-07
9+
8,4,2024-06-07

‎tests/conftest.py

Lines changed: 42 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -382,22 +382,56 @@ def sample_sqlmesh_test_context(
382382
yield test_context
383383

384384

385+
@pytest.fixture
386+
def permanent_sqlmesh_project() -> str:
387+
"""Returns the path to the permanent sample SQLMesh project.
388+
389+
This fixture provides access to the sample project without copying to a temp directory,
390+
which is useful for debugging and investigating issues with file handling.
391+
It creates a permanent copy of the sample project in tests/temp/sqlmesh_project
392+
if it doesn't exist.
393+
394+
Returns:
395+
str: Absolute path to the sample SQLMesh project directory
396+
"""
397+
# Define source and target paths
398+
source_dir = os.path.abspath("sample/sqlmesh_project")
399+
project_dir = os.path.abspath("tests/temp/sqlmesh_project")
400+
401+
# Create the temp directory if it doesn't exist
402+
os.makedirs(os.path.dirname(project_dir), exist_ok=True)
403+
404+
# If project directory doesn't exist or is empty, copy from sample
405+
if not os.path.exists(project_dir) or not os.listdir(project_dir):
406+
if os.path.exists(project_dir):
407+
shutil.rmtree(project_dir)
408+
shutil.copytree(source_dir, project_dir)
409+
410+
# Clean up any existing db file
411+
# db_path = os.path.join(project_dir, "db.db")
412+
# if os.path.exists(db_path):
413+
# os.remove(db_path)
414+
415+
return project_dir
416+
417+
385418
@pytest.fixture
386419
def model_change_test_context(
387-
sample_sqlmesh_project: str,
420+
permanent_sqlmesh_project: str,
388421
) -> t.Generator[SQLMeshTestContext, None, None]:
389422
"""Creates a SQLMesh test context specifically for testing model code changes.
390423
391424
This fixture provides a context that allows modifying SQL model files and ensures
392-
they are properly restored after the test completes.
425+
they are properly restored after the test completes. It uses a permanent project
426+
directory instead of a temporary one for better debugging and investigation.
393427
394428
Args:
395-
sample_sqlmesh_project: The base project directory
429+
permanent_sqlmesh_project: The permanent project directory
396430
397431
Yields:
398432
SQLMeshTestContext: A test context with additional methods for modifying model files
399433
"""
400-
db_path = os.path.join(sample_sqlmesh_project, "db.db")
434+
db_path = os.path.join(permanent_sqlmesh_project, "db.db")
401435
config = SQLMeshConfig(
402436
gateways={
403437
"local": GatewayConfig(connection=DuckDBConnectionConfig(database=db_path)),
@@ -407,19 +441,19 @@ def model_change_test_context(
407441
)
408442
config_as_dict = config.dict()
409443
context_config = SQLMeshContextConfig(
410-
path=sample_sqlmesh_project, gateway="local", config_override=config_as_dict
444+
path=permanent_sqlmesh_project, gateway="local", config_override=config_as_dict
411445
)
412446
test_context = SQLMeshTestContext(
413447
db_path=db_path,
414448
context_config=context_config,
415-
project_path=sample_sqlmesh_project,
449+
project_path=permanent_sqlmesh_project,
416450
)
417-
test_context.initialize_test_source()
451+
# test_context.initialize_test_source()
418452

419453
yield test_context
420454

421455
# Cleanup: restore any modified files
422-
test_context.cleanup_modified_files()
456+
# test_context.cleanup_modified_files()
423457

424458

425459
if __name__ == "__main__":

‎tests/context/plan_and_run/test_model_code_change.py

Lines changed: 67 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
import pytest
44

5-
from dagster_sqlmesh.controller.base import PlanOptions, RunOptions
5+
from dagster_sqlmesh.controller.base import PlanOptions
66
from tests.conftest import SQLMeshTestContext
77

88
logger = logging.getLogger(__name__)
@@ -12,7 +12,7 @@
1212
"no_auto_upstream,skip_backfill,expected_changes",
1313
[
1414
(
15-
True,
15+
False,
1616
False,
1717
{
1818
"staging_1": ">=", # Should increase (skip_backfill disabled)
@@ -26,6 +26,7 @@
2626
"only_skip_backfill",
2727
],
2828
)
29+
@pytest.mark.skip(reason="Work in progress test")
2930
def test_given_model_chain_when_running_with_different_flags_then_behaves_as_expected(
3031
model_change_test_context: SQLMeshTestContext,
3132
no_auto_upstream: bool,
@@ -54,85 +55,80 @@ def test_given_model_chain_when_running_with_different_flags_then_behaves_as_exp
5455
">=" means final count should be greater than or equal to initial
5556
"""
5657
# Initial run to set up all models
57-
model_change_test_context.plan_and_run(
58-
environment="dev",
59-
start="2023-02-01",
60-
end="2023-02-03",
61-
plan_options=PlanOptions(
62-
execution_time="2023-02-03",
63-
),
64-
run_options=RunOptions(
65-
execution_time="2023-02-03",
66-
),
67-
)
58+
# model_change_test_context.plan_and_run(
59+
# environment="dev",
60+
# start="2023-02-01",
61+
# end="2023-02-03",
62+
# plan_options=PlanOptions(
63+
# execution_time="2023-02-03",
64+
# ),
65+
# run_options=RunOptions(
66+
# execution_time="2023-02-03",
67+
# ),
68+
# )
6869

69-
# Get initial counts for the model chain
70-
initial_counts = {
71-
"staging_1": model_change_test_context.query(
72-
"SELECT COUNT(*) FROM sqlmesh_example__dev.staging_model_1"
73-
)[0][0],
74-
"staging_2": model_change_test_context.query(
75-
"SELECT COUNT(*) FROM sqlmesh_example__dev.staging_model_2"
76-
)[0][0],
77-
"intermediate": model_change_test_context.query(
78-
"SELECT COUNT(*) FROM sqlmesh_example__dev.intermediate_model_1"
79-
)[0][0],
80-
"full": model_change_test_context.query(
81-
"SELECT COUNT(*) FROM sqlmesh_example__dev.full_model"
82-
)[0][0],
83-
}
70+
# # Get initial counts for the model chain
71+
# initial_counts = {
72+
# "staging_1": model_change_test_context.query(
73+
# "SELECT COUNT(*) FROM sqlmesh_example__dev.staging_model_1"
74+
# )[0][0],
75+
# "staging_2": model_change_test_context.query(
76+
# "SELECT COUNT(*) FROM sqlmesh_example__dev.staging_model_2"
77+
# )[0][0],
78+
# "intermediate": model_change_test_context.query(
79+
# "SELECT COUNT(*) FROM sqlmesh_example__dev.intermediate_model_1"
80+
# )[0][0],
81+
# "full": model_change_test_context.query(
82+
# "SELECT COUNT(*) FROM sqlmesh_example__dev.full_model"
83+
# )[0][0],
84+
# }
85+
86+
# print(f"initial_counts: {initial_counts}")
87+
# print(
88+
# f"intermediate_model_1 first run: {
89+
# model_change_test_context.query(
90+
# 'SELECT * FROM sqlmesh_example__dev.intermediate_model_1',
91+
# return_df=True,
92+
# )
93+
# }"
94+
# )
8495

85-
print(f"initial_counts: {initial_counts}")
86-
print(
87-
f"intermediate_model_1 first run: {
88-
model_change_test_context.query(
89-
'SELECT * FROM sqlmesh_example__dev.intermediate_model_1',
90-
return_df=True,
91-
)
92-
}"
93-
)
96+
# # Modify staging_model_1 to include more data
97+
# model_change_test_context.modify_model_file(
98+
# "intermediate_model_1.sql",
99+
# """
100+
# MODEL (
101+
# name sqlmesh_example.intermediate_model_1,
102+
# kind INCREMENTAL_BY_TIME_RANGE (
103+
# time_column event_date
104+
# ),
105+
# start '2020-01-01',
106+
# cron '@daily',
107+
# grain (id, event_date)
108+
# );
94109

95-
# Modify staging_model_1 to include more data
96-
model_change_test_context.modify_model_file(
97-
"intermediate_model_1.sql",
98-
"""
99-
MODEL (
100-
name sqlmesh_example.intermediate_model_1,
101-
kind INCREMENTAL_BY_TIME_RANGE (
102-
time_column event_date
103-
),
104-
start '2020-01-01',
105-
cron '@daily',
106-
grain (id, event_date)
107-
);
108-
109-
SELECT
110-
main.id,
111-
main.item_id,
112-
main.event_date,
113-
CONCAT(sub.item_name, ' - modified') as item_name
114-
FROM sqlmesh_example.staging_model_1 AS main
115-
INNER JOIN sqlmesh_example.staging_model_2 as sub
116-
ON main.id = sub.id
117-
WHERE
118-
event_date BETWEEN @start_date AND @end_date
119-
120-
""",
121-
)
110+
# SELECT
111+
# main.id,
112+
# main.item_id,
113+
# main.event_date,
114+
# CONCAT(sub.item_name, ' - modified18') as item_name
115+
# FROM sqlmesh_example.staging_model_1 AS main
116+
# INNER JOIN sqlmesh_example.staging_model_2 as sub
117+
# ON main.id = sub.id
118+
# WHERE
119+
# event_date BETWEEN @start_date AND @end_date
120+
121+
# """,
122+
# )
123+
124+
# raise Exception("Stop here")
122125

123126
# Run with specified flags
124127
model_change_test_context.plan_and_run(
125128
environment="dev",
126-
start="2023-02-01",
127-
end="2023-02-03",
128129
plan_options=PlanOptions(
129130
skip_backfill=skip_backfill,
130131
enable_preview=True,
131-
execution_time="2023-02-03",
132-
),
133-
run_options=RunOptions(
134-
no_auto_upstream=no_auto_upstream,
135-
execution_time="2023-02-03",
136132
),
137133
select_models=["sqlmesh_example.intermediate_model_1"],
138134
)

0 commit comments

Comments
 (0)