Skip to content

Commit 651184a

Browse files
author
Dat Nguyen
committed
chore: update docs
1 parent 5747778 commit 651184a

File tree

4 files changed

+24
-15
lines changed

4 files changed

+24
-15
lines changed

‎README.md

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,14 +4,17 @@
44
<img align="right" width="150" height="150" src="./docs/assets/img/il-logo.png">
55

66
[![dbt-hub](https://img.shields.io/badge/Visit-dbt--hub%20↗️-FF694B?logo=dbt&logoColor=FF694B)](https://hub.getdbt.com/infinitelambda/dbt-data-diff)
7-
[![support-snowflake](https://img.shields.io/badge/support-Snowflake-7faecd?logo=snowflake&logoColor=7faecd)](https://docs.snowflake.com/)
8-
[![support-dbt](https://img.shields.io/badge/support-dbt%20v1.6+-FF694B?logo=dbt&logoColor=FF694B)](https://docs.getdbt.com/)
7+
[![support-snowflake](https://img.shields.io/badge/support-Snowflake-7faecd?logo=snowflake&logoColor=7faecd)](https://docs.snowflake.com?ref=infinitelambda)
8+
[![support-dbt](https://img.shields.io/badge/support-dbt%20v1.6+-FF694B?logo=dbt&logoColor=FF694B)](https://docs.getdbt.com?ref=infinitelambda)
9+
[![built-in-sis](https://img.shields.io/badge/built--in-SiS-BD4042?logo=streamlit&logoColor=FF694B)](https://www.snowflake.com/en/data-cloud/overview/streamlit-in-snowflake?ref=infinitelambda)
910

1011
Data-diff solution for dbt-ers with Snowflake ❄️ 🚀
1112

1213
> [!TIP]
1314
> 📖 For more details, please help to visit [the documentation site](https://data-diff.iflambda.com/latest/) (or go to the [docs/index.md](./docs/index.md)) for more details
1415
16+
<img src="./docs/assets/img/data-diff.jpeg" alt="Sample diffing">
17+
1518
## Installation
1619

1720
- Add to `packages.yml` file:
@@ -51,6 +54,10 @@ dbt run -s data_diff --vars '{data_diff__on_migration: true}'
5154

5255
Let's jump to the [Quick Start](https://data-diff.iflambda.com/latest/#quick-start) section and the next [demo one](https://data-diff.iflambda.com/latest/#demo) 🏃
5356

57+
📊 Here is the sample Streamlit in Snowflake application based on the result produced by the package:
58+
59+
<img src="./docs/assets/img/sis_ui.png" alt="Sample SiS">
60+
5461
## How to Contribute
5562

5663
`dbt-data-diff` is an open-source dbt package. Whether you are a seasoned open-source contributor or a first-time committer, we welcome and encourage you to contribute code, documentation, ideas, or problem statements to this project.

‎docs/assets/img/sis_ui.png

45.5 KB
Loading

‎docs/index.md

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,10 @@
33

44
<img align="right" width="150" height="150" src="./assets/img/il-logo.png">
55

6-
[![dbt-hub](https://img.shields.io/badge/Visit-dbt--hub%20↗️-FF694B?logo=dbt&logoColor=FF694B)](https://hub.getdbt.com/infinitelambda/dbt-data-diff)[![support-snowflake](https://img.shields.io/badge/support-Snowflake-7faecd?logo=snowflake&logoColor=7faecd)](https://docs.snowflake.com/)
7-
[![support-dbt](https://img.shields.io/badge/support-dbt%20v1.6+-FF694B?logo=dbt&logoColor=FF694B)](https://docs.getdbt.com/)
6+
[![dbt-hub](https://img.shields.io/badge/Visit-dbt--hub%20↗️-FF694B?logo=dbt&logoColor=FF694B)](https://hub.getdbt.com/infinitelambda/dbt-data-diff)
7+
[![support-snowflake](https://img.shields.io/badge/support-Snowflake-7faecd?logo=snowflake&logoColor=7faecd)](https://docs.snowflake.com?ref=infinitelambda)
8+
[![support-dbt](https://img.shields.io/badge/support-dbt%20v1.6+-FF694B?logo=dbt&logoColor=FF694B)](https://docs.getdbt.com?ref=infinitelambda)
9+
[![built-in-sis](https://img.shields.io/badge/built--in-SiS-BD4042?logo=streamlit&logoColor=FF694B)](https://www.snowflake.com/en/data-cloud/overview/streamlit-in-snowflake?ref=infinitelambda)
810

911
Data-diff solution for dbt-ers with Snowflake ❄️ 🌟
1012

‎macros/resources/stored-procedures/create__check_data_diff.sql

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -77,32 +77,32 @@
7777
qualify row_number() over(
7878
partition by src_db, src_schema, src_table, trg_db, trg_schema, trg_table, column_name, pipe_name
7979
order by last_data_diff_timestamp desc
80-
) = 1
80+
) = 1 --get last schema diff result
8181

8282
),
8383

8484
base as (
8585

8686
select t.*
87-
,listagg(column_name, ',') as col_list
87+
,listagg(v.column_name, ',') as col_list
8888
,'cast(md5_binary(concat_ws(''||'','
89-
|| listagg('ifnull(nullif(upper(trim(cast(' || column_name || ' as varchar))), ''''), ''^^'')', ',' )
89+
|| listagg('ifnull(nullif(upper(trim(cast(' || v.column_name || ' as varchar))), ''''), ''^^'')', ',' )
9090
|| ' )) as binary(16)) as hashdiff' as hash_calc
91-
,listagg('ifnull(nullif(upper(trim(cast(src.'|| column_name ||' as varchar))),''''),''^^'')= ifnull(nullif(upper(trim(cast(trg.'|| column_name ||' as varchar))),''''),''^^'') as '|| column_name || '_is_equal', ',' ) as is_equal
92-
,listagg('sum(case when '|| column_name ||'_is_equal then 1 else 0 end) as '|| column_name || '_diff', ',' ) as diff_calc
93-
,listagg(column_name ||'_diff / cnt as '|| column_name, ',') as result_calc
91+
,listagg('ifnull(nullif(upper(trim(cast(src.'|| v.column_name ||' as varchar))),''''),''^^'')= ifnull(nullif(upper(trim(cast(trg.'|| v.column_name ||' as varchar))),''''),''^^'') as '|| v.column_name || '_is_equal', ',' ) as is_equal
92+
,listagg('sum(case when '|| v.column_name ||'_is_equal then 1 else 0 end) as '|| v.column_name || '_diff', ',' ) as diff_calc
93+
,listagg(v.column_name ||'_diff / cnt as '|| v.column_name, ',') as result_calc
9494

95-
from {{ configured_table_model }}_final t
96-
join schema_validation v
95+
from {{ configured_table_model }}_final as t
96+
join schema_validation as v
9797
on t.src_schema = v.src_schema
9898
and t.src_table = v.src_table
9999
where true
100100
--excluded columns i.e always changing column, added or removed column
101-
and (not array_contains(upper(column_name)::variant, t.exclude_columns))
101+
and (not array_contains(upper(v.column_name)::variant, t.exclude_columns))
102102
and (
103103
case
104-
when array_size(include_columns) > 0
105-
then array_contains(column_name::variant, t.include_columns)
104+
when array_size(t.include_columns) > 0
105+
then array_contains(v.column_name::variant, t.include_columns)
106106
else true
107107
end
108108
)

0 commit comments

Comments
 (0)