Fix migrate tables when default catalog is set (#4012)

Fixes #3735

Author: FastLee

src/databricks/labs/ucx/hive_metastore/mapping.py

@@ -241,7 +241,9 @@ def _get_database_in_scope_task(self, database: str) -> str | None:
             return database
         describe = {}
         try:
-            for value in self._sql_backend.fetch(f"DESCRIBE SCHEMA EXTENDED {escape_sql_identifier(database)}"):
+            for value in self._sql_backend.fetch(
+                f"DESCRIBE SCHEMA EXTENDED hive_metastore.{escape_sql_identifier(database)}"
+            ):
                 describe[value["database_description_item"]] = value["database_description_value"]
         except NotFound:
             logger.warning(
@@ -290,10 +292,12 @@ def _get_table_in_scope_task(self, table_to_migrate: TableToMigrate, check_uc_ta
         return table_to_migrate
 
     def _get_table_properties(self, table: Table):
-        table_identifier = f"{escape_sql_identifier(table.database)}.{escape_sql_identifier(table.name)}"
         if table.is_table_in_mount:
             table_identifier = f"delta.`{table.location}`"
-
+        else:
+            table_identifier = (
+                f"hive_metastore.{escape_sql_identifier(table.database)}.{escape_sql_identifier(table.name)}"
+            )
         try:
             return self._sql_backend.fetch(f"SHOW TBLPROPERTIES {table_identifier}")
         except DatabricksError as err:

tests/unit/hive_metastore/test_mapping.py

@@ -330,18 +330,18 @@ def test_skip_tables_marked_for_skipping_or_upgraded():
             ["test_schema2"],
             ["test_schema3"],
         ],
-        "SHOW TBLPROPERTIES `test_schema1`.`test_table1`": [
+        "SHOW TBLPROPERTIES hive_metastore.`test_schema1`.`test_table1`": [
             {"key": "upgraded_to", "value": "fake_dest"},
         ],
-        "SHOW TBLPROPERTIES `test_schema1`.`test_view1`": [
+        "SHOW TBLPROPERTIES hive_metastore.`test_schema1`.`test_view1`": [
             {"key": "databricks.labs.ucx.skip", "value": "true"},
         ],
-        "SHOW TBLPROPERTIES `test_schema1`.`test_table2`": [
+        "SHOW TBLPROPERTIES hive_metastore.`test_schema1`.`test_table2`": [
             {"key": "upgraded_to", "value": "fake_dest"},
         ],
-        "DESCRIBE SCHEMA EXTENDED `test_schema1`": [],
-        "DESCRIBE SCHEMA EXTENDED `test_schema2`": [],
-        "DESCRIBE SCHEMA EXTENDED `test_schema3`": [
+        "DESCRIBE SCHEMA EXTENDED hive_metastore.`test_schema1`": [],
+        "DESCRIBE SCHEMA EXTENDED hive_metastore.`test_schema2`": [],
+        "DESCRIBE SCHEMA EXTENDED hive_metastore.`test_schema3`": [
             {
                 "database_description_item": "Properties",
                 "database_description_value": "((databricks.labs.ucx.skip,true))",
@@ -418,7 +418,7 @@ def test_skip_tables_marked_for_skipping_or_upgraded():
 def test_table_with_no_target_reverted():
     errors = {}
     rows = {
-        "SHOW TBLPROPERTIES `schema1`.`table1`": [
+        "SHOW TBLPROPERTIES hive_metastore.`schema1`.`table1`": [
             {"key": "upgraded_to", "value": "non.existing.table"},
         ],
     }
@@ -490,7 +490,7 @@ def test_skipping_rules_existing_targets():
     ]
     table_mapping.get_tables_to_migrate(tables_crawler)
 
-    assert ["DESCRIBE SCHEMA EXTENDED `schema1`"] == backend.queries
+    assert ["DESCRIBE SCHEMA EXTENDED hive_metastore.`schema1`"] == backend.queries
 
 
 def test_mismatch_from_table_raises_exception():
@@ -530,7 +530,7 @@ def test_mismatch_from_table_raises_exception():
             assert len(e.errs) == 1
             raise e.errs[0]
 
-    assert ["DESCRIBE SCHEMA EXTENDED `schema1`"] == backend.queries
+    assert ["DESCRIBE SCHEMA EXTENDED hive_metastore.`schema1`"] == backend.queries
 
 
 def test_table_not_in_crawled_tables():
@@ -548,7 +548,7 @@ def test_table_not_in_crawled_tables():
     tables_crawler.snapshot.return_value = []
     table_mapping.get_tables_to_migrate(tables_crawler)
 
-    assert ["DESCRIBE SCHEMA EXTENDED `schema1`"] == backend.queries
+    assert ["DESCRIBE SCHEMA EXTENDED hive_metastore.`schema1`"] == backend.queries
 
 
 def test_skipping_rules_database_skipped():
@@ -560,7 +560,7 @@ def test_skipping_rules_database_skipped():
     )
     errors = {}
     rows = {
-        "DESCRIBE SCHEMA EXTENDED `schema2`": [
+        "DESCRIBE SCHEMA EXTENDED hive_metastore.`schema2`": [
             {
                 "database_description_item": "Properties",
                 "database_description_value": "((databricks.labs.ucx.skip,true))",
@@ -594,8 +594,8 @@ def test_skipping_rules_database_skipped():
     ]
     table_mapping.get_tables_to_migrate(tables_crawler)
 
-    assert "SHOW TBLPROPERTIES `schema1`.`table1`" in backend.queries
-    assert "SHOW TBLPROPERTIES `schema2`.`table2`" not in backend.queries
+    assert "SHOW TBLPROPERTIES hive_metastore.`schema1`.`table1`" in backend.queries
+    assert "SHOW TBLPROPERTIES hive_metastore.`schema2`.`table2`" not in backend.queries
 
 
 def test_skip_missing_table_in_snapshot():
@@ -754,7 +754,7 @@ def test_tables_in_mounts():
     tables_crawler.snapshot.return_value = []
     table_mapping.get_tables_to_migrate(tables_crawler)
 
-    assert ["DESCRIBE SCHEMA EXTENDED `schema1`"] == backend.queries
+    assert ["DESCRIBE SCHEMA EXTENDED hive_metastore.`schema1`"] == backend.queries
 
 
 def test_mapping_table_in_mount():
@@ -913,7 +913,7 @@ def test_mapping_broken_table(caplog):
 def test_table_with_no_target_reverted_failed(caplog):
     errors = {"ALTER TABLE": "ALTER_TABLE_FAILED"}
     rows = {
-        "SHOW TBLPROPERTIES `schema1`.`table1`": [
+        "SHOW TBLPROPERTIES hive_metastore.`schema1`.`table1`": [
             {"key": "upgraded_to", "value": "non.existing.table"},
         ],
     }

tests/unit/recon/test_migration_recon.py

@@ -42,8 +42,12 @@ def test_migrate_recon_should_produce_proper_queries(
             (src_3.schema, src_3.table, tgt_3.catalog, tgt_3.schema, tgt_3.table, "2021-01-01T00:00:00Z"),
             ("schema_none", "table_none", None, "schema_a", "table_a", "2021-01-01T00:00:00Z"),
         ],
-        f"SHOW TBLPROPERTIES {src.schema}.{src.table} \\('upgraded_to'\\)": UPGRADED_TO[("value", "fake_dest"),],
-        f"SHOW TBLPROPERTIES {src_2.schema}.{src_2.table} \\('upgraded_to'\\)": UPGRADED_TO[("value", "fake_dest"),],
+        f"SHOW TBLPROPERTIES hive_metastore.{src.schema}.{src.table} \\('upgraded_to'\\)": UPGRADED_TO[
+            ("value", "fake_dest"),
+        ],
+        f"SHOW TBLPROPERTIES hive_metastore.{src_2.schema}.{src_2.table} \\('upgraded_to'\\)": UPGRADED_TO[
+            ("value", "fake_dest"),
+        ],
         "DESCRIBE TABLE": metadata_row_factory[
             ("col1", "int"),
             ("col2", "string"),