import copy
import datetime
import json
import logging
import re
from dataclasses import dataclass, field as dataclass_field
from functools import lru_cache
from typing import (
    Any,
    Dict,
    Iterable,
    Iterator,
    List,
    Literal,
    Mapping,
    Optional,
    Set,
    Tuple,
)
from urllib.parse import urlparse

import botocore.exceptions
import botocore.session
import yaml
from pydantic import field_validator, model_validator
from pydantic.fields import Field

from datahub.api.entities.dataset.dataset import Dataset
from datahub.api.entities.external.lake_formation_external_entites import (
    LakeFormationTag,
)
from datahub.configuration.common import AllowDenyPattern, ConfigModel
from datahub.configuration.source_common import DatasetSourceConfigMixin
from datahub.configuration.validate_field_rename import pydantic_renamed_field
from datahub.emitter import mce_builder
from datahub.emitter.mce_builder import (
    get_sys_time,
    make_data_platform_urn,
    make_dataplatform_instance_urn,
    make_dataset_urn_with_platform_instance,
    make_domain_urn,
    make_tag_urn,
    make_ts_millis,
)
from datahub.emitter.mcp import MetadataChangeProposalWrapper
from datahub.emitter.mcp_builder import (
    DatabaseKey,
    add_dataset_to_container,
    add_domain_to_entity_wu,
    gen_containers,
)
from datahub.emitter.rest_emitter import EmitMode
from datahub.ingestion.api.common import PipelineContext
from datahub.ingestion.api.decorators import (
    SourceCapability,
    SupportStatus,
    capability,
    config_class,
    platform_name,
    support_status,
)
from datahub.ingestion.api.incremental_properties_helper import (
    IncrementalPropertiesConfigMixin,
)
from datahub.ingestion.api.report import EntityFilterReport
from datahub.ingestion.api.workunit import MetadataWorkUnit
from datahub.ingestion.source.aws import s3_util
from datahub.ingestion.source.aws.aws_common import AwsSourceConfig
from datahub.ingestion.source.aws.platform_resource_repository import (
    GluePlatformResourceRepository,
)
from datahub.ingestion.source.aws.s3_util import (
    is_s3_uri,
    make_s3_urn,
    make_s3_urn_for_lineage,
)
from datahub.ingestion.source.aws.tag_entities import (
    LakeFormationTagPlatformResourceId,
)
from datahub.ingestion.source.common.presto_view_decoder import (
    decode_presto_view,
    is_presto_view,
)
from datahub.ingestion.source.common.subtypes import (
    DatasetContainerSubTypes,
    DatasetSubTypes,
    FlowContainerSubTypes,
    SourceCapabilityModifier,
)
from datahub.ingestion.source.glue_profiling_config import GlueProfilingConfig
from datahub.ingestion.source.state.stale_entity_removal_handler import (
    StaleEntityRemovalSourceReport,
    StatefulStaleMetadataRemovalConfig,
)
from datahub.ingestion.source.state.stateful_ingestion_base import (
    StatefulIngestionConfigBase,
    StatefulIngestionSourceBase,
)
from datahub.ingestion.source_config.operation_config import is_profiling_enabled
from datahub.metadata.com.linkedin.pegasus2avro.common import Status, SubTypes
from datahub.metadata.com.linkedin.pegasus2avro.metadata.snapshot import DatasetSnapshot
from datahub.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent
from datahub.metadata.com.linkedin.pegasus2avro.schema import (
    MySqlDDL,
    SchemaField,
    SchemaMetadata,
)
from datahub.metadata.schema_classes import (
    DataFlowInfoClass,
    DataFlowSnapshotClass,
    DataJobInfoClass,
    DataJobInputOutputClass,
    DataJobSnapshotClass,
    DataPlatformInstanceClass,
    DatasetFieldProfileClass,
    DatasetLineageTypeClass,
    DatasetProfileClass,
    DatasetPropertiesClass,
    DataTransformClass,
    DataTransformLogicClass,
    FineGrainedLineageClass,
    FineGrainedLineageDownstreamTypeClass,
    FineGrainedLineageUpstreamTypeClass,
    GlobalTagsClass,
    MetadataAttributionClass,
    MetadataChangeEventClass,
    OperationClass,
    OperationTypeClass,
    OwnerClass,
    OwnershipClass,
    OwnershipTypeClass,
    PartitionSpecClass,
    PartitionTypeClass,
    QueryLanguageClass,
    QueryStatementClass,
    SchemaMetadataClass,
    StructuredPropertiesClass,
    StructuredPropertyDefinitionClass,
    StructuredPropertyValueAssignmentClass,
    TagAssociationClass,
    TimeStampClass,
    UpstreamClass,
    UpstreamLineageClass,
    ViewPropertiesClass,
)
from datahub.sql_parsing.sql_parsing_aggregator import (
    SqlAggregatorReport,
    SqlParsingAggregator,
)
from datahub.sql_parsing.sqlglot_lineage import create_lineage_sql_parsed_result
from datahub.utilities.delta import delta_type_to_hive_type
from datahub.utilities.hive_schema_to_avro import get_schema_fields_for_hive_column
from datahub.utilities.lossy_collections import LossyList
from datahub.utilities.urns.error import InvalidUrnError

logger = logging.getLogger(__name__)

DEFAULT_PLATFORM = "glue"
VALID_PLATFORMS = [DEFAULT_PLATFORM, "athena"]

GLUE_TABLE_TYPE_ICEBERG = "ICEBERG"

# Structural shape of a Glue catalog ARN authority: arn:{partition}:glue:{region}:{account}. The
# partition/region agreement is validated separately (check_catalog_arn_keys) against botocore's
# authoritative partition data, so the shape here only pins the segments and the 12-digit account.
GLUE_CATALOG_ARN_PATTERN = re.compile(r"^arn:aws[a-z-]*:glue:([^:\s]+):\d{12}$")


@lru_cache(maxsize=None)
def _aws_partition_for_region(region: str) -> str:
    """Authoritative ARN partition for an AWS region (aws, aws-us-gov, aws-cn, aws-iso*, ...).

    Delegates to botocore's shipped endpoint data rather than a hand-maintained region-prefix table,
    so it stays correct as AWS adds partitions. Falls back to the commercial partition for regions
    botocore doesn't recognize (e.g. a not-yet-published region).
    """
    try:
        return botocore.session.get_session().get_partition_for_region(region)
    except botocore.exceptions.UnknownRegionError:
        # Only swallow UnknownRegionError, so a genuine botocore API change surfaces instead of
        # silently degrading every region to the commercial partition.
        return "aws"


JDBC_PLATFORM_MAP: Dict[str, str] = {
    "postgresql": "postgres",
    "mysql": "mysql",
    "mariadb": "mysql",
    "redshift": "redshift",
    "oracle": "oracle",
    "sqlserver": "mssql",
}

JDBC_DEFAULT_SCHEMA: Dict[str, str] = {
    "postgres": "public",
    "redshift": "public",
    "mssql": "dbo",
}

GLUE_NATIVE_CONNECTION_TYPE_MAP: Dict[str, str] = {
    "POSTGRESQL": "postgres",
    "MYSQL": "mysql",
    "REDSHIFT": "redshift",
    "ORACLE": "oracle",
    "SQLSERVER": "mssql",
}

JDBC_PREFIX = "jdbc:"


def _sanitize_jdbc_url(jdbc_url: str) -> str:
    """Strip credentials and query parameters from a JDBC URL for safe logging."""
    inner = (
        jdbc_url[len(JDBC_PREFIX) :] if jdbc_url.startswith(JDBC_PREFIX) else jdbc_url
    )
    parsed = urlparse(inner)
    safe_netloc = parsed.hostname or ""
    if parsed.port:
        safe_netloc = f"{safe_netloc}:{parsed.port}"
    return f"{JDBC_PREFIX}{parsed.scheme}://{safe_netloc}{parsed.path}"


class TargetPlatformConfig(ConfigModel):
    """Config for aligning dataset URNs with a separately ingested platform."""

    platform_instance: Optional[str] = Field(
        default=None,
        description="Platform instance used by the separate ingestion of this platform.",
    )
    env: Optional[str] = Field(
        default=None,
        description="Environment used by the separate ingestion of this platform. Defaults to the Glue source env.",
    )


@dataclass
class ResolvedTarget:
    """The platform_instance/env a table's URN is stamped with, after catalog resolution.

    Distinct from the ``TargetPlatformConfig`` *input* type: ``env`` is always resolved here (never
    ``None``), so call sites don't need to re-apply the source-env fallback.
    """

    platform_instance: Optional[str]
    env: str


@dataclass
class OwnerSchemaFetch:
    """Outcome of one attempt to read a resource link's owning-table schema.

    ``warned`` is True when the attempt already reported its own (exception-bearing) warning, so the
    caller can avoid adding a redundant generic warning for the same unresolved owner.
    """

    schema: Optional[SchemaMetadata]
    warned: bool


class GlueSourceConfig(
    StatefulIngestionConfigBase,
    DatasetSourceConfigMixin,
    AwsSourceConfig,
    IncrementalPropertiesConfigMixin,
):
    platform: str = Field(
        default=DEFAULT_PLATFORM,
        description=f"The platform to use for the dataset URNs. Must be one of {VALID_PLATFORMS}.",
    )

    # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-glue-table-tableinput.html#cfn-glue-table-tableinput-owner
    extract_owners: Optional[bool] = Field(
        default=True,
        description="When enabled, extracts ownership from Glue table property and overwrites existing owners (DATAOWNER). When disabled, ownership is left empty for datasets. Expects a corpGroup urn, a corpuser urn or only the identifier part for the latter. Not used in the normal course of AWS Glue operations.",
    )
    extract_transforms: Optional[bool] = Field(
        default=True, description="Whether to extract Glue transform jobs."
    )
    ignore_unsupported_connectors: Optional[bool] = Field(
        default=True,
        description="Whether to ignore unsupported connectors. If disabled, an error will be raised.",
    )
    emit_storage_lineage: bool = Field(
        default=False,
        description="Whether to emit storage-to-Glue lineage. When enabled, creates lineage relationships between Glue tables and their underlying storage locations (S3 or Iceberg).",
    )
    _rename_emit_s3_lineage = pydantic_renamed_field(
        "emit_s3_lineage",
        "emit_storage_lineage",
    )
    glue_storage_lineage_direction: Literal["upstream", "downstream"] = Field(
        default="upstream",
        description="If `upstream`, storage locations are upstream to Glue. If `downstream`, they are downstream to Glue.",
    )
    _rename_glue_s3_lineage_direction = pydantic_renamed_field(
        "glue_s3_lineage_direction",
        "glue_storage_lineage_direction",
    )
    domain: Dict[str, AllowDenyPattern] = Field(
        default=dict(),
        description="regex patterns for tables to filter to assign domain_key. ",
    )
    catalog_id: Optional[str] = Field(
        default=None,
        description="The aws account id where the target glue catalog lives. If None, datahub will ingest glue in aws caller's account.",
    )
    ignore_resource_links: Optional[bool] = Field(
        default=False,
        description="If set to True, ignore database resource links.",
    )
    use_s3_bucket_tags: Optional[bool] = Field(
        default=False,
        description="If an S3 Buckets Tags should be created for the Tables ingested by Glue. Please Note that this will not apply tags to any folders ingested, only the files.",
    )
    use_s3_object_tags: Optional[bool] = Field(
        default=False,
        description="If an S3 Objects Tags should be created for the Tables ingested by Glue.",
    )

    extract_lakeformation_tags: Optional[bool] = Field(
        default=False,
        description="When True, extracts Lake Formation tags directly assigned to Glue tables/databases. Tags inherited from databases are excluded by AWS from the per-table response; enable `propagate_lakeformation_tags` to apply database tags to their tables as well.",
    )

    propagate_lakeformation_tags: bool = Field(
        default=False,
        description="When True (requires `extract_lakeformation_tags`), Lake Formation tags assigned to a Glue database are inherited by every table in that database, and (when `extract_lakeformation_column_tags` is also enabled) by every column as well — matching Lake Formation's own inheritance. AWS returns the database's tags alongside each table's own tags, so no extra API calls are made. A more-specific assignment overrides an inherited value for the same key (table over database, column over table/database).",
    )

    extract_lakeformation_column_tags: bool = Field(
        default=True,
        description="When True (requires `extract_lakeformation_tags`), Lake Formation tags assigned directly to columns are applied to the corresponding schema fields. If `propagate_lakeformation_tags` is also enabled, the table's and database's tags are inherited by every column as well (matching Lake Formation's inheritance), with a column's own tag taking precedence for the same key.",
    )

    profiling: GlueProfilingConfig = Field(
        default_factory=GlueProfilingConfig,
        description="Configs to ingest data profiles from glue table",
    )
    # Custom Stateful Ingestion settings
    stateful_ingestion: Optional[StatefulStaleMetadataRemovalConfig] = Field(
        default=None, description=""
    )

    extract_delta_schema_from_parameters: Optional[bool] = Field(
        default=False,
        description="If enabled, delta schemas can be alternatively fetched from table parameters.",
    )

    include_column_lineage: bool = Field(
        default=True,
        description="When enabled, column-level lineage will be extracted between Glue table columns and storage location fields.",
    )

    resolve_resource_link_schema: bool = Field(
        default=True,
        description=(
            "When enabled, populate a Lake Formation resource link's schema so its columns are "
            "visible on the ingested dataset — a resource link is schemaless in the catalog because "
            "it is only a pointer to the owning account's table. The owning table's schema is read "
            "from DataHub first (when that account has already been ingested, so no AWS call is "
            "made) and otherwise from a cross-account `glue:GetTable` on the target (requires "
            "`glue:GetTable` permission on the shared table). When neither is available the link is "
            "left schemaless and its columns remain reachable through its upstream lineage edge. "
            "Set to False to always keep resource links schemaless."
        ),
    )

    include_view_lineage: bool = Field(
        default=True,
        description=(
            "When enabled, the SQL definition of each Glue view (TableType "
            "VIRTUAL_VIEW, e.g. Athena/Presto views) is parsed to extract upstream "
            "table/view lineage, including column-level lineage. Referenced tables "
            "must also be ingested for the lineage to resolve to DataHub entities."
        ),
    )

    extract_column_parameters: bool = Field(
        default=False,
        description=(
            "When enabled, column-level Parameters from Glue are ingested as structured properties "
            "on each schemaField entity. A StructuredPropertyDefinition is upserted once per unique "
            "parameter key per recipe run; subsequent columns reuse the cached definition."
        ),
    )

    target_platform_configs: Dict[str, TargetPlatformConfig] = Field(
        default_factory=dict,
        description=(
            "Optional per-platform config for aligning dataset URNs with separately ingested platforms. "
            "Keys are DataHub platform names (e.g. 'postgres', 'mysql', 'redshift'). "
            "When provided, the platform_instance and env are applied to dataset URNs so they match "
            "the URNs produced by the platform's own connector. "
            "Only needed when the target platform's connector uses a platform_instance or a different env."
        ),
    )

    catalog_to_platform_instance: Dict[str, TargetPlatformConfig] = Field(
        default_factory=dict,
        description=(
            "Maps a Glue catalog's ARN authority `arn:{partition}:glue:{region}:{account}` to the "
            "platform_instance (and optionally env) to stamp on tables owned by that catalog. "
            "Use this for cross-account catalogs (e.g. ingesting another account via `catalog_id`, "
            "or Lake Formation shared tables) so each table's URN matches the one the owning "
            "account's own Glue ingestion produces, instead of the ingestion account's instance. "
            "The key is account + region — account alone is not unique across regions — and matches "
            "the key used by the Spark/OpenLineage `connections` map. The partition must match the "
            "region as resolved by boto3 — `aws` for commercial, `aws-us-gov` for GovCloud, "
            "`aws-cn` for China, and the `aws-iso*` partitions for isolated regions."
        ),
    )

    def is_profiling_enabled(self) -> bool:
        return self.profiling.enabled and is_profiling_enabled(
            self.profiling.operation_config
        )

    @property
    def glue_client(self):
        return self.get_glue_client()

    @property
    def s3_client(self):
        return self.get_s3_client()

    @property
    def lakeformation_client(self):
        return self.get_lakeformation_client()

    @field_validator("glue_storage_lineage_direction", mode="after")
    @classmethod
    def check_direction(cls, v: str) -> str:
        if v.lower() not in ["upstream", "downstream"]:
            raise ValueError(
                "glue_storage_lineage_direction must be either upstream or downstream"
            )
        return v.lower()

    @field_validator("catalog_to_platform_instance")
    @classmethod
    def check_catalog_arn_keys(
        cls, v: Dict[str, TargetPlatformConfig]
    ) -> Dict[str, TargetPlatformConfig]:
        invalid = []
        for key in v:
            match = GLUE_CATALOG_ARN_PATTERN.match(key)
            # Reject a bad shape, or a partition that disagrees with its region — the lookup derives
            # the partition from the region, so a mismatched key would validate but never match.
            if not match or not key.startswith(
                f"arn:{_aws_partition_for_region(match.group(1))}:glue:"
            ):
                invalid.append(key)
        if invalid:
            raise ValueError(
                "catalog_to_platform_instance keys must be Glue catalog ARN authorities of the "
                "form arn:{partition}:glue:{region}:{account} (account + region), where the "
                "partition matches the region as resolved by boto3 (commercial=aws, "
                "GovCloud=aws-us-gov, China=aws-cn, isolated regions=aws-iso*); "
                f"invalid keys: {invalid}"
            )
        return v

    @field_validator("platform", mode="after")
    @classmethod
    def platform_validator(cls, v: str) -> str:
        if not v or v in VALID_PLATFORMS:
            return v
        else:
            raise ValueError(
                f"'platform' can only take following values: {VALID_PLATFORMS}"
            )

    @model_validator(mode="after")
    def warn_lakeformation_subflags_without_parent(self) -> "GlueSourceConfig":
        if self.propagate_lakeformation_tags and not self.extract_lakeformation_tags:
            logger.warning(
                "propagate_lakeformation_tags has no effect "
                "unless extract_lakeformation_tags is enabled."
            )
        return self


@dataclass
class GlueSourceReport(StaleEntityRemovalSourceReport):
    catalog_id: Optional[str] = None
    tables_scanned = 0
    views_scanned = 0
    filtered: LossyList[str] = dataclass_field(default_factory=LossyList)
    databases: EntityFilterReport = EntityFilterReport.field(type="database")

    # Surfaces SQL-parsing metrics/warnings (e.g. view definitions that failed to
    # parse) when view lineage is enabled; None otherwise.
    sql_aggregator: Optional[SqlAggregatorReport] = None

    num_job_script_location_missing: int = 0
    num_job_script_location_invalid: int = 0
    num_job_script_failed_download: int = 0
    num_job_script_failed_parsing: int = 0
    num_job_without_nodes: int = 0
    num_dataset_to_dataset_edges_in_job: int = 0
    num_dataset_invalid_delta_schema: int = 0
    num_dataset_valid_delta_schema: int = 0
    num_resource_link_missing_target: int = 0
    num_resource_link_self_referential: int = 0
    num_resource_link_schema_from_datahub: int = 0
    num_resource_link_schema_from_glue: int = 0
    num_resource_link_schema_unresolved: int = 0

    def report_table_scanned(self) -> None:
        self.tables_scanned += 1

    def report_view_scanned(self) -> None:
        self.views_scanned += 1

    def report_table_dropped(self, table: str) -> None:
        self.filtered.append(table)


@dataclass
class TableLakeFormationTags:
    """Lake Formation tags returned for a Glue table in a single GetResourceLFTags call.

    AWS returns all three layers in one response (with ``ShowAssignedLFTags=True``):
    the database's tags, the table's directly-assigned tags, and each column's
    directly-assigned tags. This lets us propagate database tags to tables and
    apply column tags without any extra API calls.
    """

    database_tags: List[LakeFormationTag]
    table_tags: List[LakeFormationTag]
    column_tags: Dict[str, List[LakeFormationTag]]


@dataclass
class ResolvedLakeFormationTags:
    """LF tags resolved for an entity, split by provenance.

    ``direct`` tags are assigned directly to the entity; ``propagated`` tags are
    inherited from a parent (database for tables, table/database for columns) and
    are marked with propagation attribution when emitted.
    """

    direct: List[LakeFormationTag]
    propagated: List[LakeFormationTag]


@dataclass
class GlueViewDefinition:
    """SQL definition of a Glue view and the SQL dialect it is written in.

    Glue catalogs hold views from two engines: Athena/Presto views (base64-encoded
    Trino SQL) and raw Spark/Hive views (Spark SQL). Lineage parsing must use the
    matching dialect, so we carry it alongside the SQL.
    """

    sql: str
    dialect: str


@platform_name("Glue")
@config_class(GlueSourceConfig)
@support_status(SupportStatus.CERTIFIED)
@capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default")
@capability(SourceCapability.DOMAINS, "Supported via the `domain` config field")
@capability(
    SourceCapability.DELETION_DETECTION,
    "Enabled by default via stateful ingestion.",
)
@capability(SourceCapability.LINEAGE_COARSE, "Enabled by default")
@capability(
    SourceCapability.LINEAGE_FINE, "Support via the `emit_storage_lineage` config field"
)
@capability(
    SourceCapability.OPERATION_CAPTURE,
    "Enabled by default from Glue table created and last modified timestamps",
)
@capability(
    SourceCapability.CONTAINERS,
    "Enabled by default",
    subtype_modifier=[
        SourceCapabilityModifier.DATABASE,
    ],
)
class GlueSource(StatefulIngestionSourceBase):
    """
    Source that extracts tables, databases, and jobs from AWS Glue Data Catalog.

    Implementation notes:
    - Uses boto3 Glue client to fetch metadata
    - Supports cross-account access via catalog_id parameter
    - Job lineage extraction requires Glue Studio scripts with proper annotations
    - Caches LF tags to reduce API calls
    """

    source_config: GlueSourceConfig
    report: GlueSourceReport

    lf_tag_cache: Dict[str, Dict[str, List[str]]] = {}

    # Glue marks views (Athena/Presto or Spark/Hive views registered in the
    # catalog) with this TableType. Everything else is a table.
    _VIRTUAL_VIEW_TABLE_TYPE = "VIRTUAL_VIEW"

    # sqlglot dialects for the two kinds of Glue view SQL. Presto/Athena views are
    # Trino SQL; raw views are authored by Spark (Glue ETL/EMR) or Hive. sqlglot's
    # "spark" dialect extends "hive", so it parses both Spark and Hive SQL — the
    # safer default for raw views than "hive" alone.
    _PRESTO_VIEW_DIALECT = "trino"
    _RAW_VIEW_DIALECT = "spark"

    def __init__(self, config: GlueSourceConfig, ctx: PipelineContext):
        super().__init__(config, ctx)
        self.ctx = ctx
        self.extract_owners = config.extract_owners
        self.source_config = config
        self.report = GlueSourceReport()
        self.report.catalog_id = self.source_config.catalog_id
        self.glue_client = config.glue_client
        self.s3_client = config.s3_client
        # Initialize Lake Formation client
        self.lf_client = config.lakeformation_client
        self.extract_transforms = config.extract_transforms
        self.env = config.env
        self._glue_connection_cache: Dict[str, Optional[Tuple[str, str]]] = {}
        # Tracks which structured property definitions have been emitted this run
        # so each key's definition is only upserted once.
        self._seen_column_param_urns: Set[str] = set()

        self.platform_resource_repository: Optional[
            "GluePlatformResourceRepository"
        ] = None
        if self.ctx.graph:
            self.platform_resource_repository = GluePlatformResourceRepository(
                self.ctx.graph,
                platform_instance=self.source_config.platform_instance,
                catalog=self.source_config.catalog_id,
            )

        # Parses VIRTUAL_VIEW SQL definitions into view->upstream lineage. View
        # parsing is deferred to gen_metadata(), by which point every table and
        # view schema has been registered.
        self.aggregator: Optional[SqlParsingAggregator] = None
        if self.source_config.include_view_lineage:
            self.aggregator = SqlParsingAggregator(
                platform=self.platform,
                platform_instance=self.source_config.platform_instance,
                env=self.env,
                graph=self.ctx.graph,
                eager_graph_load=False,
            )
            # Surface the aggregator's parse metrics/warnings in the source report.
            self.report.sql_aggregator = self.aggregator.report

    def close(self) -> None:
        if self.aggregator:
            self.aggregator.close()
        super().close()

    def get_database_lf_tags(
        self,
        catalog_id: str,
        database_name: str,
    ) -> List[LakeFormationTag]:
        """Get the Lake Formation tags directly assigned to a database."""
        try:
            response = self.lf_client.get_resource_lf_tags(
                CatalogId=catalog_id,
                Resource={
                    "Database": {
                        "CatalogId": catalog_id,
                        "Name": database_name,
                    }
                },
                ShowAssignedLFTags=True,
            )
            return self._parse_lf_tag_dicts(response.get("LFTagOnDatabase", []))
        except Exception as e:
            self.report.warning(
                title="Failed to extract Lake Formation tags for database",
                message="Lake Formation tags will be missing for this database "
                "(and its tables if propagation is enabled). Check that the "
                "ingestion role has lakeformation:GetResourceLFTags.",
                context=f"{catalog_id}.{database_name}",
                exc=e,
            )
            return []

    @staticmethod
    def _parse_lf_tag_dicts(lf_tags: List[Dict]) -> List[LakeFormationTag]:
        """Convert AWS LF-tag dicts ({CatalogId, TagKey, TagValues}) to LakeFormationTag."""
        tags = []
        for lf_tag in lf_tags:
            tag_key = lf_tag.get("TagKey")
            if not tag_key:
                # Malformed response entry; nothing meaningful to emit.
                continue
            catalog_id = lf_tag.get("CatalogId")
            for tag_value in lf_tag.get("TagValues", []):
                tags.append(
                    LakeFormationTag(key=tag_key, value=tag_value, catalog=catalog_id)
                )
        return tags

    def get_table_lf_tags(
        self,
        catalog_id: str,
        database_name: str,
        table_name: str,
    ) -> TableLakeFormationTags:
        """Get the LF tags for a table, its database, and its columns in one call.

        A single GetResourceLFTags call on the Table resource returns the
        database's tags (``LFTagOnDatabase``), the table's directly-assigned tags
        (``LFTagsOnTable``), and each column's directly-assigned tags
        (``LFTagsOnColumns``) — so no per-column or per-database calls are needed.
        """
        try:
            response = self.lf_client.get_resource_lf_tags(
                CatalogId=catalog_id,
                Resource={
                    "Table": {
                        "CatalogId": catalog_id,
                        "DatabaseName": database_name,
                        "Name": table_name,
                    },
                },
                ShowAssignedLFTags=True,
            )
            column_tags: Dict[str, List[LakeFormationTag]] = {}
            for column in response.get("LFTagsOnColumns", []):
                parsed = self._parse_lf_tag_dicts(column.get("LFTags", []))
                if parsed:
                    column_tags[column["Name"]] = parsed
            return TableLakeFormationTags(
                database_tags=self._parse_lf_tag_dicts(
                    response.get("LFTagOnDatabase", [])
                ),
                table_tags=self._parse_lf_tag_dicts(response.get("LFTagsOnTable", [])),
                column_tags=column_tags,
            )
        except Exception as e:
            self.report.warning(
                title="Failed to extract Lake Formation tags for table",
                message="Lake Formation tags will be missing for this table. "
                "Check that the ingestion role has lakeformation:GetResourceLFTags.",
                context=f"{catalog_id}.{database_name}.{table_name}",
                exc=e,
            )
            return TableLakeFormationTags(
                database_tags=[], table_tags=[], column_tags={}
            )

    def _resolve_table_lf_tags(
        self, lf: TableLakeFormationTags
    ) -> ResolvedLakeFormationTags:
        """Resolve the LF tags to apply to a table, split into direct and propagated.

        The table's own tags are direct. When
        `propagate_lakeformation_tags` is enabled, database
        tags are inherited (propagated) for keys the table does not already
        assign — a value assigned directly to the table overrides the inherited
        database value, matching Lake Formation's inheritance/override semantics.
        """
        direct = list(lf.table_tags)
        propagated: List[LakeFormationTag] = []
        if self.source_config.propagate_lakeformation_tags:
            table_keys = {str(tag.key) for tag in direct}
            for db_tag in lf.database_tags:
                if str(db_tag.key) not in table_keys:
                    propagated.append(db_tag)
                    table_keys.add(str(db_tag.key))
        return ResolvedLakeFormationTags(direct=direct, propagated=propagated)

    def get_all_lf_tags(self) -> List:
        # 1. Get all LF-Tags in your account (metadata only)
        response = self.lf_client.list_lf_tags(
            MaxResults=50  # Adjust as needed
        )
        all_lf_tags = response["LFTags"]
        # Continue pagination if necessary
        while "NextToken" in response:
            response = self.lf_client.list_lf_tags(
                NextToken=response["NextToken"], MaxResults=50
            )
            all_lf_tags.extend(response["LFTags"])
        return all_lf_tags

    def get_glue_arn(
        self, account_id: str, database: str, table: Optional[str] = None
    ) -> str:
        region = self.source_config.aws_region or ""
        prefix = f"arn:{_aws_partition_for_region(region)}:glue:{region}:{account_id}"
        if table:
            return f"{prefix}:table/{database}/{table}"
        return f"{prefix}:database/{database}"

    def _resolve_platform_instance(self, catalog_id: Optional[str]) -> ResolvedTarget:
        """Resolve the platform_instance/env to stamp on a table owned by ``catalog_id``.

        Cross-account tables (a different ``catalog_id``, or Lake Formation shared tables) must be
        stamped with the *owning* account's instance so their URNs match what that account's own
        Glue ingestion emits — not the ingestion account's instance. The owning catalog is keyed by
        its ARN authority ``arn:{partition}:glue:{region}:{account}`` (account + region; account
        alone is not unique across regions), the same key the Spark/OpenLineage ``connections`` map
        uses. Falls back to the source's own ``platform_instance``/``env`` when there is no mapping.
        """
        instance = self.source_config.platform_instance
        env = self.env
        if catalog_id and self.source_config.catalog_to_platform_instance:
            region = self.source_config.aws_region or ""
            arn = f"arn:{_aws_partition_for_region(region)}:glue:{region}:{catalog_id}"
            detail = self.source_config.catalog_to_platform_instance.get(arn)
            if detail is not None:
                if detail.platform_instance is not None:
                    instance = detail.platform_instance
                if detail.env is not None:
                    env = detail.env
        return ResolvedTarget(platform_instance=instance, env=env)

    def _resource_link_owner_urn(self, target: Dict) -> Optional[str]:
        """URN of the table a resource link points at, stamped with the owning catalog's instance."""
        database_name = target.get("DatabaseName")
        name = target.get("Name")
        if not database_name or not name:
            return None
        owner = self._resolve_platform_instance(target.get("CatalogId"))
        return make_dataset_urn_with_platform_instance(
            platform=self.platform,
            name=f"{database_name}.{name}",
            env=owner.env,
            platform_instance=owner.platform_instance,
        )

    def _resource_link_owner_upstreams(
        self, table: Dict, dataset_urn: str
    ) -> List[UpstreamClass]:
        """Table-level upstream edge from a Lake Formation resource link to its owning table.

        A resource-link table carries a ``TargetTable`` pointing at the shared table in the owning
        account. Emit a COPY edge to the owner's table URN — stamped with the owner account's
        instance — so the shared dataset stitches back to its source instead of appearing as an
        unrelated duplicate. Only the table-level upstream edge is built here; the link's own schema
        is populated separately by ``_resource_link_schema`` (when ``resolve_resource_link_schema``
        is enabled). Returns [] for non-resource-link tables, or when the target is missing
        identifiers or resolves to the dataset's own URN (no catalog mapping).
        """
        target = table.get("TargetTable")
        if not target:
            return []

        owner_urn = self._resource_link_owner_urn(target)
        if owner_urn is None:
            self.report.num_resource_link_missing_target += 1
            self.report.warning(
                title="Resource link missing target identifiers",
                message="A Lake Formation resource link's TargetTable is missing its database or "
                "table name, so its cross-account upstream edge could not be built. The shared "
                "table will be ingested without lineage back to its owner.",
                context=f"{table.get('DatabaseName')}.{table.get('Name')} -> {target}",
            )
            return []

        if owner_urn == dataset_urn:
            # No catalog mapping resolved the owner to a distinct instance and the link/target names
            # coincide, so the owner URN is the dataset's own URN — a self-upstream edge is
            # meaningless. Counted so an operator can see cross-account links that produced no
            # lineage because their owning catalog wasn't mapped in catalog_to_platform_instance.
            self.report.num_resource_link_self_referential += 1
            logger.debug(
                f"Skipping self-referential resource-link lineage for {dataset_urn} "
                f"(no catalog mapping for target {target.get('CatalogId')})"
            )
            return []

        return [UpstreamClass(dataset=owner_urn, type=DatasetLineageTypeClass.COPY)]

    def _resource_link_schema(
        self, table: Dict, dataset_urn: str, table_name: str
    ) -> Optional[SchemaMetadata]:
        """Resolve the schema to stamp on a Lake Formation resource link.

        Runs only for a link that exposed no schema of its own (no ``StorageDescriptor``) — the usual
        case for a resource link, which is just a pointer to the owning account's table. When
        ``resolve_resource_link_schema`` is enabled, populate its schema from the owning table:
        DataHub first (the owner's already-ingested schema, no AWS call), falling back to a
        cross-account ``glue:GetTable`` on the target. Returns ``None`` — leaving the link schemaless
        and reached through its upstream edge — for non-links, when disabled, when the owner resolves
        to the dataset's own URN, or when neither source can supply the schema.
        """
        if not self.source_config.resolve_resource_link_schema:
            return None
        target = table.get("TargetTable")
        if not target:
            return None
        owner_urn = self._resource_link_owner_urn(target)
        if owner_urn is None or owner_urn == dataset_urn:
            return None

        owner_schema = self._resolve_owner_schema(owner_urn, target, table_name)
        if owner_schema is None:
            return None

        # Copy before re-stamping schemaName so we never mutate the object the fetch handed back
        # (the DataHub graph read in particular returns an aspect we don't own). Deep so an
        # in-place schema-field transformer (e.g. add_dataset_schema_tags mutating
        # SchemaField.globalTags) on the link can't reach back into that source object.
        link_schema = copy.deepcopy(owner_schema)
        link_schema.schemaName = table_name
        return link_schema

    def _resolve_owner_schema(
        self, owner_urn: str, target: Dict, table_name: str
    ) -> Optional[SchemaMetadata]:
        """Resolve a shared table's schema: DataHub first, then cross-account Glue.

        Counters and the unresolved warning fire once per call (i.e. per resource link); a shared
        table linked more than once is resolved independently each time.
        """
        datahub = self._resource_link_schema_from_datahub(owner_urn)
        if datahub.schema is not None:
            self.report.num_resource_link_schema_from_datahub += 1
            return datahub.schema

        glue = self._resource_link_schema_from_glue(target, table_name, owner_urn)
        if glue.schema is not None:
            self.report.num_resource_link_schema_from_glue += 1
            return glue.schema

        self.report.num_resource_link_schema_unresolved += 1
        # A failed DataHub/Glue read already warned (with the underlying exception); only add the
        # generic summary when neither did, so one unresolved owner produces exactly one report entry.
        if not (datahub.warned or glue.warned):
            self.report.warning(
                title="Resource link schema could not be resolved",
                message="A Lake Formation resource link's owning table is neither ingested in "
                "DataHub nor readable via cross-account glue:GetTable, so the link is ingested "
                "without a schema. Its columns remain reachable through its upstream lineage edge. "
                "Ingest the owning account, grant glue:GetTable on the shared table, or set "
                "resolve_resource_link_schema=false to silence this.",
                context=f"{table_name} -> {owner_urn}",
            )
        return None

    def _resource_link_schema_from_datahub(self, owner_urn: str) -> OwnerSchemaFetch:
        """Read the owning table's already-ingested schema from DataHub."""
        if self.ctx.graph is None:
            return OwnerSchemaFetch(schema=None, warned=False)
        try:
            # Read the full SchemaMetadata aspect directly rather than via SchemaResolver.
            # SchemaResolver caches only a lossy column-name -> native-type map (built for SQL
            # column-lineage matching) that drops nested struct fields, descriptions, and tags,
            # and it fetches one URN at a time -- so it offers no speedup here, only fidelity loss.
            return OwnerSchemaFetch(
                schema=self.ctx.graph.get_schema_metadata(owner_urn), warned=False
            )
        except Exception as e:
            # A GMS/network error must not abort the run — fall back to Glue (or schemaless).
            self.report.warning(
                title="Failed to read resource-link owner schema from DataHub",
                message="Could not fetch the owning table's schema from DataHub; will fall back to "
                "a cross-account Glue read.",
                context=owner_urn,
                exc=e,
            )
            return OwnerSchemaFetch(schema=None, warned=True)

    def _resource_link_schema_from_glue(
        self, target: Dict, table_name: str, owner_urn: str
    ) -> OwnerSchemaFetch:
        """Read the owning table's schema via a cross-account ``glue:GetTable`` on the target."""
        try:
            response = self.glue_client.get_table(
                CatalogId=target["CatalogId"],
                DatabaseName=target["DatabaseName"],
                Name=target["Name"],
            )
        except Exception as e:
            # Missing glue:GetTable permission on the shared table is expected for many Lake
            # Formation table-level grants — degrade to schemaless rather than failing the run.
            self.report.warning(
                title="Failed to read resource-link owner schema from Glue",
                message="Could not fetch the owning table via cross-account glue:GetTable; the "
                "resource link will be ingested without a schema. Grant glue:GetTable on the "
                "shared table to enable this.",
                context=f"{target.get('DatabaseName')}.{target.get('Name')}",
                exc=e,
            )
            return OwnerSchemaFetch(schema=None, warned=True)
        return OwnerSchemaFetch(
            schema=self._get_schema_metadata(
                response["Table"], table_name, dataset_urn=owner_urn
            ),
            warned=False,
        )

    @classmethod
    def create(cls, config_dict, ctx):
        config = GlueSourceConfig.model_validate(config_dict)
        return cls(config, ctx)

    @property
    def platform(self) -> str:
        return self.source_config.platform

    def get_all_jobs(self):
        """
        List all jobs in Glue. boto3 get_jobs api call doesn't support cross account access.
        """

        jobs = []

        # see https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/glue.html#Glue.Client.get_jobs
        paginator = self.glue_client.get_paginator("get_jobs")
        for page in paginator.paginate():
            jobs += page["Jobs"]

        return jobs

    def get_dataflow_script(self, script_path: str, flow_urn: str) -> Optional[str]:
        # handle a bug in AWS where script path has duplicate prefixes
        if script_path.lower().startswith("s3://s3://"):
            script_path = script_path[5:]

        # catch any other cases where the script path is invalid
        if not script_path.startswith("s3://"):
            self.report.warning(
                message="Error parsing DAG for Glue job: invalid S3 script path",
                context=f"{flow_urn}: {script_path}",
                log=False,
            )
            self.report.num_job_script_location_invalid += 1

            return None

        # extract the script's bucket and key
        url = urlparse(script_path, allow_fragments=False)
        bucket = url.netloc
        key = url.path[1:]

        # validate that we have a non-empty key
        if not key:
            self.report.num_job_script_location_invalid += 1
            logger.warning(
                f"Error parsing DAG for Glue job. The script {script_path} is not a valid S3 path for flow urn: {flow_urn}."
            )
            return None

        # download the script contents
        # see https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#S3.Client.get_object
        try:
            obj = self.s3_client.get_object(Bucket=bucket, Key=key)
        except botocore.exceptions.ClientError as e:
            self.report.warning(
                message="Unable to download DAG for Glue job, so job subtasks and lineage will be missing",
                context=f"{flow_urn}: {script_path}",
                exc=e,
                log=False,
            )
            self.report.num_job_script_failed_download += 1
            return None
        except botocore.exceptions.ParamValidationError as e:
            self.report.warning(
                message="Invalid S3 path for Glue job script",
                context=f"{flow_urn}: {script_path}",
                exc=e,
                log=False,
            )
            self.report.num_job_script_location_invalid += 1
            return None

        return obj["Body"].read().decode("utf-8")

    def get_dataflow_graph(
        self, script: str, script_path: str, flow_urn: str
    ) -> Optional[Dict[str, Any]]:
        """
        Get the DAG of transforms and data sources/sinks for a job.

        Parameters
        ----------
            script_path:
                S3 path to the job's Python script.
        """

        try:
            # extract the job DAG from the script
            # see https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/glue.html#Glue.Client.get_dataflow_graph
            return self.glue_client.get_dataflow_graph(PythonScript=script)

        # sometimes the Python script can be user-modified and the script is not valid for graph extraction
        except self.glue_client.exceptions.InvalidInputException as e:
            self.report.warning(
                message="Error parsing DAG for Glue job: script cannot be processed by Glue (this usually occurs when it has been user-modified)",
                context=f"{flow_urn}: {script_path}",
                exc=e,
                log=False,
            )
            self.report.num_job_script_failed_parsing += 1

            return None

    def get_s3_uri(self, node_args):
        s3_uri = node_args.get("connection_options", {}).get("path")

        # sometimes the path is a single element in a list rather than a single one
        if s3_uri is None:
            s3_uri = node_args.get("connection_options", {}).get("paths")[0]

        return s3_uri

    def _parse_jdbc_url(self, jdbc_url: str) -> Tuple[str, str]:
        """Parse a JDBC URL and return (platform, database).

        Strips the "jdbc:" prefix then uses urlparse to extract the protocol
        (mapped to a DataHub platform name) and the database from the path.
        """
        if not jdbc_url.startswith(JDBC_PREFIX):
            raise ValueError(f"Not a valid JDBC URL: {_sanitize_jdbc_url(jdbc_url)}")
        url = urlparse(jdbc_url[len(JDBC_PREFIX) :])
        protocol = url.scheme.lower()
        platform = JDBC_PLATFORM_MAP.get(protocol, protocol)
        database = url.path.lstrip("/").split("?")[0]
        if not database:
            props = dict(
                part.split("=", 1) for part in url.netloc.split(";")[1:] if "=" in part
            )
            database = props.get("databaseName", "")
        return platform, database

    def _extract_urns_from_query(
        self, query: str, platform: str, database: str, flow_urn: str, node_label: str
    ) -> Optional[List[str]]:
        """Parse a SQL query and return DataHub URNs for all referenced input tables.

        Uses dialect-aware parsing via sqlglot_lineage with default schema resolution.
        Returns None on parse failure or when no tables are found.
        """
        # Apply the target platform's instance/env (if configured) so the parsed upstream URNs match
        # what that platform's own connector emits — same as _make_dataset_urn_for_platform.
        target_config = self.source_config.target_platform_configs.get(platform)
        result = create_lineage_sql_parsed_result(
            query=query,
            default_db=database,
            platform=platform,
            platform_instance=(
                target_config.platform_instance if target_config else None
            ),
            env=target_config.env if target_config and target_config.env else self.env,
            default_schema=JDBC_DEFAULT_SCHEMA.get(platform),
            schema_aware=False,
            generate_column_lineage=False,
        )
        if result.debug_info.error:
            self.report.warning(
                message="Failed to parse SQL query for node",
                context=f"{flow_urn}: {node_label}: {result.debug_info.error}",
                log=False,
            )
            return None
        if not result.in_tables:
            self.report.warning(
                message="No tables found in SQL query for node",
                context=f"{flow_urn}: {node_label}",
                log=False,
            )
            return None
        return result.in_tables

    def _resolve_glue_connection(
        self, connection_name: str, flow_urn: str
    ) -> Optional[Tuple[str, str]]:
        """Resolve a named Glue connection to (platform, database)."""
        if connection_name in self._glue_connection_cache:
            return self._glue_connection_cache[connection_name]

        result: Optional[Tuple[str, str]] = None
        try:
            kwargs: Dict[str, Any] = {"Name": connection_name, "HidePassword": True}
            if self.source_config.catalog_id:
                kwargs["CatalogId"] = self.source_config.catalog_id
            response = self.glue_client.get_connection(**kwargs)
            connection = response["Connection"]
            conn_type = connection.get("ConnectionType", "")
            props = connection.get("ConnectionProperties", {})
            spark_props = connection.get("SparkProperties", {})

            if conn_type == "JDBC":
                jdbc_url = props.get("JDBC_CONNECTION_URL") or spark_props.get(
                    "JDBC_CONNECTION_URL"
                )
                if not jdbc_url:
                    self.report.warning(
                        message="Glue connection has no JDBC_CONNECTION_URL",
                        context=f"{flow_urn}: {connection_name!r}",
                        log=False,
                    )
                else:
                    try:
                        result = self._parse_jdbc_url(jdbc_url)
                    except Exception as e:
                        self.report.warning(
                            message="Failed to parse JDBC URL for Glue connection",
                            context=f"{flow_urn}: {connection_name!r}",
                            exc=e,
                            log=False,
                        )
                        result = None
            elif conn_type in GLUE_NATIVE_CONNECTION_TYPE_MAP:
                platform = GLUE_NATIVE_CONNECTION_TYPE_MAP[conn_type]
                database = props.get("DATABASE")
                if not database:
                    self.report.warning(
                        message="Glue connection has no DATABASE property",
                        context=f"{flow_urn}: {connection_name!r}",
                        log=False,
                    )
                else:
                    result = (platform, database)
            else:
                self.report.warning(
                    message="Unsupported Glue connection type",
                    context=f"{flow_urn}: type={conn_type!r}, connection={connection_name!r}",
                    log=False,
                )
        except Exception as e:
            self.report.warning(
                message="Failed to fetch Glue connection",
                context=f"{flow_urn}: {connection_name!r}",
                exc=e,
                log=False,
            )
            return None

        self._glue_connection_cache[connection_name] = result
        return result

    def get_dataflow_s3_names(
        self, dataflow_graph: Dict[str, Any]
    ) -> Iterator[Tuple[str, Optional[str]]]:
        # iterate through each node to populate processed nodes
        for node in dataflow_graph["DagNodes"]:
            node_type = node["NodeType"]

            # for nodes representing datasets, we construct a dataset URN accordingly
            if node_type in ["DataSource", "DataSink"]:
                node_args = {
                    x["Name"]: yaml.safe_load(x["Value"]) for x in node["Args"]
                }

                # if data object is S3 bucket
                if node_args.get("connection_type") == "s3":
                    s3_uri = self.get_s3_uri(node_args)

                    if s3_uri is None:
                        continue

                    extension = node_args.get("format")

                    yield s3_uri, extension

    def _make_dataset_urn_for_platform(self, platform: str, dataset_name: str) -> str:
        """Build a dataset URN using target_platform_configs if available.

        If target_platform_configs has an entry for this platform, applies its
        platform_instance and env so the URN matches what the platform's own
        connector produces.
        """
        target_config = self.source_config.target_platform_configs.get(platform)
        return make_dataset_urn_with_platform_instance(
            platform=platform,
            name=dataset_name,
            env=target_config.env if target_config and target_config.env else self.env,
            platform_instance=(
                target_config.platform_instance if target_config else None
            ),
        )

    def _build_jdbc_dataset_name(
        self, platform: str, database: str, dbtable: str
    ) -> str:
        if "." in dbtable:
            schema, table = dbtable.rsplit(".", 1)
            return f"{database}.{schema}.{table}"
        default_schema = JDBC_DEFAULT_SCHEMA.get(platform)
        if default_schema:
            return f"{database}.{default_schema}.{dbtable}"
        return f"{database}.{dbtable}"

    def _process_glue_connection_node(
        self, node: Dict[str, Any], node_args: Dict[str, Any], flow_urn: str
    ) -> Optional[List[str]]:
        connection_options = node_args.get("connection_options", {})
        connection_name = connection_options.get("connectionName")
        if not connection_name:
            return None
        node_label = f"{node['NodeType']}-{node['Id']}"

        resolved = self._resolve_glue_connection(connection_name, flow_urn)
        if resolved is None:
            return None
        platform, database = resolved

        dbtable = connection_options.get("dbtable")
        if dbtable:
            return [
                self._make_dataset_urn_for_platform(
                    platform,
                    self._build_jdbc_dataset_name(platform, database, dbtable),
                )
            ]

        query = connection_options.get("query")
        if query:
            return self._extract_urns_from_query(
                query, platform, database, flow_urn, node_label
            )

        self.report.warning(
            message="Missing dbtable or query for node",
            context=f"{flow_urn}: {node_label}",
            log=False,
        )
        return None

    def _process_jdbc_node(
        self, node: Dict[str, Any], node_args: Dict[str, Any], flow_urn: str
    ) -> Optional[List[str]]:
        connection_options = node_args.get("connection_options", {})
        jdbc_url = connection_options.get("url")
        node_label = f"{node['NodeType']}-{node['Id']}"

        if not jdbc_url:
            self.report.warning(
                message="Missing JDBC URL for node",
                context=f"{flow_urn}: {node_label}",
                log=False,
            )
            return None

        try:
            platform, database = self._parse_jdbc_url(jdbc_url)
        except ValueError as e:
            self.report.warning(
                message="Failed to parse JDBC URL for node",
                context=f"{flow_urn}: {node_label}",
                exc=e,
                log=False,
            )
            return None

        dbtable = connection_options.get("dbtable")
        if dbtable:
            return [
                self._make_dataset_urn_for_platform(
                    platform,
                    self._build_jdbc_dataset_name(platform, database, dbtable),
                )
            ]

        query = connection_options.get("query")
        if query:
            return self._extract_urns_from_query(
                query, platform, database, flow_urn, node_label
            )

        self.report.warning(
            message="Missing dbtable or query for node",
            context=f"{flow_urn}: {node_label}",
            log=False,
        )
        return None

    def process_dataflow_node(
        self,
        node: Dict[str, Any],
        flow_urn: str,
    ) -> Optional[Dict[str, Any]]:
        node_type = node["NodeType"]
        dataset_urns: Optional[List[str]] = None

        # for nodes representing datasets, we construct a dataset URN accordingly
        if node_type in ["DataSource", "DataSink"]:
            node_args = {x["Name"]: yaml.safe_load(x["Value"]) for x in node["Args"]}

            # if data object is Glue table
            if "database" in node_args and "table_name" in node_args:
                full_table_name = f"{node_args['database']}.{node_args['table_name']}"

                # Stamp the owning catalog's instance/env so this URN matches the dataset entity
                # emitted when ingesting Glue tables. A cross-account job reading via
                # from_catalog(catalog_id=...) carries that catalog id in the node args; fall back
                # to the source's own catalog_id, then to no mapping (the source's own instance).
                resolved = self._resolve_platform_instance(
                    node_args.get("catalog_id") or self.source_config.catalog_id
                )
                node_urn = make_dataset_urn_with_platform_instance(
                    platform=self.platform,
                    name=full_table_name,
                    env=resolved.env,
                    platform_instance=resolved.platform_instance,
                )

            # if data object is S3 bucket
            elif node_args.get("connection_type") == "s3":
                s3_uri = self.get_s3_uri(node_args)

                if s3_uri is None:
                    self.report.warning(
                        message="Could not find S3 path for job node",
                        context=f"{flow_urn}: {node['NodeType']}-{node['Id']}",
                        log=False,
                    )
                    return None

                node_urn = make_s3_urn(s3_uri, self.env)

            # if data object references a named Glue connection (visual editor style)
            elif (node_args.get("connection_options") or {}).get("connectionName"):
                _urns = self._process_glue_connection_node(node, node_args, flow_urn)
                if not _urns:
                    return None
                node_urn = _urns[0]
                if len(_urns) > 1:
                    dataset_urns = _urns

            # if data object is a JDBC source (e.g. Postgres, MySQL, Redshift)
            elif node_args.get("connection_type") in JDBC_PLATFORM_MAP:
                _urns = self._process_jdbc_node(node, node_args, flow_urn)
                if not _urns:
                    return None
                node_urn = _urns[0]
                if len(_urns) > 1:
                    dataset_urns = _urns

            else:
                if self.source_config.ignore_unsupported_connectors:
                    self.report.warning(
                        message="Unrecognized node in Glue flow",
                        context=f"{flow_urn}: {node['NodeType']}-{node['Id']}, args={node_args}",
                        log=False,
                    )
                    return None
                else:
                    raise ValueError(f"Unrecognized Glue data object type: {node_args}")

        # otherwise, a node represents a transformation
        else:
            node_urn = mce_builder.make_data_job_urn_with_flow(
                flow_urn, job_id=f"{node['NodeType']}-{node['Id']}"
            )

        result: Dict[str, Any] = {
            **node,
            "urn": node_urn,
            # to be filled in after traversing edges
            "inputDatajobs": [],
            "inputDatasets": [],
            "outputDatasets": [],
        }
        if dataset_urns is not None:
            result["dataset_urns"] = dataset_urns
        return result

    def process_dataflow_graph(
        self,
        dataflow_graph: Dict[str, Any],
        flow_urn: str,
    ) -> Dict[str, Dict[str, Any]]:
        """
        Prepare a job's DAG for ingestion.
        Parameters
        ----------
            dataflow_graph:
                Job DAG returned from get_dataflow_graph()
            flow_urn:
                URN of the flow (i.e. the AWS Glue job itself).
        """

        nodes: dict = {}

        # iterate through each node to populate processed nodes
        for node in dataflow_graph["DagNodes"]:
            processed_node = self.process_dataflow_node(node, flow_urn)

            if processed_node is not None:
                nodes[node["Id"]] = processed_node

        # traverse edges to fill in node properties
        for edge in dataflow_graph["DagEdges"]:
            source_node = nodes.get(edge["Source"])
            target_node = nodes.get(edge["Target"])

            # Currently, in case of unsupported connectors,
            # Source and Target for some edges is not available
            # in nodes. this may lead to broken edge in lineage.
            if source_node is None or target_node is None:
                self.report.warning(
                    message="Unrecognized source or target node in edge, this may lead to missing lineage",
                    context=f"{flow_urn}: {edge}",
                    log=False,
                )
                continue

            source_node_type = source_node["NodeType"]
            target_node_type = target_node["NodeType"]

            # note that source nodes can't be data sinks
            if source_node_type == "DataSource":
                target_node["inputDatasets"].extend(
                    source_node.get("dataset_urns", [source_node["urn"]])
                )
            # keep track of input data jobs (as defined in schemas)
            else:
                target_node["inputDatajobs"].append(source_node["urn"])
            # track output datasets (these can't be input datasets)
            if target_node_type == "DataSink":
                source_node["outputDatasets"].append(target_node["urn"])

        return nodes

    def get_dataflow_wus(
        self, flow_urn: str, job: Dict[str, Any]
    ) -> Iterable[MetadataWorkUnit]:
        """
        Generate a DataFlow workunit for a Glue job.

        Parameters
        ----------
            flow_urn:
                URN for the flow
            job:
                Job object from get_all_jobs()
        """

        region = self.source_config.aws_region

        custom_props = {
            "role": job["Role"],
        }

        if job.get("CreatedOn") is not None:
            custom_props["created"] = str(job["CreatedOn"])

        if job.get("LastModifiedOn") is not None:
            custom_props["modified"] = str(job["LastModifiedOn"])

        command = job.get("Command", {}).get("ScriptLocation")
        if command is not None:
            custom_props["command"] = command

        mce = MetadataChangeEventClass(
            proposedSnapshot=DataFlowSnapshotClass(
                urn=flow_urn,
                aspects=[
                    DataFlowInfoClass(
                        name=job["Name"],
                        description=job.get("Description"),
                        externalUrl=f"https://{region}.console.aws.amazon.com/gluestudio/home?region={region}#/editor/job/{job['Name']}/graph",
                        # specify a few Glue-specific properties
                        customProperties=custom_props,
                    ),
                ],
            )
        )
        yield MetadataWorkUnit(id=job["Name"], mce=mce)

        yield MetadataChangeProposalWrapper(
            entityUrn=flow_urn,
            aspect=SubTypes(typeNames=[FlowContainerSubTypes.GLUE_JOB]),
        ).as_workunit()

    def get_datajob_wus_for_dataflow(
        self, flow_urn: str, job_name: str, script: Optional[str]
    ) -> Iterable[MetadataWorkUnit]:
        """
        Generate a DataJob workunit for a Glue job with no nodes.
        """

        job_urn = mce_builder.make_data_job_urn_with_flow(flow_urn, job_id=job_name)

        region = self.source_config.aws_region
        yield MetadataChangeProposalWrapper(
            entityUrn=job_urn,
            aspect=DataJobInfoClass(
                name=job_name,
                type="GLUE",
                externalUrl=f"https://{region}.console.aws.amazon.com/gluestudio/home?region={region}#/editor/job/{job_name}/graph",
                customProperties={},
            ),
        ).as_workunit()

        if script:
            redacted_script = _redact_secret_fields_in_dataflow_script(script)
            yield MetadataChangeProposalWrapper(
                entityUrn=job_urn,
                aspect=DataTransformLogicClass(
                    transforms=[
                        DataTransformClass(
                            queryStatement=QueryStatementClass(
                                value=redacted_script,
                                # The language field uses a pretty limited enum.
                                # The "UNKNOWN" enum value is pretty new, so we don't want to
                                # emit it until it has broader server-side support. As a
                                # short-term solution, we map all languages to "SQL".
                                # TODO: Once we've released server 1.1.0, we should change
                                # this to be "UNKNOWN" for all languages except "SQL".
                                language=QueryLanguageClass.SQL,
                            )
                        )
                    ]
                ),
            ).as_workunit()

    def get_datajob_wu(self, node: Dict[str, Any], job_name: str) -> MetadataWorkUnit:
        """
        Generate a DataJob workunit for a component (node) in a Glue job.

        Parameters
        ----------
            node:
                Node from process_dataflow_graph()
            job:
                Job object from get_all_jobs()
        """

        region = self.source_config.aws_region

        mce = MetadataChangeEventClass(
            proposedSnapshot=DataJobSnapshotClass(
                urn=node["urn"],
                aspects=[
                    DataJobInfoClass(
                        name=f"{job_name}:{node['NodeType']}-{node['Id']}",
                        type="GLUE",
                        # there's no way to view an individual job node by link, so just show the graph
                        externalUrl=f"https://{region}.console.aws.amazon.com/gluestudio/home?region={region}#/editor/job/{job_name}/graph",
                        customProperties={
                            **{x["Name"]: x["Value"] for x in node["Args"]},
                            "transformType": node["NodeType"],
                            "nodeId": node["Id"],
                        },
                    ),
                    DataJobInputOutputClass(
                        inputDatasets=node["inputDatasets"],
                        outputDatasets=node["outputDatasets"],
                        inputDatajobs=node["inputDatajobs"],
                    ),
                ],
            )
        )

        return MetadataWorkUnit(id=f"{job_name}-{node['Id']}", mce=mce)

    def get_all_databases(self) -> Iterable[Mapping[str, Any]]:
        logger.debug("Getting all databases")
        # see https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/glue/paginator/GetDatabases.html
        paginator = self.glue_client.get_paginator("get_databases")

        if self.source_config.catalog_id:
            paginator_response = paginator.paginate(
                CatalogId=self.source_config.catalog_id
            )
        else:
            paginator_response = paginator.paginate()

        pattern = "DatabaseList"
        if self.source_config.ignore_resource_links:
            # exclude resource links by using a JMESPath conditional query against the TargetDatabase struct key
            pattern += "[?!TargetDatabase]"

        for database in paginator_response.search(pattern):
            if (not self.source_config.database_pattern.allowed(database["Name"])) or (
                self.source_config.catalog_id
                and database.get("CatalogId")
                and database.get("CatalogId") != self.source_config.catalog_id
            ):
                self.report.databases.dropped(database["Name"])
            else:
                self.report.databases.processed(database["Name"])
                yield database

    def get_tables_from_database(self, database: Mapping[str, Any]) -> Iterable[Dict]:
        logger.debug(f"Getting tables from database {database['Name']}")
        # see https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/glue/paginator/GetTables.html
        paginator = self.glue_client.get_paginator("get_tables")
        database_name = database["Name"]

        if self.source_config.catalog_id:
            paginator_response = paginator.paginate(
                DatabaseName=database_name, CatalogId=self.source_config.catalog_id
            )
        else:
            paginator_response = paginator.paginate(DatabaseName=database_name)

        for table in paginator_response.search("TableList"):
            # Lake Formation can share individual tables across accounts as table-level
            # resource links (table has a TargetTable pointing at the shared table).
            # Database-level filtering in get_all_databases() does not catch these,
            # since they live inside non-resource-link databases.
            if self.source_config.ignore_resource_links and "TargetTable" in table:
                logger.debug(
                    f"Skipping resource link table {database_name}.{table.get('Name')} "
                    f"(TargetTable: {table.get('TargetTable')})"
                )
                continue

            # When ingesting resource-link databases (ignore_resource_links=False),
            # rewrite DatabaseName to the local alias so downstream URN construction
            # uses the catalog-local name instead of the target catalog's name.
            if (
                not self.source_config.ignore_resource_links
                and "TargetDatabase" in database
            ):
                table["DatabaseName"] = database["Name"]
            yield table

    def get_all_databases_and_tables(
        self,
    ) -> Tuple[List[Mapping[str, Any]], List[Dict]]:
        all_databases = [*self.get_all_databases()]
        all_tables = []
        for database in all_databases:
            try:
                for tables in self.get_tables_from_database(database):
                    all_tables.append(tables)
            except Exception as e:
                self.report.warning(
                    message="Failed to get tables from database",
                    context=database["Name"],
                    exc=e,
                )
        return all_databases, all_tables

    def get_lineage_if_enabled(
        self,
        mce: MetadataChangeEventClass,
        extra_upstreams: Optional[List[UpstreamClass]] = None,
    ) -> Iterable[MetadataWorkUnit]:
        """Emit the dataset's UpstreamLineage, combining storage lineage with any ``extra_upstreams``.

        This method is the single owner of the dataset's own ``UpstreamLineage`` aspect: callers pass
        additional upstream edges (e.g. the Lake Formation resource-link edge) and this method folds
        them into the one aspect that targets ``dataset_urn`` — so two independent sources can't emit
        two full-replace aspects on the same URN and clobber each other.
        """
        dataset_urn = mce.proposedSnapshot.urn
        extra_upstreams = extra_upstreams or []

        storage_upstreams: List[UpstreamClass] = []
        storage_fine_grained: List[FineGrainedLineageClass] = []
        if self.source_config.emit_storage_lineage:
            dataset_properties: Optional[DatasetPropertiesClass] = (
                mce_builder.get_aspect_if_available(mce, DatasetPropertiesClass)
            )
            schema_metadata: Optional[SchemaMetadataClass] = (
                mce_builder.get_aspect_if_available(mce, SchemaMetadataClass)
            )

            # get urn for underlying table storage
            table_storage_urn: Optional[str] = None
            if (
                dataset_properties
                and dataset_properties.customProperties.get("table_type")
                == GLUE_TABLE_TYPE_ICEBERG
            ):
                table_storage_urn = dataset_urn.replace(
                    "urn:li:dataPlatform:glue", "urn:li:dataPlatform:iceberg"
                )
            elif (
                dataset_properties and "Location" in dataset_properties.customProperties
            ):
                location = dataset_properties.customProperties["Location"]
                if is_s3_uri(location):
                    table_storage_urn = make_s3_urn_for_lineage(
                        location, self.source_config.env
                    )

            if table_storage_urn:
                if self.source_config.glue_storage_lineage_direction == "upstream":
                    schema_metadata_for_upstream = (
                        self.ctx.graph.get_schema_metadata(table_storage_urn)
                        if self.ctx.graph
                        else None
                    )
                    if self.source_config.include_column_lineage and schema_metadata:
                        storage_fine_grained = (
                            self.get_fine_grained_lineages(
                                dataset_urn,
                                table_storage_urn,
                                schema_metadata,
                                schema_metadata_for_upstream or schema_metadata,
                            )
                            or []
                        )
                    storage_upstreams = [
                        UpstreamClass(
                            dataset=table_storage_urn,
                            type=DatasetLineageTypeClass.COPY,
                        )
                    ]
                else:
                    # Downstream direction: the storage edge lives on the S3 dataset's own URN, so it
                    # never collides with dataset_urn's aspect — emit it standalone.
                    yield MetadataChangeProposalWrapper(
                        entityUrn=table_storage_urn,
                        aspect=UpstreamLineageClass(
                            upstreams=[
                                UpstreamClass(
                                    dataset=dataset_urn,
                                    type=DatasetLineageTypeClass.COPY,
                                )
                            ]
                        ),
                    ).as_workunit()

        upstreams = storage_upstreams + extra_upstreams
        if upstreams:
            yield MetadataChangeProposalWrapper(
                entityUrn=dataset_urn,
                aspect=UpstreamLineageClass(
                    upstreams=upstreams,
                    fineGrainedLineages=storage_fine_grained or None,
                ),
            ).as_workunit()

    def get_fine_grained_lineages(
        self,
        dataset_urn: str,
        upstream_urn: str,
        schema_metadata: SchemaMetadata,
        schema_metadata_for_upstream: SchemaMetadata,
    ) -> Optional[List[FineGrainedLineageClass]]:
        def simplify_field_path(field_path):
            return Dataset._simplify_field_path(field_path)

        if schema_metadata and schema_metadata_for_upstream:
            fine_grained_lineages: List[FineGrainedLineageClass] = []
            for field in schema_metadata.fields:
                field_path_v1 = simplify_field_path(field.fieldPath)
                matching_upstream_field = next(
                    (
                        f
                        for f in schema_metadata_for_upstream.fields
                        if simplify_field_path(f.fieldPath) == field_path_v1
                    ),
                    None,
                )
                if matching_upstream_field:
                    fine_grained_lineages.append(
                        FineGrainedLineageClass(
                            downstreamType=FineGrainedLineageDownstreamTypeClass.FIELD,
                            downstreams=[
                                mce_builder.make_schema_field_urn(
                                    dataset_urn, field_path_v1
                                )
                            ],
                            upstreamType=FineGrainedLineageUpstreamTypeClass.FIELD_SET,
                            upstreams=[
                                mce_builder.make_schema_field_urn(
                                    upstream_urn,
                                    simplify_field_path(
                                        matching_upstream_field.fieldPath
                                    ),
                                )
                            ],
                        )
                    )
            return fine_grained_lineages
        return None

    def _create_profile_mcp(
        self,
        mce: MetadataChangeEventClass,
        table_stats: dict,
        column_stats: dict,
        partition_spec: Optional[str] = None,
    ) -> Optional[MetadataChangeProposalWrapper]:
        assert self.source_config.profiling

        # instantiate profile class
        dataset_profile = DatasetProfileClass(timestampMillis=get_sys_time())

        # Inject table level stats
        if self.source_config.profiling.row_count in table_stats:
            dataset_profile.rowCount = int(
                float(table_stats[self.source_config.profiling.row_count])
            )
        if self.source_config.profiling.column_count in table_stats:
            dataset_profile.columnCount = int(
                float(table_stats[self.source_config.profiling.column_count])
            )

        # inject column level stats
        dataset_profile.fieldProfiles = []
        for profile in column_stats:
            column_name = profile["Name"]

            # some columns may not be profiled
            if "Parameters" in profile:
                column_params = profile["Parameters"]
            else:
                continue

            logger.debug(f"column_name: {column_name}")
            # instantiate column profile class for each column
            column_profile = DatasetFieldProfileClass(fieldPath=column_name)

            if not self.source_config.profiling.profile_table_level_only:
                if self.source_config.profiling.unique_count in column_params:
                    column_profile.uniqueCount = int(
                        float(column_params[self.source_config.profiling.unique_count])
                    )
                if self.source_config.profiling.unique_proportion in column_params:
                    column_profile.uniqueProportion = float(
                        column_params[self.source_config.profiling.unique_proportion]
                    )
                if self.source_config.profiling.null_count in column_params:
                    column_profile.nullCount = int(
                        float(column_params[self.source_config.profiling.null_count])
                    )
                if self.source_config.profiling.null_proportion in column_params:
                    column_profile.nullProportion = float(
                        column_params[self.source_config.profiling.null_proportion]
                    )
                if self.source_config.profiling.min in column_params:
                    column_profile.min = column_params[self.source_config.profiling.min]
                if self.source_config.profiling.max in column_params:
                    column_profile.max = column_params[self.source_config.profiling.max]
                if self.source_config.profiling.mean in column_params:
                    column_profile.mean = column_params[
                        self.source_config.profiling.mean
                    ]
                if self.source_config.profiling.median in column_params:
                    column_profile.median = column_params[
                        self.source_config.profiling.median
                    ]
                if self.source_config.profiling.stdev in column_params:
                    column_profile.stdev = column_params[
                        self.source_config.profiling.stdev
                    ]

            dataset_profile.fieldProfiles.append(column_profile)

        # if no stats are available, skip ingestion
        if (
            not dataset_profile.fieldProfiles
            and dataset_profile.rowCount is None
            and dataset_profile.columnCount is None
        ):
            return None

        if partition_spec:
            # inject partition level stats
            dataset_profile.partitionSpec = PartitionSpecClass(
                partition=partition_spec,
                type=PartitionTypeClass.PARTITION,
            )

        mcp = MetadataChangeProposalWrapper(
            entityUrn=mce.proposedSnapshot.urn,
            aspect=dataset_profile,
        )
        return mcp

    def get_profile_if_enabled(
        self, mce: MetadataChangeEventClass, database_name: str, table_name: str
    ) -> Iterable[MetadataWorkUnit]:
        if self.source_config.is_profiling_enabled():
            # for cross-account ingestion
            kwargs = dict(
                DatabaseName=database_name,
                Name=table_name,
                CatalogId=self.source_config.catalog_id,
            )
            response = self.glue_client.get_table(
                **{k: v for k, v in kwargs.items() if v}
            )

            # check if this table is partitioned
            partition_keys = response["Table"].get("PartitionKeys")
            if partition_keys:
                # ingest data profile with partitions
                # for cross-account ingestion
                kwargs = dict(
                    DatabaseName=database_name,
                    TableName=table_name,
                    CatalogId=self.source_config.catalog_id,
                )
                response = self.glue_client.get_partitions(
                    **{k: v for k, v in kwargs.items() if v}
                )

                partitions = response["Partitions"]
                partition_keys = [k["Name"] for k in partition_keys]

                for p in partitions:
                    table_stats = p.get("Parameters", {})
                    column_stats = p["StorageDescriptor"]["Columns"]

                    # only support single partition key
                    partition_spec = str({partition_keys[0]: p["Values"][0]})

                    if self.source_config.profiling.partition_patterns.allowed(
                        partition_spec
                    ):
                        profile_mcp = self._create_profile_mcp(
                            mce, table_stats, column_stats, partition_spec
                        )
                        if profile_mcp:
                            yield profile_mcp.as_workunit()
                    else:
                        continue
            else:
                # ingest data profile without partition
                table_stats = response["Table"]["Parameters"]
                column_stats = response["Table"]["StorageDescriptor"]["Columns"]
                profile_mcp = self._create_profile_mcp(mce, table_stats, column_stats)
                if profile_mcp:
                    yield profile_mcp.as_workunit()

    def gen_database_key(self, database: str) -> DatabaseKey:
        return DatabaseKey(
            database=database,
            platform=self.platform,
            instance=self.source_config.platform_instance,
            env=self.source_config.env,
            backcompat_env_as_instance=True,
        )

    def gen_platform_resource(
        self, tag: LakeFormationTag
    ) -> Iterable[MetadataWorkUnit]:
        if self.ctx.graph and self.platform_resource_repository:
            platform_resource_id = (
                LakeFormationTagPlatformResourceId.get_or_create_from_tag(
                    tag=tag,
                    platform_resource_repository=self.platform_resource_repository,
                    catalog_id=tag.catalog,
                )
            )
            logger.info(f"Created platform resource {platform_resource_id}")

            lf_tag = self.platform_resource_repository.get_entity_from_datahub(
                platform_resource_id, False
            )
            if (
                tag.to_datahub_tag_urn().urn()
                not in lf_tag.datahub_linked_resources().urns
            ):
                try:
                    lf_tag.datahub_linked_resources().add(
                        tag.to_datahub_tag_urn().urn()
                    )
                    platform_resource = lf_tag.as_platform_resource()
                    for mcp in platform_resource.to_mcps():
                        yield MetadataWorkUnit(
                            id=f"platform_resource-{platform_resource.id}",
                            mcp=mcp,
                        )
                except Exception as e:
                    logger.warning(
                        f"Failed to create platform resource for tag {tag}: {e}",
                        exc_info=True,
                    )
                    self.report.warning(
                        message="Failed to create platform resource for tag",
                        context=str(tag),
                        exc=e,
                        log=False,
                    )

    def gen_database_containers(
        self, database: Mapping[str, Any]
    ) -> Iterable[MetadataWorkUnit]:
        container_tags: Optional[List] = None
        if self.source_config.extract_lakeformation_tags:
            try:
                tags = self.get_database_lf_tags(
                    catalog_id=database["CatalogId"], database_name=database["Name"]
                )
                container_tags = []
                for tag in tags:
                    try:
                        container_tags.append(tag.to_datahub_tag_urn().name)
                        yield from self.gen_platform_resource(tag)
                    except InvalidUrnError:
                        continue
            except Exception:
                self.report.warning(
                    "Failed to extract Lake Formation tags for database",
                    context=database["Name"],
                    log=False,
                )
        domain_urn = self._gen_domain_urn(database["Name"])
        database_container_key = self.gen_database_key(database["Name"])
        parameters = database.get("Parameters", {})
        if database.get("LocationUri") is not None:
            parameters["LocationUri"] = database["LocationUri"]
        if database.get("CreateTime") is not None:
            create_time: datetime.datetime = database["CreateTime"]
            parameters["CreateTime"] = create_time.strftime("%B %d, %Y at %H:%M:%S")
        yield from gen_containers(
            container_key=database_container_key,
            name=database["Name"],
            sub_types=[DatasetContainerSubTypes.DATABASE],
            domain_urn=domain_urn,
            description=database.get("Description"),
            qualified_name=self.get_glue_arn(
                account_id=database["CatalogId"], database=database["Name"]
            ),
            tags=container_tags,
            extra_properties=parameters,
        )

    def add_table_to_database_container(
        self, dataset_urn: str, db_name: str
    ) -> Iterable[MetadataWorkUnit]:
        database_container_key = self.gen_database_key(db_name)
        yield from add_dataset_to_container(
            container_key=database_container_key,
            dataset_urn=dataset_urn,
        )

    def _gen_domain_urn(self, dataset_name: str) -> Optional[str]:
        for domain, pattern in self.source_config.domain.items():
            if pattern.allowed(dataset_name):
                return make_domain_urn(domain)

        return None

    def _get_domain_wu(
        self, dataset_name: str, entity_urn: str
    ) -> Iterable[MetadataWorkUnit]:
        domain_urn = self._gen_domain_urn(dataset_name)
        if domain_urn:
            yield from add_domain_to_entity_wu(
                entity_urn=entity_urn,
                domain_urn=domain_urn,
            )

    def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
        databases, tables = self.get_all_databases_and_tables()

        for database in databases:
            yield from self.gen_database_containers(database)

        for table in tables:
            table_name = table["Name"]
            try:
                yield from self._gen_table_wu(table=table)
            except KeyError as e:
                self.report.failure(
                    message="Failed to extract workunit for table",
                    context=f"Table: {table_name}",
                    exc=e,
                )
        if self.extract_transforms:
            yield from self._transform_extraction()

        # Flush view lineage parsed from VIRTUAL_VIEW definitions. Deferred to here
        # so all table/view schemas are registered before the SQL is parsed.
        # This is the single funnel for view lineage, so a failure here must be
        # reported rather than aborting the run after tables/views were emitted.
        if self.aggregator:
            try:
                for mcp in self.aggregator.gen_metadata():
                    yield mcp.as_workunit()
            except Exception as e:
                self.report.failure(
                    message="Failed to generate view lineage from SQL parsing",
                    context="SqlParsingAggregator.gen_metadata",
                    exc=e,
                )

    def _gen_table_wu(self, table: Dict) -> Iterable[MetadataWorkUnit]:
        database_name = table["DatabaseName"]
        table_name = table["Name"]
        full_table_name = f"{database_name}.{table_name}"
        self.report.report_table_scanned()
        if not self.source_config.database_pattern.allowed(
            database_name
        ) or not self.source_config.table_pattern.allowed(full_table_name):
            self.report.report_table_dropped(full_table_name)
            return

        # Stamp the owning catalog's instance/env (cross-account tables resolve to the owner's
        # instance, not the ingestion account's) so URNs match the owner's own Glue ingestion.
        resolved = self._resolve_platform_instance(table.get("CatalogId"))
        dataset_urn = make_dataset_urn_with_platform_instance(
            platform=self.platform,
            name=full_table_name,
            env=resolved.env,
            platform_instance=resolved.platform_instance,
        )

        yield from self._extract_record(dataset_urn, table, full_table_name)

        is_view = table.get("TableType") == self._VIRTUAL_VIEW_TABLE_TYPE

        # We assign the "Table"/"View" subType to the dataset representing the glue
        # table - unfortunately it is not possible via the Dataset snapshot embedded
        # in an mce, so we have to generate a separate mcp.
        yield MetadataChangeProposalWrapper(
            entityUrn=dataset_urn,
            aspect=SubTypes(
                typeNames=[DatasetSubTypes.VIEW if is_view else DatasetSubTypes.TABLE]
            ),
        ).as_workunit()

        if is_view:
            self.report.report_view_scanned()
            view_definition = self._get_view_definition(table, full_table_name)
            yield MetadataChangeProposalWrapper(
                entityUrn=dataset_urn,
                aspect=ViewPropertiesClass(
                    materialized=False,
                    viewLanguage="SQL",
                    viewLogic=view_definition.sql if view_definition else "",
                ),
            ).as_workunit()
            if view_definition and self.aggregator:
                # Glue is a two-level namespace (database.table); the view's Glue
                # database is the default schema for resolving unqualified tables.
                # Parse with the view's own dialect (Trino for Athena/Presto views,
                # Spark for raw Spark/Hive views) rather than the platform default.
                self.aggregator.add_view_definition(
                    view_urn=dataset_urn,
                    view_definition=view_definition.sql,
                    default_db=None,
                    default_schema=database_name,
                    override_dialect=view_definition.dialect,
                )

        yield from self._get_domain_wu(
            dataset_name=full_table_name,
            entity_urn=dataset_urn,
        )
        yield from self.add_table_to_database_container(
            dataset_urn=dataset_urn, db_name=database_name
        )

    def _get_view_definition(
        self, table: Dict, full_table_name: str
    ) -> Optional[GlueViewDefinition]:
        """Extract the SQL definition (and dialect) of a Glue VIRTUAL_VIEW table.

        Decodes the Presto/Athena base64 encoding when present (Trino dialect),
        otherwise returns the raw text as a Spark view, falling back to
        ViewExpandedText. Returns ``None`` (and reports a warning) when no usable
        SQL can be extracted, so the caller never persists an unusable encoded blob
        as the view definition.
        """
        view_text = table.get("ViewOriginalText") or table.get("ViewExpandedText")
        if not isinstance(view_text, str) or not view_text:
            self.report.warning(
                message="View has no SQL definition",
                context=f"table={full_table_name}",
                log=False,
            )
            return None

        if is_presto_view(view_text):
            # A failed decode leaves only the encoded marker, which is not usable
            # SQL, so we return None rather than store the raw blob as viewLogic.
            try:
                original_sql = decode_presto_view(view_text).get("originalSql")
            except Exception as e:
                self.report.warning(
                    message="Failed to decode Presto view definition",
                    context=f"table={full_table_name}: {e}",
                    log=False,
                )
                return None
            if isinstance(original_sql, str) and original_sql:
                return GlueViewDefinition(
                    sql=original_sql, dialect=self._PRESTO_VIEW_DIALECT
                )
            self.report.warning(
                message="Presto view definition missing 'originalSql'",
                context=f"table={full_table_name}",
                log=False,
            )
            return None

        return GlueViewDefinition(sql=view_text, dialect=self._RAW_VIEW_DIALECT)

    def _transform_extraction(self) -> Iterable[MetadataWorkUnit]:
        for job in self.get_all_jobs():
            flow_urn = mce_builder.make_data_flow_urn(
                self.platform, job["Name"], self.env
            )

            yield from self.get_dataflow_wus(flow_urn, job)

            job_script_location = job.get("Command", {}).get("ScriptLocation")

            job_script: Optional[str] = None
            if job_script_location is not None:
                job_script = self.get_dataflow_script(job_script_location, flow_urn)
            else:
                self.report.num_job_script_location_missing += 1

            dag: Optional[Dict[str, Any]] = None
            if job_script:
                dag = self.get_dataflow_graph(job_script, job_script_location, flow_urn)

            nodes: Optional[dict] = None
            if dag is not None:
                nodes = self.process_dataflow_graph(dag, flow_urn)
                if not nodes:
                    self.report.num_job_without_nodes += 1

            if not nodes:
                yield from self.get_datajob_wus_for_dataflow(
                    flow_urn, job["Name"], job_script
                )
                continue

            for node in nodes.values():
                if node["NodeType"] not in ["DataSource", "DataSink"]:
                    yield self.get_datajob_wu(node, job["Name"])
                elif (node["NodeType"] == "DataSource" and node["outputDatasets"]) or (
                    node["NodeType"] == "DataSink" and node["inputDatasets"]
                ):
                    # Not common, but capturing counts here for reporting
                    self.report.num_dataset_to_dataset_edges_in_job += 1

    def _extract_record(
        self, dataset_urn: str, table: Dict, table_name: str
    ) -> Iterable[MetadataWorkUnit]:
        """Extract and yield metadata work units for a Glue table."""
        logger.debug(
            f"extract record from table={table_name} for dataset={dataset_urn}"
        )

        # Create the main dataset snapshot
        dataset_properties = self._get_dataset_properties(table)
        dataset_snapshot = DatasetSnapshot(
            urn=dataset_urn,
            aspects=[
                Status(removed=False),
                dataset_properties,
            ],
        )

        # Add operations if available
        if dataset_properties.created:
            yield MetadataChangeProposalWrapper(
                entityUrn=dataset_urn,
                aspect=OperationClass(
                    operationType=OperationTypeClass.CREATE,
                    lastUpdatedTimestamp=dataset_properties.created.time,
                    timestampMillis=dataset_properties.created.time,
                ),
            ).as_workunit()
        if (
            dataset_properties.lastModified
            and dataset_properties.lastModified != dataset_properties.created
        ):
            yield MetadataChangeProposalWrapper(
                entityUrn=dataset_urn,
                aspect=OperationClass(
                    operationType=OperationTypeClass.UPDATE,
                    lastUpdatedTimestamp=dataset_properties.lastModified.time,
                    timestampMillis=dataset_properties.lastModified.time,
                ),
            ).as_workunit()

        # Fetch Lake Formation tags once: a single call returns the database's
        # tags, the table's tags, and the columns' tags (see get_table_lf_tags).
        table_lf_tags: Optional[TableLakeFormationTags] = None
        resolved_table_tags = ResolvedLakeFormationTags(direct=[], propagated=[])
        if self.source_config.extract_lakeformation_tags:
            table_lf_tags = self.get_table_lf_tags(
                catalog_id=table["CatalogId"],
                database_name=table["DatabaseName"],
                table_name=table["Name"],
            )
            resolved_table_tags = self._resolve_table_lf_tags(table_lf_tags)

        # Add schema metadata if available. When propagation is enabled, the
        # table's effective tags (its own + inherited database tags) are
        # inherited by every column (matching Lake Formation's inheritance),
        # marked as propagated, with a column's direct tags taking precedence
        # for the same key.
        column_lf_tags = (
            table_lf_tags.column_tags
            if table_lf_tags and self.source_config.extract_lakeformation_column_tags
            else None
        )
        column_parent_tags = (
            [*resolved_table_tags.direct, *resolved_table_tags.propagated]
            if table_lf_tags
            and self.source_config.extract_lakeformation_column_tags
            and self.source_config.propagate_lakeformation_tags
            else None
        )
        schema_metadata = self._get_schema_metadata(
            table,
            table_name,
            dataset_urn,
            column_lf_tags,
            column_parent_tags,
            dataset_urn,
        )
        if schema_metadata is None:
            # Schemaless so far — try to populate a Lake Formation resource link from its owner.
            schema_metadata = self._resource_link_schema(table, dataset_urn, table_name)
        if schema_metadata:
            dataset_snapshot.aspects.append(schema_metadata)
            # Register every table/view schema so the aggregator can resolve
            # column-level lineage when it parses view definitions.
            if self.aggregator:
                self.aggregator.register_schema(dataset_urn, schema_metadata)

        # Add platform instance
        dataset_snapshot.aspects.append(self._get_data_platform_instance())

        # Add ownership if enabled
        if self.extract_owners:
            ownership = GlueSource._get_ownership(table.get("Owner"))
            if ownership:
                dataset_snapshot.aspects.append(ownership)

        # Add S3 tags if enabled
        s3_tags = self._get_s3_tags(table, dataset_urn)
        if s3_tags:
            dataset_snapshot.aspects.append(s3_tags)

        # Add Lake Formation tags if enabled
        if table_lf_tags is not None:
            # Tags inherited from the database are marked as propagated, with the
            # database container as their origin.
            db_container_urn = self.gen_database_key(table["DatabaseName"]).as_urn()
            global_tags = self._build_lf_global_tags(
                resolved_table_tags.direct,
                resolved_table_tags.propagated,
                propagated_origin=db_container_urn,
            )
            if global_tags:
                dataset_snapshot.aspects.append(global_tags)
                # Generate platform resources for table-level LF tags
                for tag in [
                    *resolved_table_tags.direct,
                    *resolved_table_tags.propagated,
                ]:
                    yield from self.gen_platform_resource(tag)
            # Generate platform resources for column-level LF tags
            if column_lf_tags:
                for column_tag in {
                    tag for tags in column_lf_tags.values() for tag in tags
                }:
                    yield from self.gen_platform_resource(column_tag)

        # Create and yield the main metadata work unit
        metadata_record = MetadataChangeEvent(proposedSnapshot=dataset_snapshot)
        yield MetadataWorkUnit(table_name, mce=metadata_record)

        if self.source_config.extract_column_parameters:
            yield from self._get_column_param_workunits(table, dataset_urn)

        # Pass the resource-link table-level edge to get_lineage_if_enabled so it merges with any
        # storage lineage into one aspect rather than being emitted as a competing one.
        yield from self.get_lineage_if_enabled(
            metadata_record,
            self._resource_link_owner_upstreams(table, dataset_urn),
        )

        # Add profile if enabled
        try:
            yield from self.get_profile_if_enabled(
                metadata_record, table["DatabaseName"], table["Name"]
            )
        except KeyError as e:
            self.report.failure(
                message="Failed to extract profile for table",
                context=f"Table: {dataset_urn}",
                exc=e,
            )

    def _get_dataset_properties(self, table: Dict) -> DatasetPropertiesClass:
        """Extract dataset properties from Glue table."""
        storage_descriptor = table.get("StorageDescriptor", {})
        custom_properties = {
            **table.get("Parameters", {}),
            **{
                k: str(v)
                for k, v in storage_descriptor.items()
                if k not in ["Columns", "Parameters"]
            },
        }

        created = None
        if table.get("CreateTime"):
            created_ts = make_ts_millis(
                table["CreateTime"].replace(tzinfo=datetime.timezone.utc)
            )
            if created_ts is not None:
                created = TimeStampClass(created_ts)

        last_modified = None
        if table.get("UpdateTime"):
            updated_ts = make_ts_millis(
                table["UpdateTime"].replace(tzinfo=datetime.timezone.utc)
            )
            if updated_ts is not None:
                last_modified = TimeStampClass(updated_ts)

        return DatasetPropertiesClass(
            description=table.get("Description"),
            customProperties=custom_properties,
            uri=table.get("Location"),
            tags=[],
            name=table["Name"],
            qualifiedName=self.get_glue_arn(
                account_id=table["CatalogId"],
                database=table["DatabaseName"],
                table=table["Name"],
            ),
            created=created,
            lastModified=last_modified,
        )

    def _get_schema_metadata(
        self,
        table: Dict,
        table_name: str,
        dataset_urn: str,
        column_tags: Optional[Dict[str, List[LakeFormationTag]]] = None,
        column_parent_tags: Optional[List[LakeFormationTag]] = None,
        column_parent_origin: Optional[str] = None,
    ) -> Optional[SchemaMetadata]:
        """Extract schema metadata from Glue table."""
        if not table.get("StorageDescriptor"):
            return None

        # Check if this is a delta table with schema in parameters
        if self._is_delta_schema(table):
            return self._get_delta_schema_metadata(
                table,
                table_name,
                dataset_urn,
                column_tags,
                column_parent_tags,
                column_parent_origin,
            )
        else:
            return self._get_glue_schema_metadata(
                table,
                table_name,
                column_tags,
                column_parent_tags,
                column_parent_origin,
            )

    def _apply_column_lf_tags(
        self,
        schema_fields: List[SchemaField],
        column_name: str,
        column_tags: Optional[Dict[str, List[LakeFormationTag]]],
        parent_tags: Optional[List[LakeFormationTag]] = None,
        parent_origin: Optional[str] = None,
    ) -> None:
        """Attach a column's LF tags to its root schema field (in place).

        ``get_schema_fields_for_hive_column`` returns the column's own field
        first, followed by any nested struct/array children, so the tag is
        applied to ``schema_fields[0]``.

        ``parent_tags`` are the table/database tags inherited by every column
        when database-tag propagation is enabled (matching Lake Formation's
        inheritance); they are marked as propagated. A tag the column assigns
        directly overrides an inherited value for the same key.
        """
        if not schema_fields:
            return
        direct = list((column_tags or {}).get(column_name, []))
        propagated: List[LakeFormationTag] = []
        if parent_tags:
            column_keys = {str(tag.key) for tag in direct}
            for parent_tag in parent_tags:
                if str(parent_tag.key) not in column_keys:
                    propagated.append(parent_tag)
                    column_keys.add(str(parent_tag.key))
        global_tags = self._build_lf_global_tags(
            direct, propagated, propagated_origin=parent_origin
        )
        if global_tags:
            schema_fields[0].globalTags = global_tags

    def _is_delta_schema(self, table: Dict) -> bool:
        """Check if table uses delta format with schema in parameters."""
        if not self.source_config.extract_delta_schema_from_parameters:
            return False

        provider = table.get("Parameters", {}).get("spark.sql.sources.provider", "")
        num_parts = int(
            table.get("Parameters", {}).get("spark.sql.sources.schema.numParts", "0")
        )
        columns = table.get("StorageDescriptor", {}).get("Columns", [])

        return (
            provider == "delta"
            and num_parts > 0
            and columns
            and len(columns) == 1
            and columns[0].get("Name", "") == "col"
            and columns[0].get("Type", "") == "array<string>"
        )

    def _get_glue_schema_metadata(
        self,
        table: Dict,
        table_name: str,
        column_tags: Optional[Dict[str, List[LakeFormationTag]]] = None,
        column_parent_tags: Optional[List[LakeFormationTag]] = None,
        column_parent_origin: Optional[str] = None,
    ) -> Optional[SchemaMetadata]:
        """Extract schema metadata from Glue table columns."""
        schema = table["StorageDescriptor"]["Columns"]
        fields: List[SchemaField] = []

        # Process regular columns
        for field in schema:
            schema_fields = get_schema_fields_for_hive_column(
                hive_column_name=field["Name"],
                hive_column_type=field["Type"],
                description=field.get("Comment"),
                default_nullable=True,
            )
            if schema_fields:
                self._apply_column_lf_tags(
                    schema_fields,
                    field["Name"],
                    column_tags,
                    column_parent_tags,
                    column_parent_origin,
                )
                fields.extend(schema_fields)

        # Process partition keys
        partition_keys = table.get("PartitionKeys", [])
        for partition_key in partition_keys:
            schema_fields = get_schema_fields_for_hive_column(
                hive_column_name=partition_key["Name"],
                hive_column_type=partition_key.get("Type", "unknown"),
                description=partition_key.get("Comment"),
                default_nullable=False,
            )
            if schema_fields:
                self._apply_column_lf_tags(
                    schema_fields,
                    partition_key["Name"],
                    column_tags,
                    column_parent_tags,
                    column_parent_origin,
                )
                fields.extend(schema_fields)

        return SchemaMetadata(
            schemaName=table_name,
            version=0,
            fields=fields,
            platform=f"urn:li:dataPlatform:{self.platform}",
            hash="",
            platformSchema=MySqlDDL(tableSchema=""),
        )

    def _get_delta_schema_metadata(
        self,
        table: Dict,
        table_name: str,
        dataset_urn: str,
        column_tags: Optional[Dict[str, List[LakeFormationTag]]] = None,
        column_parent_tags: Optional[List[LakeFormationTag]] = None,
        column_parent_origin: Optional[str] = None,
    ) -> Optional[SchemaMetadata]:
        """Extract schema metadata from Delta table parameters."""
        try:
            # Reconstruct schema from parameters
            num_parts = int(table["Parameters"]["spark.sql.sources.schema.numParts"])
            schema_str = "".join(
                table["Parameters"][f"spark.sql.sources.schema.part.{i}"]
                for i in range(num_parts)
            )
            schema_json = json.loads(schema_str)

            fields: List[SchemaField] = []
            for field in schema_json["fields"]:
                field_type = delta_type_to_hive_type(field.get("type", "unknown"))
                schema_fields = get_schema_fields_for_hive_column(
                    hive_column_name=field["name"],
                    hive_column_type=field_type,
                    description=field.get("description"),
                    default_nullable=bool(field.get("nullable", True)),
                )
                if schema_fields:
                    self._apply_column_lf_tags(
                        schema_fields,
                        field["name"],
                        column_tags,
                        column_parent_tags,
                        column_parent_origin,
                    )
                    fields.extend(schema_fields)

            self.report.num_dataset_valid_delta_schema += 1
            return SchemaMetadata(
                schemaName=table_name,
                version=0,
                fields=fields,
                platform=f"urn:li:dataPlatform:{self.platform}",
                hash="",
                platformSchema=MySqlDDL(tableSchema=""),
            )

        except Exception as e:
            self.report.warning(
                message="Could not parse delta schema for table",
                context=f"{dataset_urn}: {table_name}",
                exc=e,
                log=False,
            )
            self.report.num_dataset_invalid_delta_schema += 1
            return None

    def _get_data_platform_instance(self) -> DataPlatformInstanceClass:
        """Get data platform instance aspect."""
        return DataPlatformInstanceClass(
            platform=make_data_platform_urn(self.platform),
            instance=(
                make_dataplatform_instance_urn(
                    self.platform, self.source_config.platform_instance
                )
                if self.source_config.platform_instance
                else None
            ),
        )

    @staticmethod
    @lru_cache(maxsize=None)
    def _get_ownership(owner: str) -> Optional[OwnershipClass]:
        """Get ownership aspect for a given owner."""
        if not owner:
            return None

        owners = [
            OwnerClass(
                owner=mce_builder.make_user_urn(owner),
                type=OwnershipTypeClass.DATAOWNER,
            )
        ]
        return OwnershipClass(owners=owners)

    @staticmethod
    def _column_param_property_urn(key: str) -> str:
        qualified_name = (
            f"io.datahubproject.glue.column.{re.sub(r'[^a-zA-Z0-9._]', '_', key)}"
        )
        return f"urn:li:structuredProperty:{qualified_name}"

    def _get_column_param_workunits(
        self, table: Dict, dataset_urn: str
    ) -> Iterable[MetadataWorkUnit]:
        columns = table.get("StorageDescriptor", {}).get("Columns", [])
        columns = columns + table.get("PartitionKeys", [])
        for column in columns:
            try:
                params = column.get("Parameters")
                if not params:
                    continue
                schema_fields = get_schema_fields_for_hive_column(
                    hive_column_name=column["Name"],
                    hive_column_type=column.get("Type", "string"),
                    description=column.get("Comment"),
                    default_nullable=True,
                )
                if not schema_fields:
                    continue
                field_urn = mce_builder.make_schema_field_urn(
                    dataset_urn, schema_fields[0].fieldPath
                )
                assignments = []
                for key, value in params.items():
                    property_urn = self._column_param_property_urn(key)
                    qualified_name = property_urn.removeprefix(
                        "urn:li:structuredProperty:"
                    )
                    if property_urn not in self._seen_column_param_urns:
                        definition_mcp = MetadataChangeProposalWrapper(
                            entityUrn=property_urn,
                            aspect=StructuredPropertyDefinitionClass(
                                qualifiedName=qualified_name,
                                displayName=key,
                                valueType="urn:li:dataType:datahub.string",
                                entityTypes=["urn:li:entityType:datahub.schemaField"],
                            ),
                        )
                        if self.ctx.graph is not None:
                            # Emit synchronously so GMS persists the definition
                            # before the structuredProperties MCP below is
                            # validated — GMS rejects assignments that reference
                            # a definition not yet in the database.
                            self.ctx.graph.emit_mcp(
                                definition_mcp, emit_mode=EmitMode.SYNC_PRIMARY
                            )
                        else:
                            yield definition_mcp.as_workunit()
                        self._seen_column_param_urns.add(property_urn)
                    assignments.append(
                        StructuredPropertyValueAssignmentClass(
                            propertyUrn=property_urn,
                            values=[value],
                        )
                    )
                if assignments:
                    yield MetadataChangeProposalWrapper(
                        entityUrn=field_urn,
                        aspect=StructuredPropertiesClass(properties=assignments),
                    ).as_workunit()
            except Exception as e:
                self.report.warning(
                    message="Failed to emit column parameters for column",
                    context=f"dataset={dataset_urn} column={column.get('Name', '?')!r}: {e}",
                    log=False,
                )

    def _get_s3_tags(self, table: Dict, dataset_urn: str) -> Optional[GlobalTagsClass]:
        """Extract S3 tags if enabled."""
        if not (
            self.source_config.use_s3_bucket_tags
            or self.source_config.use_s3_object_tags
        ):
            return None

        # Check if table has a location (VIRTUAL_VIEW tables may not)
        location = table.get("StorageDescriptor", {}).get("Location")
        if not location:
            return None

        bucket_name = s3_util.get_bucket_name(location)
        tags_to_add: List[str] = []

        # Get bucket tags
        if self.source_config.use_s3_bucket_tags:
            try:
                bucket_tags = self.s3_client.get_bucket_tagging(Bucket=bucket_name)
                tags_to_add.extend(
                    make_tag_urn(f"{tag['Key']}:{tag['Value']}")
                    for tag in bucket_tags["TagSet"]
                )
            except self.s3_client.exceptions.ClientError:
                logger.warning(f"No tags found for bucket={bucket_name}")

        # Get object tags
        if self.source_config.use_s3_object_tags:
            key_prefix = s3_util.get_key_prefix(location)
            try:
                object_tagging = self.s3_client.get_object_tagging(
                    Bucket=bucket_name, Key=key_prefix
                )
                if object_tagging["TagSet"]:
                    tags_to_add.extend(
                        make_tag_urn(f"{tag['Key']}:{tag['Value']}")
                        for tag in object_tagging["TagSet"]
                    )
                else:
                    logger.warning(
                        f"No tags found for bucket={bucket_name} key={key_prefix}"
                    )
            except Exception as e:
                logger.warning(f"Failed to get object tags: {e}")

        if not tags_to_add:
            return None

        # Merge with existing tags if connected to DataHub API
        if self.ctx.graph:
            logger.debug("Connected to DatahubApi, grabbing current tags to maintain.")
            current_tags: Optional[GlobalTagsClass] = self.ctx.graph.get_aspect(
                entity_urn=dataset_urn, aspect_type=GlobalTagsClass
            )
            if current_tags:
                tags_to_add.extend(current_tag.tag for current_tag in current_tags.tags)
        else:
            logger.warning(
                "Could not connect to DatahubApi. No current tags to maintain"
            )

        # Remove duplicates and create tags
        unique_tags = sorted(set(tags_to_add))
        return GlobalTagsClass(tags=[TagAssociationClass(tag) for tag in unique_tags])

    @staticmethod
    def _to_tag_urns(tags: List[LakeFormationTag]) -> List[str]:
        urns: List[str] = []
        for tag in tags:
            try:
                urns.append(tag.to_datahub_tag_urn().urn())
            except InvalidUrnError as e:
                logger.warning(
                    f"Invalid Lake Formation tag URN for {tag}: {e}", exc_info=True
                )
                continue  # Skip invalid tags
        return urns

    def _propagated_tag_attribution(
        self, origin: Optional[str]
    ) -> Tuple[MetadataAttributionClass, str]:
        """Build the attribution + legacy context marking a tag as propagated.

        Mirrors the DataHub tag-propagation action: ``sourceDetail.propagated``
        drives the "Propagated" indicator in the UI, and ``context`` carries the
        same information for older readers.
        """
        source_detail: Dict[str, str] = {"propagated": "true"}
        if origin:
            source_detail["origin"] = origin
        attribution = MetadataAttributionClass(
            time=get_sys_time(),
            actor="urn:li:corpuser:__datahub_system",
            source=origin,
            sourceDetail=source_detail,
        )
        return attribution, json.dumps(source_detail)

    def _build_lf_global_tags(
        self,
        direct_tags: List[LakeFormationTag],
        propagated_tags: Optional[List[LakeFormationTag]] = None,
        propagated_origin: Optional[str] = None,
    ) -> Optional[GlobalTagsClass]:
        """Build a GlobalTags aspect, marking propagated tags with attribution.

        Direct tags take precedence: a propagated tag is skipped if the same tag
        URN is already assigned directly.
        """
        associations: Dict[str, TagAssociationClass] = {}
        for urn in self._to_tag_urns(direct_tags):
            associations.setdefault(urn, TagAssociationClass(tag=urn))

        if propagated_tags:
            attribution, context = self._propagated_tag_attribution(propagated_origin)
            for urn in self._to_tag_urns(propagated_tags):
                if urn not in associations:
                    associations[urn] = TagAssociationClass(
                        tag=urn, context=context, attribution=attribution
                    )

        if not associations:
            return None
        # Sort for stable, deterministic output.
        return GlobalTagsClass(tags=[associations[urn] for urn in sorted(associations)])

    def get_report(self):
        return self.report


def _redact_secret_fields_in_dataflow_script(script: str) -> str:
    # It's possible for Glue Job nodes to contain sensitive values in their connection_info
    # arguments. Redact all string values that are preceded by a key containing "password" or
    # "secret".
    return re.sub(
        r'(?i)("(\\"|[^"])*(?:password|secret)(\\"|[^"])*"\s*:\s*)"(\\"|[^"])*"',
        r'\1"*****"',
        script,
    )
