from ws_package.repositories.server_repo import server_manager
import sys
import os
import json
import gzip
import io
import pkgutil
from pathlib import Path

# Fix botocore data path for PyInstaller (must be done before importing boto3)
# This ensures botocore can find its data files (endpoints.json, etc.) in the PyInstaller bundle
if getattr(sys, 'frozen', False):
    try:
        import botocore
        # In PyInstaller, botocore data files should be in the bundle
        # Try multiple possible locations where PyInstaller might place them
        possible_paths = [
            Path(botocore.__file__).parent / 'data',  # Standard location (should work if bundled correctly)
            Path(sys._MEIPASS) / 'botocore' / 'data' if hasattr(sys, '_MEIPASS') else None,  # PyInstaller temp folder (all platforms)
            Path(sys.executable).parent / 'botocore' / 'data',  # Next to executable (onedir mode, all platforms)
            Path(sys._MEIPASS) / 'data' / 'botocore' if hasattr(sys, '_MEIPASS') else None,  # Alternative location (all platforms)
            Path(sys.executable).parent / '_internal' / 'botocore' / 'data',  # Windows onedir _internal folder (harmless on Mac/Linux if not exists)
        ]
        
        # Filter out None paths
        possible_paths = [p for p in possible_paths if p is not None]
        
        botocore_data_path = None
        for path in possible_paths:
            if path.exists() and path.is_dir():
                # Check if endpoints.json exists (key file botocore needs)
                endpoints_file = path / 'endpoints.json'
                if endpoints_file.exists():
                    botocore_data_path = path
                    break
        
        # If still not found, try to find any botocore data directory
        if botocore_data_path is None:
            for path in possible_paths:
                if path.exists() and path.is_dir():
                    # Check if directory has any .json files (indicates it's a data directory)
                    json_files = list(path.glob('*.json'))
                    if json_files:
                        botocore_data_path = path
                        break
        
        # Last resort: try to find botocore data in the original installation
        if botocore_data_path is None:
            try:
                import botocore
                original_data_path = Path(botocore.__file__).parent / 'data'
                if original_data_path.exists() and (original_data_path / 'endpoints.json').exists():
                    botocore_data_path = original_data_path
            except (ImportError, AttributeError):
                pass
        
        if botocore_data_path:
            # Set environment variable to help botocore find data files
            os.environ['BOTO_DATA_PATH'] = str(botocore_data_path)
            
            # Log success (only if logger is available)
            try:
                from ws_package.log.writer import LogWriter
                logger = LogWriter.file(__name__)
                logger.debug("Found botocore data files at: %s", botocore_data_path)
            except Exception:
                pass
            
            def _build_search_paths():
                """Return every directory where botocore data might live (deduped)."""
                search_paths = []
                seen = set()

                def add_path(path):
                    if not path:
                        return
                    path = Path(path)
                    try:
                        resolved = path.resolve()
                    except Exception:
                        resolved = path
                    if path.exists() and resolved not in seen:
                        search_paths.append(path)
                        seen.add(resolved)

                add_path(botocore_data_path)

                if hasattr(sys, '_MEIPASS'):
                    meipass_path = Path(sys._MEIPASS)
                    add_path(meipass_path / 'botocore' / 'data')
                    add_path(meipass_path / 'data' / 'botocore')
                    add_path(meipass_path / 'data')

                if hasattr(sys, 'executable'):
                    exe_parent = Path(sys.executable).parent
                    add_path(exe_parent / 'botocore' / 'data')
                    add_path(exe_parent / '_internal' / 'botocore' / 'data')

                try:
                    import botocore as _botocore_mod
                    botocore_base = Path(_botocore_mod.__file__).parent
                    add_path(botocore_base / 'data')
                    add_path(botocore_base.parent / 'botocore' / 'data')
                    add_path(botocore_base / 'botocore' / 'data')
                except Exception:
                    pass

                return search_paths

            def _candidate_names_for(name: str):
                """Generate all filename variants for the requested resource."""
                normalized = name.replace('\\', '/')
                candidates = []
                seen = set()

                def add(value: str):
                    if not value:
                        return
                    if value not in seen:
                        seen.add(value)
                        candidates.append(value)

                add(normalized)
                if normalized != name:
                    add(name)
                add(f"{normalized}.json")
                add(f"{normalized}.json.bak")
                add(f"{normalized}.sdk-extras")
                add(f"{normalized}.sdk-extras.json")
                add(f"{normalized}.json.gz")
                add(f"{normalized}.sdk-extras.json.gz")

                if normalized.endswith('.sdk-extras'):
                    base_without = normalized[: -len('.sdk-extras')]
                    add(base_without)
                    add(f"{base_without}.json")
                    add(f"{base_without}.json.gz")
                    add(f"{base_without}.sdk-extras.json")

                return candidates

            def _read_json_from_path(data_file: Path):
                """Read JSON data from a file path, handling optional gzip."""
                try:
                    if data_file.suffix == '.gz':
                        with gzip.open(data_file, 'rt', encoding='utf-8') as fh:
                            return json.load(fh)
                    with open(data_file, 'r', encoding='utf-8') as fh:
                        return json.load(fh)
                except Exception:
                    return None

            def _read_json_from_bytes(candidate_name: str, data_bytes: bytes):
                """Read JSON data from in-memory bytes (optionally gzipped)."""
                try:
                    if candidate_name.endswith('.gz'):
                        with gzip.GzipFile(fileobj=io.BytesIO(data_bytes)) as gz:
                            return json.loads(gz.read().decode('utf-8'))
                    return json.loads(data_bytes.decode('utf-8'))
                except Exception:
                    return None

            def _resolve_botocore_data_file(name: str):
                """Try every known location to load the requested botocore data file."""
                search_paths = _build_search_paths()
                candidates = _candidate_names_for(name)

                for search_path in search_paths:
                    if not search_path or not search_path.exists():
                        continue

                    # Direct candidate lookup
                    for candidate in candidates:
                        data = _read_json_from_path(search_path / candidate)
                        if data is not None:
                            return data

                    # Recursive lookup for filename variants
                    try:
                        filename = Path(name.replace('\\', '/')).name
                    except Exception:
                        filename = name

                    filename_variants = [
                        filename,
                        f"{filename}.json",
                        f"{filename}.sdk-extras",
                        f"{filename}.json.gz",
                        f"{filename}.sdk-extras.json",
                    ]
                    if filename.endswith('.sdk-extras'):
                        stripped = filename[: -len('.sdk-extras')]
                        filename_variants.extend([
                            stripped,
                            f"{stripped}.json",
                            f"{stripped}.json.gz",
                            f"{stripped}.sdk-extras.json",
                        ])

                    for variant in filename_variants:
                        try:
                            found_files = list(search_path.rglob(variant))
                        except Exception:
                            continue
                        for data_file in found_files:
                            data = _read_json_from_path(data_file)
                            if data is not None:
                                return data

                # Exhaustive search inside _MEIPASS
                if hasattr(sys, '_MEIPASS'):
                    meipass_path = Path(sys._MEIPASS)
                    for candidate in candidates:
                        data = _read_json_from_path(meipass_path / candidate)
                        if data is not None:
                            return data

                    filename = Path(name.replace('\\', '/')).name
                    filename_variants = [
                        filename,
                        f"{filename}.json",
                        f"{filename}.sdk-extras",
                        f"{filename}.json.gz",
                        f"{filename}.sdk-extras.json",
                        name,
                    ]
                    if filename.endswith('.sdk-extras'):
                        stripped = filename[: -len('.sdk-extras')]
                        filename_variants.extend([
                            stripped,
                            f"{stripped}.json",
                            f"{stripped}.json.gz",
                            f"{stripped}.sdk-extras.json",
                        ])

                    for variant in filename_variants:
                        try:
                            found_files = list(meipass_path.rglob(variant))
                        except Exception:
                            continue
                        for data_file in found_files:
                            data = _read_json_from_path(data_file)
                            if data is not None:
                                return data

                # pkgutil fallback (for resources inside botocore's package/zip)
                for candidate in candidates:
                    resource_path = candidate if candidate.startswith('data/') else f"data/{candidate}"
                    try:
                        data_bytes = pkgutil.get_data('botocore', resource_path)
                    except Exception:
                        data_bytes = None
                    if not data_bytes:
                        continue
                    data = _read_json_from_bytes(candidate, data_bytes)
                    if data is not None:
                        return data

                raise FileNotFoundError(f"Could not find botocore data file '{name}' in any search path")
            
            # Patch botocore's loader to find data files in PyInstaller bundle
            # We need to patch both the class method and the module-level function
            try:
                import botocore.loaders as loaders_module
                from botocore.loaders import Loader
                
                # Patch the module-level function load_data_with_path (most important)
                if hasattr(loaders_module, 'load_data_with_path'):
                    original_module_func = loaders_module.load_data_with_path
                    
                    def patched_module_func(name):
                        """Patched module-level function that checks PyInstaller bundle paths"""
                        try:
                            return original_module_func(name)
                        except Exception as e:
                            try:
                                return _resolve_botocore_data_file(name)
                            except Exception:
                                raise e
                    
                    loaders_module.load_data_with_path = patched_module_func
                
                # Also patch the class method (for instances)
                original_load_data_with_path = Loader.load_data_with_path
                
                def patched_load_data_with_path(self, name):
                    """Patched loader method that checks PyInstaller bundle paths"""
                    try:
                        # Try original loader first
                        return original_load_data_with_path(self, name)
                    except Exception as e:
                        try:
                            return _resolve_botocore_data_file(name)
                        except Exception:
                            raise e
                
                # Apply the patch to the class method
                Loader.load_data_with_path = patched_load_data_with_path
                
                # Also patch the _wrapper method if it exists (used by some botocore versions)
                if hasattr(Loader, '_wrapper'):
                    original_wrapper = Loader._wrapper
                    
                    def patched_wrapper(self, name):
                        """Patched wrapper method that checks PyInstaller bundle paths"""
                        try:
                            return original_wrapper(self, name)
                        except Exception as e:
                            try:
                                return _resolve_botocore_data_file(name)
                            except Exception:
                                raise e
                    
                    Loader._wrapper = patched_wrapper
                
                # Patch the _search_paths method to include our data path
                if hasattr(Loader, '_search_paths'):
                    original_search_paths = Loader._search_paths
                    
                    def patched_search_paths(self):
                        """Patched search paths to include PyInstaller bundle path"""
                        paths = original_search_paths(self)
                        if botocore_data_path and botocore_data_path.exists():
                            # Add our path at the beginning of the search list
                            paths.insert(0, str(botocore_data_path))
                        return paths
                    
                    Loader._search_paths = patched_search_paths
                
                # CRITICAL: Patch load_data method to prevent None return values and unpacking errors
                # This is the method that's actually called and causing the unpacking error
                # The error occurs when load_data returns None and botocore tries to unpack it
                if hasattr(Loader, 'load_data'):
                    original_load_data = Loader.load_data
                    
                    def patched_load_data(self, name):
                        """Patched load_data to prevent None returns and handle PyInstaller paths on all platforms"""
                        try:
                            result = original_load_data(self, name)
                            if result is None:
                                return _resolve_botocore_data_file(name)
                            return result
                        except (TypeError, ValueError, AttributeError) as unpack_error:
                            error_str = str(unpack_error).lower()
                            if "unpack" in error_str or "non-iterable" in error_str or "nonetype" in error_str:
                                return _resolve_botocore_data_file(name)
                            raise
                        except Exception as e:
                            try:
                                return _resolve_botocore_data_file(name)
                            except Exception:
                                raise e

                    Loader.load_data = patched_load_data
                
                # Log successful patching
                try:
                    from ws_package.log.writer import LogWriter
                    logger = LogWriter.file(__name__)
                    logger.debug("Successfully patched botocore loader for PyInstaller bundle")
                except Exception:
                    pass
                
            except Exception as patch_error:
                # If patching fails, log it but continue (environment variable might still help)
                try:
                    from ws_package.log.writer import LogWriter
                    logger = LogWriter.file(__name__)
                    logger.warning("Failed to patch botocore loader: %s", patch_error)
                except Exception:
                    pass
        else:
            # Log warning - data files not found (only if logger is available)
            try:
                from ws_package.log.writer import LogWriter
                logger = LogWriter.file(__name__)
                logger.warning("botocore data files not found. Checked paths: %s", [str(p) for p in possible_paths])
                logger.warning("boto3 functionality may not work properly without botocore data files")
            except Exception:
                pass
    except (ImportError, AttributeError):
        pass  # botocore not available, will fail later

# NOTE: `import boto3` was previously at module level here. It cost ~1-2 s on
# Windows cold start because boto3's top-level init eagerly loads service
# models. We defer it to first use via _boto3_module() below. The botocore
# data-file patching above MUST stay at module level — it must run before any
# code imports boto3.
#
# `botocore.exceptions.ClientError` is lightweight (just exception classes;
# botocore itself is already loaded above for the patching) so we keep it eager
# — except clauses need a class at parse time, not a lazy callable.
import asyncio
import threading
from ws_package.log.writer import LogWriter
from botocore.exceptions import ClientError
from ws_package.utils.error_codes import AppError, ErrorCode
from ws_package.models.base import AppConfig
from ws_package.models.base import S3details  # Backward compatibility alias
from ws_package.utils.resource_recovery import resource_recovery
from ws_package.services.api_request_service import create_robust_ssl_context

# Lazy boto3 loader. First call ~1-2 s on Windows cold; subsequent calls ~0 ms.
_boto3_cached = None

def _boto3_module():
    """Return the boto3 module, importing it on first call only."""
    global _boto3_cached
    if _boto3_cached is None:
        import boto3 as _b
        _boto3_cached = _b
    return _boto3_cached


def _boto3_client(service_name, **kwargs):
    """Lazy wrapper for boto3.client(...). See _boto3_module() for rationale."""
    return _boto3_module().client(service_name, **kwargs)

# Suppress urllib3 SSL warnings when SSL verification is disabled
try:
    import urllib3
    urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
except ImportError:
    pass  # urllib3 not available, warnings will still appear

# ---------------------------------------------------------------------------
# S3 credential pre-fetch cache + upload drain queue
#
# Problem: after the onedir + ready-signal changes, the agent signals "ready"
# before the heavy init and auth complete.  If a monitoring session starts in
# that window and immediately takes a screenshot, the upload attempt calls
# s3_detail() — an API round-trip that hasn't been pre-warmed yet.
#
# Solution:
#   1. After auth, call prefetch_s3_credentials(org_id) to warm the cache.
#   2. s3_detail() returns the cached value instantly on subsequent calls.
#   3. Any upload attempted BEFORE prefetch completes is pushed to
#      _s3_upload_queue (max 50 items).  The prefetch callback drains the
#      queue inside the running event loop once credentials arrive.
# ---------------------------------------------------------------------------

_s3_credential_cache: dict = {}       # org_id -> AppConfig
_s3_cache_lock = threading.Lock()

_S3_QUEUE_MAX = 50
_s3_upload_queue: list = []           # list of (coroutine_factory, args, kwargs)
_s3_queue_lock = threading.Lock()
_s3_credentials_event: dict = {}      # org_id -> asyncio.Event (per event loop)


def _get_credentials_event(org_id) -> asyncio.Event:
    """Return (creating if needed) the asyncio.Event for org_id credential readiness."""
    with _s3_cache_lock:
        if org_id not in _s3_credentials_event:
            _s3_credentials_event[org_id] = asyncio.Event()
        return _s3_credentials_event[org_id]

logger = LogWriter.file(__name__)

def setup_ssl_for_boto3():
    """Delegate to ssl_setup (no heavy imports at module load)."""
    from ws_package.utils.ssl_setup import setup_ssl_for_boto3 as _setup
    return _setup()

def check_aws_health() -> bool:
    """
    Check if AWS dependencies (boto3/botocore) are importable.

    Previously this called boto3.client('s3') which triggers botocore's full
    credential-discovery chain — including the Windows Credential Store and a
    ~1 s timeout on the EC2 IMDSv2 endpoint — adding 2–5 s on Windows for no
    benefit at startup.  We only need to know the libraries are present; actual
    S3 connectivity is validated lazily when a real upload is attempted.
    """
    try:
        import boto3  # noqa: F401
        import botocore  # noqa: F401
        from botocore.exceptions import ClientError  # noqa: F401
        logger.info("AWS dependencies are healthy")
        return True
    except ImportError as e:
        logger.error("AWS dependencies missing: %s", e)
        return False

def recover_aws_services() -> bool:
    """
    Attempt to recover AWS services by checking dependencies and recovering if needed.
    
    Returns:
        True if AWS services are recovered, False otherwise
    """
    try:
        # Check current health
        if check_aws_health():
            logger.info("AWS services are already healthy")
            return True
        
        logger.warning("AWS services are unhealthy, attempting recovery...")
        
        # Use resource recovery to check and recover AWS dependencies
        recovery_results = resource_recovery.check_and_recover_all()
        
        # Check if AWS dependencies were recovered
        if recovery_results.get('aws_dependencies', False):
            logger.info("AWS services recovered successfully")
            return True
        else:
            logger.error("AWS services recovery failed")
            return False
            
    except Exception as e:
        logger.error("Error during AWS services recovery: %s", e)
        return False

def clear_s3_credential_cache(org_id=None) -> None:
    """Drop cached S3 credentials (e.g. after org switch or failed prefetch)."""
    with _s3_cache_lock:
        if org_id is None:
            _s3_credential_cache.clear()
        else:
            _s3_credential_cache.pop(org_id, None)


def warm_s3_credential_cache(org_id, config) -> None:
    """Store decrypted app_config in the in-process S3 credential cache."""
    if org_id is None or config is None:
        return
    with _s3_cache_lock:
        _s3_credential_cache[org_id] = config


def _parse_expire_time(value: str):
    """Parse STS expiry string into an aware or naive datetime."""
    from datetime import datetime

    text = str(value or "").strip()
    if not text:
        return None
    for fmt in (
        "%Y-%m-%d %H:%M:%S",
        "%Y-%m-%dT%H:%M:%S%z",
        "%Y-%m-%dT%H:%M:%S",
    ):
        try:
            return datetime.strptime(text, fmt)
        except ValueError:
            continue
    try:
        return datetime.fromisoformat(text.replace("Z", "+00:00"))
    except ValueError:
        return None


def s3_credentials_expired(s3_detail) -> bool:
    """True when stored STS credentials are past expire_time."""
    if s3_detail is None:
        return True
    expire_time = getattr(s3_detail, "expire_time", None) or ""
    expires_at = _parse_expire_time(expire_time)
    if expires_at is None:
        return False
    from datetime import datetime, timezone

    now = datetime.now(expires_at.tzinfo) if expires_at.tzinfo else datetime.now()
    return now >= expires_at


def _build_boto3_s3_client_kwargs(s3_detail, *, endpoint_url=None) -> dict:
    """Build boto3 S3 client kwargs including STS session token when present."""
    import botocore.config

    config = botocore.config.Config(
        retries={"max_attempts": 3},
        connect_timeout=60,
        read_timeout=60,
    )
    kwargs = {
        "aws_access_key_id": s3_detail.access_key,
        "aws_secret_access_key": s3_detail.secret_access_key,
        "config": config,
    }
    if endpoint_url:
        kwargs["endpoint_url"] = endpoint_url
    elif getattr(s3_detail, "region", None):
        kwargs["region_name"] = s3_detail.region
    session_token = getattr(s3_detail, "session_token", None) or ""
    if session_token:
        kwargs["aws_session_token"] = session_token
    return kwargs


async def s3_detail(org_id):
    """Return S3 credentials for org_id, using the in-process cache if warm."""
    with _s3_cache_lock:
        cached = _s3_credential_cache.get(org_id)
        if cached is not None:
            if s3_credentials_expired(cached):
                _s3_credential_cache.pop(org_id, None)
            else:
                return cached
    result = await server_manager.get_s3bucket(org_id)
    if result is not None:
        if s3_credentials_expired(result):
            logger.warning(
                "Cached S3 credentials expired for org_id=%s (expire_time=%s) — refetch required",
                org_id,
                getattr(result, "expire_time", "") or "n/a",
            )
            clear_s3_credential_cache(org_id)
            return None
        with _s3_cache_lock:
            _s3_credential_cache[org_id] = result
        evt = _s3_credentials_event.get(org_id)
        if evt and not evt.is_set():
            evt.set()
    return result


async def s3_detail_for_upload(org_id, user_entity):
    """Load S3 config for upload, refreshing STS credentials when expired."""
    if user_entity is not None:
        from ws_package.controllers.server_controller import ServerController

        await ServerController().ensure_fresh_s3_bucket(user_entity)
    return await s3_detail(org_id)


async def prefetch_s3_credentials(org_id) -> None:
    """
    Warm the s3_detail cache and drain any queued upload coroutines.

    Call this once, right after authentication, to ensure the first screenshot
    upload never blocks on a cold API round-trip.

    Queued uploads are retried in order; failures are logged and skipped so a
    single bad upload never blocks the drain.
    """
    logger.debug("Pre-fetching S3 credentials for org %s", org_id)
    try:
        detail = await s3_detail(org_id)
        if detail:
            logger.info("S3 credentials pre-fetched and cached for org %s", org_id)
        else:
            logger.warning(
                "S3 credentials not in database for org %s after pre-fetch "
                "(screenshots will save locally until credentials are persisted)",
                org_id,
            )
    except Exception as e:
        logger.warning("S3 credential pre-fetch failed (uploads will fall back to on-demand): %s", e)
        return

    with _s3_queue_lock:
        pending = list(_s3_upload_queue)
        _s3_upload_queue.clear()

    if not pending:
        return

    logger.info("Draining %d queued S3 upload(s)", len(pending))
    for factory, args, kwargs in pending:
        try:
            coro = factory(*args, **kwargs)
            await coro
        except Exception as e:
            logger.warning("Queued S3 upload failed during drain: %s", e)


def enqueue_upload(coroutine_factory, *args, **kwargs) -> bool:
    """
    Push an upload coroutine onto the drain queue.

    Returns True if enqueued, False if the queue is full (upload dropped).
    Use this when upload code runs before prefetch_s3_credentials completes.
    """
    with _s3_queue_lock:
        if len(_s3_upload_queue) >= _S3_QUEUE_MAX:
            logger.warning(
                "S3 upload queue full (%d items) — dropping upload. "
                "Increase _S3_QUEUE_MAX or check that auth completes promptly.",
                _S3_QUEUE_MAX,
            )
            return False
        _s3_upload_queue.append((coroutine_factory, args, kwargs))
        logger.debug("Upload enqueued (queue depth: %d)", len(_s3_upload_queue))
        return True


# Use synchronous put_object for typical screenshots/logs. boto3's upload_file()
# routes through s3transfer's ThreadPoolExecutor, which raises
# "cannot schedule new futures after interpreter shutdown" during app restart
# or teardown (common on macOS after sleep/wake when the agent relaunches).
_PUT_OBJECT_MAX_BYTES = 100 * 1024 * 1024


def _is_process_shutting_down() -> bool:
    """True when the interpreter is finalizing (upload would crash or hang)."""
    try:
        if hasattr(sys, "is_finalizing") and sys.is_finalizing():
            return True
    except Exception:
        pass
    return False


def _is_executor_shutdown_error(exc: BaseException) -> bool:
    msg = str(exc).lower()
    return isinstance(exc, RuntimeError) and "cannot schedule new futures" in msg


def _s3_put_object_upload(client, src: str, bucket: str, key: str, extra_args=None) -> None:
    """Upload without s3transfer's thread pool (safe during fragile shutdown windows)."""
    kwargs = dict(extra_args or {})
    with open(src, "rb") as body:
        client.put_object(Bucket=bucket, Key=key, Body=body, **kwargs)


def _s3_upload_file(client, src: str, bucket: str, key: str, extra_args=None) -> None:
    """Pick put_object vs upload_file based on size."""
    file_size = os.path.getsize(src)
    if file_size <= _PUT_OBJECT_MAX_BYTES:
        _s3_put_object_upload(client, src, bucket, key, extra_args)
        return
    if extra_args:
        client.upload_file(src, bucket, key, ExtraArgs=extra_args)
    else:
        client.upload_file(src, bucket, key)


class S3sync:
    def __init__(self, s3_detail: AppConfig):
        """
        Initialize S3 sync with AppConfig (encrypted configuration)
        
        Args:
            s3_detail: AppConfig instance with decrypted credentials
        """
        self.s3_detail = s3_detail
        self._client = None
        self._health_checked = False
        self._ssl_setup = False

    def client(self):
        if self.s3_detail is None:
            raise AppError(ErrorCode.S3_CONFIG_MISSING, "S3 credentials not configured — set up S3 bucket details before uploading",
                           is_recoverable=False, recovery_hint="Configure S3 bucket in organisation settings.")
        
        # Setup SSL configuration for boto3
        if not self._ssl_setup:
            setup_ssl_for_boto3()
            self._ssl_setup = True
        
        # Check AWS health before creating client
        if not self._health_checked:
            if not check_aws_health():
                logger.warning("AWS services unhealthy, attempting recovery...")
                if not recover_aws_services():
                    logger.error("AWS services recovery failed")
                    raise AppError(ErrorCode.AWS_UNHEALTHY, "AWS services unhealthy and recovery failed",
                                  is_recoverable=False, recovery_hint="Check internet connectivity and AWS status page.")
            self._health_checked = True
        
        if self._client is None:
            try:
                client_kwargs = _build_boto3_s3_client_kwargs(self.s3_detail)
                self._client = _boto3_client("s3", **client_kwargs)
            except Exception as e:
                logger.error("Failed to create S3 client: %s", e)
                # Try with SSL verification disabled as fallback
                try:
                    import os
                    os.environ['PYTHONHTTPSVERIFY'] = '0'
                    os.environ['CURL_CA_BUNDLE'] = ''
                    os.environ['REQUESTS_CA_BUNDLE'] = ''
                    
                    client_kwargs = _build_boto3_s3_client_kwargs(self.s3_detail)
                    self._client = _boto3_client(**client_kwargs)
                    logger.warning("S3 client created with SSL verification disabled")
                except Exception as e2:
                    logger.error("Failed to create S3 client even with SSL disabled: %s", e2)
                    raise
        
        return self._client

    def _reset_client(self):
        """Drop cached boto3 client so the next upload gets a fresh connection."""
        self._client = None

    def _upload_to_bucket(self, src: str, dest: str, extra_args=None) -> None:
        """Upload src to dest; uses put_object for normal file sizes."""
        _s3_upload_file(self.client(), src, self.s3_detail.bucket, dest, extra_args)

    def _public_s3_url(self, dest: str) -> str:
        return f"https://{self.s3_detail.bucket}.s3.{self.s3_detail.region}.amazonaws.com/{dest}"

    async def upload_file(self, src, dest):
        try:
            if _is_process_shutting_down():
                logger.warning("Skipping S3 upload — process is shutting down: %s", dest)
                return src

            if self.s3_detail is None:
                logger.warning("S3 credentials not configured. Skipping file upload.")
                return None

            if s3_credentials_expired(self.s3_detail):
                logger.warning(
                    "S3 session credentials expired (expire_time=%s). Skipping upload until refreshed.",
                    getattr(self.s3_detail, "expire_time", "") or "n/a",
                )
                return None
            
            # Validate source file path
            if not src or src == 'None' or src is None:
                logger.error("Invalid source file path: %s", src)
                return None
            
            # Check if source file exists
            if not os.path.exists(src):
                logger.error("Source file does not exist: %s", src)
                return None
            
            # Attempt to upload the file
            try:
                self._upload_to_bucket(src, dest)
                return self._public_s3_url(dest)
            except Exception as ssl_error:
                if "SSL" in str(ssl_error) or "certificate" in str(ssl_error).lower():
                    logger.warning("SSL error during upload, retrying with SSL disabled: %s", ssl_error)
                    # Disable SSL verification and retry
                    os.environ['PYTHONHTTPSVERIFY'] = '0'
                    os.environ['CURL_CA_BUNDLE'] = ''
                    os.environ['REQUESTS_CA_BUNDLE'] = ''
                    
                    # Recreate client with SSL disabled
                    self._reset_client()
                    self._ssl_setup = False
                    setup_ssl_for_boto3()
                    
                    self._upload_to_bucket(src, dest)
                    return self._public_s3_url(dest)
                if _is_executor_shutdown_error(ssl_error):
                    logger.warning(
                        "S3 thread pool unavailable (%s); retrying with put_object",
                        ssl_error,
                    )
                    self._reset_client()
                    _s3_put_object_upload(
                        self.client(), src, self.s3_detail.bucket, dest
                    )
                    return self._public_s3_url(dest)
                raise ssl_error
        
        except ClientError as e:
            error_code = e.response.get('Error', {}).get('Code', '')
            error_message = e.response.get('Error', {}).get('Message', '')
            
            # Check for RequestTimeTooSkewed error (time tampering detection)
            if error_code == 'RequestTimeTooSkewed':
                logger.warning("S3 RequestTimeTooSkewed (clock skew vs AWS): %s", error_message)
                raise AppError(ErrorCode.SS_TIME_TAMPER, f"S3_TIME_TAMPERING: {error_message}", is_recoverable=False)
            
            logger.error("S3 ClientError uploading file: %s - %s", error_code, error_message)
            return src
        except Exception as e:
            # boto3 upload_file raises S3UploadFailedError (wraps ClientError), not ClientError.
            # Check for RequestTimeTooSkewed in the message to detect time tampering.
            err_str = str(e)
            if 'RequestTimeTooSkewed' in err_str or 'S3_TIME_TAMPERING' in err_str:
                logger.warning("S3 RequestTimeTooSkewed (clock skew vs AWS): %s", e)
                if 'S3_TIME_TAMPERING' not in err_str:
                    raise AppError(ErrorCode.SS_TIME_TAMPER, f"S3_TIME_TAMPERING: {e}", is_recoverable=False)
                raise
            if _is_executor_shutdown_error(e) and not _is_process_shutting_down():
                try:
                    logger.warning(
                        "S3 upload failed due to thread pool shutdown; retrying with put_object: %s",
                        e,
                    )
                    self._reset_client()
                    _s3_put_object_upload(
                        self.client(), src, self.s3_detail.bucket, dest
                    )
                    return self._public_s3_url(dest)
                except Exception as retry_err:
                    logger.error("S3 put_object retry failed: %s", retry_err, exc_info=True)
            logger.error("Error uploading file to S3: %s", e, exc_info=True)
            return src

    async def upload_file_on_demand(self, src, dest):
        try:
            if _is_process_shutting_down():
                return src
            self._upload_to_bucket(src, dest)
            return self._public_s3_url(dest)
        except Exception:
            logger.error("Error uploading file to S3:")
            return src
            
    async def upload_file_to_hetzner(self, src, dest):
        if _is_process_shutting_down():
            logger.warning("Skipping Hetzner S3 upload — process is shutting down: %s", dest)
            return src
         # Initialize the S3 client for Hetzner (S3-compatible)
        hetzner_kwargs = _build_boto3_s3_client_kwargs(
            self.s3_detail,
            endpoint_url=self.s3_detail.region,
        )
        s3 = _boto3_client("s3", **hetzner_kwargs)
        
        try:
            # Validate source file path
            if not src or src == 'None' or src is None:
                logger.error("Invalid source file path for Hetzner upload: %s", src)
                return None
            
            # Check if source file exists
            import os
            if not os.path.exists(src):
                logger.error("Source file does not exist for Hetzner upload: %s", src)
                return None
            
            hetzner_extra = {'ACL': 'public-read'}
            try:
                _s3_upload_file(s3, src, self.s3_detail.bucket, dest, hetzner_extra)
                return f"{self.s3_detail.region}/{self.s3_detail.bucket}/{dest}"
            except Exception as ssl_error:
                if "SSL" in str(ssl_error) or "certificate" in str(ssl_error).lower():
                    logger.warning("SSL error during Hetzner upload, retrying with SSL disabled: %s", ssl_error)
                    # Disable SSL verification and retry
                    os.environ['PYTHONHTTPSVERIFY'] = '0'
                    os.environ['CURL_CA_BUNDLE'] = ''
                    os.environ['REQUESTS_CA_BUNDLE'] = ''
                    
                    hetzner_kwargs = _build_boto3_s3_client_kwargs(
                        self.s3_detail,
                        endpoint_url=self.s3_detail.region,
                    )
                    s3 = _boto3_client("s3", **hetzner_kwargs)
                    
                    _s3_upload_file(s3, src, self.s3_detail.bucket, dest, hetzner_extra)
                    return f"{self.s3_detail.region}/{self.s3_detail.bucket}/{dest}"
                if _is_executor_shutdown_error(ssl_error):
                    logger.warning(
                        "Hetzner S3 thread pool unavailable (%s); retrying with put_object",
                        ssl_error,
                    )
                    _s3_put_object_upload(
                        s3, src, self.s3_detail.bucket, dest, hetzner_extra
                    )
                    return f"{self.s3_detail.region}/{self.s3_detail.bucket}/{dest}"
                raise ssl_error
        except ClientError as e:
            error_code = e.response.get('Error', {}).get('Code', '')
            error_message = e.response.get('Error', {}).get('Message', '')
            
            # Check for RequestTimeTooSkewed error (time tampering detection)
            if error_code == 'RequestTimeTooSkewed':
                logger.warning("Hetzner S3 RequestTimeTooSkewed (clock skew vs AWS): %s", error_message)
                # This will be handled by the calling code
                raise AppError(ErrorCode.SS_TIME_TAMPER, f"S3_TIME_TAMPERING: {error_message}", is_recoverable=False)
            
            logger.error("Hetzner S3 ClientError uploading file: %s - %s", error_code, error_message)
            return src
        except Exception as e:
            err_str = str(e)
            if 'RequestTimeTooSkewed' in err_str or 'S3_TIME_TAMPERING' in err_str:
                logger.warning("Hetzner S3 RequestTimeTooSkewed (clock skew vs AWS): %s", e)
                if 'S3_TIME_TAMPERING' not in err_str:
                    raise AppError(ErrorCode.SS_TIME_TAMPER, f"S3_TIME_TAMPERING: {e}", is_recoverable=False)
                raise
            logger.error("Error uploading file to Hetzner S3: %s", e, exc_info=True)
            return src

