def generate_cache_key(query_spec: dict) -> str:
    """
    Deterministic cache key generation
    """
    components = [
        # File checksums (mtime + size based)
        hash_files(query_spec['files']),
        
        # Normalized query
        normalize_query(query_spec['query']),
        
        # Engine and options
        query_spec.get('engine', 'auto'),
        query_spec.get('return_limit', 100),
        
        # Result format (different formats = different cache entries)
        query_spec.get('return_format', 'json'),
    ]
    
    return hashlib.sha256(
        '|'.join(str(c) for c in components).encode()
    ).hexdigest()[:32]

def hash_files(file_paths: list) -> str:
    """
    Fast file fingerprint using mtime and size
    """
    fingerprints = []
    for path in sorted(file_paths):
        stat = os.stat(path)
        fingerprints.append(f"{path}:{stat.st_mtime}:{stat.st_size}")
    return hashlib.sha256('|'.join(fingerprints).encode()).hexdigest()[:16]

@dataclass
class DatasetCacheEntry:
    file_path: str
    fingerprint: str  # mtime + size hash
    schema: Schema
    row_count: int
    column_stats: Dict[str, ColumnStats]
    sample_data: pd.DataFrame
    parsed_lazyframe: Optional[pl.LazyFrame]
    cached_at: datetime
    access_count: int
    last_accessed: datetime
    
    @property
    def is_stale(self) -> bool:
        # Check if file has been modified
        current_stat = os.stat(self.file_path)
        current_fingerprint = f"{current_stat.st_mtime}:{current_stat.st_size}"
        return current_fingerprint != self.fingerprint

# Original: Load all columns
df = pl.read_csv('large_file.csv')
result = df.select(['name', 'revenue'])

# Optimized: Scan only needed columns
df = pl.scan_csv('large_file.csv', columns=['name', 'revenue'])
result = df.collect()

# User queries:
# Q1: Total revenue by category
# Q2: Total revenue by category (with date filter)

# Q1 executes full scan → caches result
# Q2 uses cached aggregated data + applies filter → fast

class PrefetchManager:
    """
    Predict and prefetch likely-needed data
    """
    
    def __init__(self):
        self.query_patterns = {}
        self.access_graph = defaultdict(list)
    
    def record_access(self, dataset: str, query: dict):
        """Track query patterns for prediction"""
        pattern = self.extract_pattern(query)
        self.query_patterns[dataset].append({
            'pattern': pattern,
            'timestamp': time.time(),
            'columns': query.get('columns', []),
        })
    
    def predict_next_queries(self, dataset: str, current_query: dict) -> list:
        """Predict what user might query next"""
        predictions = []
        
        # Pattern 1: Time series → predict next time bucket
        if 'date' in current_query.get('filters', {}):
            predictions.append(self.extend_time_range(current_query))
        
        # Pattern 2: Aggregation → predict drill-down
        if current_query.get('aggregation'):
            predictions.append(self.add_dimension(current_query))
        
        # Pattern 3: Top-N → predict next N
        if current_query.get('limit'):
            predictions.append(self.increase_limit(current_query))
        
        return predictions
    
    async def prefetch(self, dataset: str, query: dict):
        """Execute predicted queries in background"""
        predictions = self.predict_next_queries(dataset, query)
        
        for prediction in predictions[:3]:  # Max 3 prefetches
            cache_key = generate_cache_key(prediction)
            if not await cache.exists(cache_key):
                # Execute in background
                asyncio.create_task(
                    self.execute_and_cache(prediction, cache_key)
                )

@dataclass
class VisualizationCache:
    chart_type: str
    data_hash: str
    options_hash: str
    base64_image: str
    width: int
    height: int
    created_at: datetime
    
    @classmethod
    def from_request(cls, data: pd.DataFrame, chart_type: str, options: dict):
        data_hash = hash_dataframe(data)
        options_hash = hash_dict(options)
        
        return cls(
            chart_type=chart_type,
            data_hash=data_hash,
            options_hash=options_hash,
            base64_image=None,  # Generated on demand
            width=options.get('width', 800),
            height=options.get('height', 600),
            created_at=datetime.now(),
        )

def get_or_create_chart(data: pd.DataFrame, chart_type: str, options: dict) -> str:
    """Get cached chart or generate new one"""
    cache_key = VisualizationCache.from_request(data, chart_type, options)
    
    # Check cache
    cached = viz_cache.get(cache_key.data_hash + cache_key.options_hash)
    if cached:
        return cached.base64_image
    
    # Generate new chart
    fig = create_chart(data, chart_type, options)
    buf = io.BytesIO()
    fig.savefig(buf, format='png', dpi=100)
    base64_image = base64.b64encode(buf.getvalue()).decode()
    
    # Cache it
    cache_key.base64_image = base64_image
    viz_cache.set(cache_key.data_hash + cache_key.options_hash, cache_key)
    
    return base64_image

class CacheInvalidator:
    """
    Intelligent cache invalidation
    """
    
    def on_file_change(self, file_path: str):
        """Called when a dataset file is modified"""
        # Invalidate all caches for this file
        pattern = f"*{hashlib.md5(file_path.encode()).hexdigest()[:8]}*"
        invalidated = cache.invalidate(pattern)
        
        logger.info(f"Invalidated {invalidated} cache entries for {file_path}")
    
    def on_query_error(self, query: dict, error: Exception):
        """Handle query errors - may invalidate related caches"""
        if isinstance(error, SchemaMismatchError):
            # Schema changed - invalidate all dataset caches
            self.on_file_change(query['files'][0])
    
    def scheduled_cleanup(self):
        """Periodic cleanup of stale cache entries"""
        # Remove expired entries
        for key, entry in list(l1_cache.items()):
            if entry.is_expired():
                l1_cache.delete(key)
        
        # Compact L2 cache if over size limit
        if l2_cache.size > l2_cache.max_size:
            l2_cache.evict_lfu(target_size=l2_cache.max_size * 0.8)