#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
51job 职位爬虫脚本
包含代理IP管理、远程数据上报和签名算法功能
"""

import httpx
import socket
import time
import hashlib
import hmac
import json
import random
from loguru import logger
import os
from datetime import datetime
from typing import List, Dict, Optional, Any, Tuple
from dataclasses import dataclass
from urllib.parse import urljoin, quote

import requests

os.makedirs("logs", exist_ok=True)
logger.add("logs/log_{time:YYYY-MM-DD}.log", level="INFO", rotation="00:00", retention="30 days", enqueue=True)


API_BASE_URL = os.getenv('API_BASE_URL', 'http://124.222.106.226:9999')

API_PUBLIC_HOST = os.getenv('API_PUBLIC_HOST')
PROXY_URL = "http://t13319619426654:ln8aj9nl@s432.kdltps.com:15818"


def sleep_random_between() -> float:
  try:
    min_seconds = float(os.getenv('SLEEP_MIN_SECONDS', '1'))
    max_seconds = float(os.getenv('SLEEP_MAX_SECONDS', '10'))
    if max_seconds < min_seconds:
      max_seconds = min_seconds
    wait_time = random.uniform(min_seconds, max_seconds)
  except Exception:
    wait_time = 1.0
  time.sleep(wait_time)
  return wait_time

def get_job_id(job: Dict[str, Any]) -> Optional[str]:
  """从职位数据中提取jobId作为唯一标识"""
  return job.get('jobId') or job.get('id') or job.get('job_id') or None

def deduplicate_jobs(jobs: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
  """对职位列表进行去重，基于jobId"""
  seen_ids = set()
  unique_jobs = []
  for job in jobs:
    job_id = get_job_id(job)
    if job_id:
      if job_id not in seen_ids:
        seen_ids.add(job_id)
        unique_jobs.append(job)
    else:
      # 如果没有jobId，保留该条记录（可能是异常数据）
      unique_jobs.append(job)
  return unique_jobs

def save_job_record(job: Dict[str, Any], job_area: str, function_type: str, 
                    page_no: int, records_file: str = "crawl_jobs_records.log") -> None:
  """保存单条职位记录到文件
  
  Args:
    job: 职位数据字典
    job_area: 地区代码
    function_type: 职能代码
    page_no: 页码
    records_file: 记录文件名
  """
  try:
    records_dir = os.path.dirname(os.path.abspath(__file__))
    records_path = os.path.join(records_dir, records_file)
    timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    
    # 提取关键信息
    job_id = get_job_id(job)
    job_name = job.get('jobName') or job.get('name') or job.get('title') or ''
    company_name = job.get('fullCompanyName') or job.get('companyName') or job.get('coName') or ''
    salary = job.get('salary') or job.get('salaryDesc') or ''
    
    # 构建记录信息
    record = {
      "timestamp": timestamp,
      "page_no": page_no,
      "job_area": job_area,
      "function_type": function_type,
      "job_id": job_id,
      "job_name": job_name,
      "company_name": company_name,
      "salary": salary,
      "job_data": job  # 保存完整的职位数据
    }
    
    # 追加到文件（JSONL格式，每行一条JSON记录）
    with open(records_path, "a", encoding="utf-8") as f:
      f.write(json.dumps(record, ensure_ascii=False) + "\n")
    
  except Exception as e:
    logger.error(f"保存职位记录失败: {e}")

def save_crawl_stats(total_count: int, unique_count: int, total_count_api: Optional[int], 
                     job_area: str, function_type: str, stats_file: str = "crawl_stats.log") -> None:
  """保存爬取统计信息到文件"""
  try:
    stats_dir = os.path.dirname(os.path.abspath(__file__))
    stats_path = os.path.join(stats_dir, stats_file)
    timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    
    stats_info = {
      "timestamp": timestamp,
      "total_crawled": total_count,
      "unique_count": unique_count,
      "duplicate_count": total_count - unique_count,
      "api_total_count": total_count_api,
      "job_area": job_area,
      "function_type": function_type
    }
    
    # 追加到文件
    with open(stats_path, "a", encoding="utf-8") as f:
      f.write(json.dumps(stats_info, ensure_ascii=False) + "\n")
    
    logger.info(f"统计信息已保存到: {stats_path}")
  except Exception as e:
    logger.error(f"保存统计信息失败: {e}")

def fetch_service_params() -> Optional[Dict[str, str]]:
  """获取当天未使用的城市与职位并占用

  Returns:
    dict: {"job_area": str, "job_name": str} 或 None
  """
  try:
    url = f"{API_BASE_URL}/api/v1/keyword/available"
    r = requests.get(url, params={"source": "qcwy", "limit": 1}, timeout=10)
    if r.status_code != 200:
      return None
    js = r.json()
    data = js.get("data") or {}
    items = data.get("items") or []
    if not items:
      return None
    item = items[0]
    ids = [item.get("id")]
    if ids and ids[0]:
      try:
        murl = f"{API_BASE_URL}/api/v1/keyword/mark-used"
        requests.post(murl, json={"source": "qcwy", "ids": ids}, timeout=10)
      except Exception:
        pass
    city = str(item.get("city", ""))
    job = str(item.get("job", ""))
    if not city or not job:
      return None
    return {"job_area": city, "job_name": job}
  except Exception:
    return None

@dataclass
class ProxyConfig:
  """代理配置"""
  host: str
  port: int
  username: Optional[str] = None
  password: Optional[str] = None
  protocol: str = "http"
  
  def to_url(self) -> str:
    """转换为代理URL"""
    if self.username and self.password:
      return f"{self.protocol}://{self.username}:{self.password}@{self.host}:{self.port}"
    return f"{self.protocol}://{self.host}:{self.port}"


class ProxyManager:
  """代理IP管理器"""
  
  def __init__(self, proxy_list: List[ProxyConfig] = None):
    self.proxy_list = proxy_list or []
    self.current_proxy_index = 0
    self.failed_proxies = set()
    
  def add_proxy(self, proxy: ProxyConfig):
    """添加代理"""
    self.proxy_list.append(proxy)
    
  def get_current_proxy(self) -> Optional[str]:
    """获取当前可用代理"""
    return PROXY_URL
  
  def mark_proxy_failed(self, proxy_url: str):
    """标记代理失败"""
    for i, proxy in enumerate(self.proxy_list):
      if proxy.to_url() == proxy_url:
        self.failed_proxies.add(i)
        logger.warning(f"代理 {proxy_url} 标记为失败")
        break
  
  def get_proxy_dict(self) -> Optional[Dict[str, str]]:
    """获取代理字典格式"""
    return {"http": PROXY_URL, "https": PROXY_URL}


class RemoteReporter:
  """远程数据上报器"""
  
  def __init__(self, report_url: str, proxy_url: Optional[str] = None):
    self.report_url = report_url
    self.public_host = API_PUBLIC_HOST
    if proxy_url:
      self.client = httpx.Client(timeout=30.0, proxy=proxy_url, verify=True)
    else:
      self.client = httpx.Client(timeout=30.0, verify=True)
  def get_local_ip(self) -> str:
      """获取本地IP地址"""
      try:
          s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
          s.connect(("8.8.8.8", 80))
          local_ip = s.getsockname()[0]
          s.close()
          return local_ip
      except Exception:
          return "127.0.0.1"
  def report_data(self, data: List[Dict[str, Any]], data_type: str) -> bool:
    """上报数据到远程API"""
    try:
      # 计算实际数据条数
      data_count = len(data) if isinstance(data, list) else 1
      
      # 使用新的数据格式
      universal_data = {
        "data_list":data,
        "data_type": data_type,
        "platform": "qcwy"
      }

      headers = {
        "accept": "application/json",
        "Content-Type": "application/json",
        'X-Forwarded-For': self.get_local_ip()

      }
      if self.public_host:
        headers["Host"] = self.public_host
        headers["X-Forwarded-Host"] = self.public_host
      
      # 使用新的API端点
      api_endpoint = f"{self.report_url}/api/v1/universal/data/batch-store-async"
      
      response = requests.post(
        api_endpoint,
        json=universal_data,
        headers=headers,
        timeout=300
      )
      
      response_size = len(response.content) if hasattr(response, 'content') else 0
      
      if 200 <= response.status_code < 300:
        logger.info(f"数据上报成功: {data_type}, 上报条数: {data_count}, 响应大小: {response_size} 字节")
        return True
      else:
        logger.error(f"数据上报失败: {response.status_code} - {response.text}, 上报条数: {data_count}, 响应大小: {response_size} 字节")
        return False
        
    except Exception as e:
      logger.error(f"数据上报异常: {e}")
      return False

  def close(self):
    """关闭客户端"""
    self.client.close()


class SignatureGenerator:
  """签名生成器"""
  
  def __init__(self, sign_key: str):
    self.sign_key = sign_key
  
  def hmac_sha256(self, message: str) -> str:
    """使用HMAC-SHA256算法生成签名"""
    key_bytes = self.sign_key.encode('utf-8')
    message_bytes = message.encode('utf-8')
    return hmac.new(key_bytes, message_bytes, hashlib.sha256).hexdigest()
  
  def generate_signature(self, url_path: str, data: Dict[str, Any] = None) -> str:
    """生成请求签名"""
    sign_message = url_path
    
    if data:
      # 将布尔值转换为字符串
      data_copy = data.copy()
      for key, value in data_copy.items():
        if isinstance(value, bool):
          data_copy[key] = "true" if value else "false"
      
      # 添加请求体到签名消息
      sign_message += json.dumps(data_copy, ensure_ascii=False, separators=(',', ':'))
    
    return self.hmac_sha256(sign_message)


class JobCrawler:
  """51job职位爬虫主类"""
  
  def __init__(self, 
        proxy_manager: ProxyManager = None,
        remote_reporter: RemoteReporter = None,
        sign_key: str = "abfc8f9dcf8c3f3d8aa294ac5f2cf2cc7767e5592590f39c3f503271dd68562b"):
    
    self.proxy_manager = proxy_manager or ProxyManager()
    self.remote_reporter = remote_reporter
    self.signature_generator = SignatureGenerator(sign_key)
    
    # API配置
    self.base_url = "https://cupid.51job.com"
    self.api_key = "51job"
    
    # 基础请求头（不包含动态property）
    self.base_headers = {
      "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 MicroMessenger/6.8.0(0x16080000) NetType/WIFI MiniProgramEnv/Mac MacWechat/WMPF MacWechat/3.8.10(0x13080a10) XWEB/1227",
      "Connection": "keep-alive",
      "Accept": "*/*",
      "Accept-Encoding": "gzip, deflate, br",
      "Content-Type": "application/json",
      "account-id": "",
      "From-Domain": "51job_weixin_wxapp",
      "xweb_xhr": "1",
      "user-token": "",
      "uuid": str(int(time.time() * 1000)) + str(random.randint(10000000, 99999999)),
      "partner": "",
      "timestamp": str(int(time.time() * 1000)),
      "Sec-Fetch-Site": "cross-site",
      "Sec-Fetch-Mode": "cors",
      "Sec-Fetch-Dest": "empty",
      "Referer": "https://servicewechat.com/wx1131e5c71e668b5d/391/page-frame.html",
      "Accept-Language": "zh-CN,zh;q=0.9"
    }
    
    # Session复用 - httpx客户端
    self.client = None
    self._init_client()
    
    # 加载城市和工作类型数据
    self.cities_data = self._load_cities_data()
    self.work_types_data = self._load_work_types_data()
  
  def build_property(self, manual_login_method: str = "", page_code: str = "home|hotjob|jobfxlist") -> str:
    """动态生成property字段
    
    Args:
      manual_login_method: 手动登录方式
      page_code: 页面代码
      
    Returns:
      URL编码的property JSON字符串
    """
    import uuid
    
    # 生成随机distinct_id（模拟微信小程序的设备ID）
    distinct_id = str(int(time.time() * 1000)) + str(random.randint(100000, 999999))
    
    property_data = {
      "frompageUrl": "", # 来源页面URL
      "pageUrl": "pages/index/index", # 当前页面URL
      "isLogin": "否", # 是否登录
      "accountid": "", # 账户ID
      "resumeId": "", # 简历ID
      "firstFrompageUrl": "", # 首次来源页面URL
      "distinct_id": distinct_id, # 设备唯一标识
      "pageCode": page_code, # 页面代码
      "shortPageCode": page_code, # 短页面代码
      "policyType": "推荐" # 策略类型
    }
    
    # 如果有手动登录方式，添加到property中
    if manual_login_method:
      property_data["manualLoginMethod"] = manual_login_method
      
    # 转换为JSON字符串并URL编码
    property_json = json.dumps(property_data, ensure_ascii=False, separators=(',', ':'))
    property_encoded = quote(property_json)
    

    
    return property_encoded
  
  def _init_client(self):
    """初始化或重新初始化httpx客户端"""
    if self.client:
      self.client.close()
    
    # 获取当前可用代理
    proxies = self.proxy_manager.get_proxy_dict()
    self.current_proxy = PROXY_URL
    
    # 创建客户端配置（不包含headers，因为签名是动态的）
    client_kwargs = {
      "timeout": 30.0,
      "verify": True,
      "trust_env": False
    }
    
    # 启用代理配置（httpx使用proxy参数，传入字符串URL）
    if self.current_proxy:
      client_kwargs["proxy"] = self.current_proxy
    logger.info(f"初始化客户端，使用代理: {self.current_proxy}")
    
    self.client = httpx.Client(**client_kwargs)
  
  def _switch_proxy_and_reinit(self):
    """切换代理并重新初始化客户端"""
    if self.current_proxy:
      logger.warning(f"标记代理失败并切换: {self.current_proxy}")
      self.proxy_manager.mark_proxy_failed(self.current_proxy)
    
    # 重新初始化客户端
    self._init_client()
  
  def _load_cities_data(self) -> Dict[str, Any]:
    """加载城市数据"""
    try:
      city_file = os.path.join(os.path.dirname(__file__), 'city.json')
      with open(city_file, 'r', encoding='utf-8') as f:
        return json.load(f)
    except Exception as e:
      logger.error(f"加载城市数据失败: {e}")
      return {}
  
  def _load_work_types_data(self) -> Dict[str, Any]:
    """加载工作类型数据"""
    try:
      work_file = os.path.join(os.path.dirname(__file__), 'work.json')
      with open(work_file, 'r', encoding='utf-8') as f:
        return json.load(f)
    except Exception as e:
      logger.error(f"加载工作类型数据失败: {e}")
      return {}
  
  def get_random_city(self) -> Dict[str, str]:
    """随机获取一个城市"""
    try:
      if not self.cities_data:
        logger.warning("城市数据为空，使用默认城市")
        return {"id": "020000", "value": "上海"}
      all_cities: List[Dict[str, str]] = []
      # 兼容多种结构：列表或包含 items 的字典
      if isinstance(self.cities_data, list):
        for item in self.cities_data:
          code = item.get("city_code") or item.get("code")
          name = item.get("city_name") or item.get("name")
          if code and name:
            all_cities.append({"id": str(code), "value": name})
      elif isinstance(self.cities_data, dict):
        for item in self.cities_data.get("items", []):
          code = item.get("city_code") or item.get("code")
          name = item.get("city_name") or item.get("name")
          if code and name:
            all_cities.append({"id": str(code), "value": name})
      if not all_cities:
        logger.warning("未找到城市数据，使用默认城市")
        return {"id": "020000", "value": "上海"}
      return random.choice(all_cities)
    except Exception:
      return {"id": "020000", "value": "上海"}
  
  def get_city_name_by_code(self, city_code: str) -> str:
    """根据城市代码获取城市名称
    
    Args:
      city_code: 城市代码
      
    Returns:
      str: 城市名称，未找到时返回代码
    """
    try:
      if not city_code:
        return city_code
      if not self.cities_data:
        return city_code
      # 兼容列表和字典格式
      items = self.cities_data if isinstance(self.cities_data, list) else self.cities_data.get("items", [])
      for item in items:
        if str(item.get("city_code")) == str(city_code):
          return item.get("city_name") or city_code
      return city_code
    except Exception:
      return city_code
  
  def get_city_pinyin_by_code(self, city_code: str) -> str:
    """根据城市代码获取拼音

    Args:
      city_code: 城市代码

    Returns:
      str: 城市对应的拼音，未找到时返回默认值
    """
    try:
      if not city_code:
        return "hanzhong"
      if not self.cities_data:
        return "hanzhong"
      # 兼容列表和字典格式
      items = self.cities_data if isinstance(self.cities_data, list) else self.cities_data.get("items", [])
      for item in items:
        if str(item.get("city_code")) == str(city_code):
          return item.get("pinyin") or "hanzhong"
      return "hanzhong"
    except Exception:
      return "hanzhong"
  
  def get_function_name_by_code(self, function_code: str) -> str:
    """根据职能代码获取职能名称
    
    Args:
      function_code: 职能代码
      
    Returns:
      str: 职能名称，未找到时返回代码
    """
    try:
      if not function_code:
        return function_code
      if isinstance(self.work_types_data, dict):
        for cat in self.work_types_data.get('items', []):
          for sub in cat.get('items', []):
            for it in sub.get('items', []):
              code = it.get('code') or it.get('funcTypeCode')
              name = it.get('value') or it.get('name')
              if str(code) == str(function_code):
                return name or function_code
      return function_code
    except Exception:
      return function_code
  
  def get_random_work_type(self) -> Dict[str, str]:
    """随机获取一个工作类型"""
    try:
      items = []
      if isinstance(self.work_types_data, dict):
        for cat in self.work_types_data.get('items', []):
          for sub in cat.get('items', []):
            for it in sub.get('items', []):
              code = it.get('code') or it.get('funcTypeCode')
              name = it.get('value') or it.get('name')
              if code and name:
                items.append({'code': str(code), 'value': name})
      if not items:
        logger.warning("工作类型数据为空，使用默认工作类型")
        return {"code": "A0N7", "value": "数据分析"}
      return random.choice(items)
    except Exception:
      return {"code": "A0N7", "value": "数据分析"}

  def find_function_code_by_name(self, name: str) -> Optional[str]:
    """根据工作类型名称查找职能代码

    Args:
      name: 工作类型中文名称

    Returns:
      职能代码或None
    """
    try:
      if not name:
        return None
      if isinstance(self.work_types_data, dict):
        for cat in self.work_types_data.get('items', []):
          for sub in cat.get('items', []):
            for it in sub.get('items', []):
              val = it.get('value') or it.get('name')
              code = it.get('code') or it.get('funcTypeCode')
              if isinstance(val, str) and isinstance(code, str) and val.strip() == name.strip():
                return code
    except Exception:
      return None
    return None

  def _normalize_job_area(self, code: Optional[str]) -> str:
    code_str = str(code or "").strip()
    return code_str if len(code_str) == 6 and code_str.isdigit() else "020000"

  def _normalize_function_type(self, code: Optional[str]) -> str:
    code_str = str(code or "").strip().upper()
    return code_str if code_str.startswith("A0") else "A0N7"
  
  def _make_request(self, url: str, data: Dict[str, Any] = None, headers: Dict[str, str] = None, max_retries: int = 3, method: str = "POST") -> Optional[Dict[str, Any]]:
    """发送HTTP请求，支持session复用和智能代理切换"""
    for attempt in range(max_retries):
      try:
        # 检查客户端是否需要重新初始化
        if not self.client or self.client.is_closed:
          self._init_client()
        
        # 使用传入的headers或默认headers
        request_headers = headers or self.base_headers
        
        # 根据方法类型发送请求
        if method.upper() == "GET":
          response = self.client.get(url, headers=request_headers)
        else:
          response = self.client.post(url, headers=request_headers, json=data)
        
        if response.status_code == 200:
          result = response.json()
          logger.info("请求参数 method={} url={} status={} resp_size={}", method.upper(), url, response.status_code, len(response.content))
          return result
        else:
          logger.warning(f"请求失败: {response.status_code} - {response.text}")
          
          # 如果是代理相关的状态码，切换代理
          if response.status_code in [407, 502, 503, 504, 403]:
            logger.warning(f"检测到代理问题，状态码: {response.status_code}")
            self._switch_proxy_and_reinit()
            continue
          
          # 其他错误，等待后重试
          if attempt < max_retries - 1:
            time.sleep(2 ** attempt)
            
      except Exception as e:
        logger.error(f"请求异常 (尝试 {attempt + 1}/{max_retries}): {e}")
        
        # 检查是否是代理相关异常
        error_str = str(e).lower()
        if any(keyword in error_str for keyword in ['proxy', 'connection', 'timeout', 'ssl', 'certificate']):
          logger.warning(f"检测到代理相关异常: {e}")
          self._switch_proxy_and_reinit()
          continue
        
        # 非代理异常，等待后重试
        if attempt < max_retries - 1:
          time.sleep(2 ** attempt) # 指数退避
    
    logger.error(f"所有重试失败，请求终止")
    return None
  
  def get_recommend_jobs(self, 
             page_no: int = 1, 
             page_size: int = 10, 
             job_area: str = "020000",
             function_type: str = "A0N7",
             seen_job_ids: Optional[set] = None) -> Tuple[List[Dict[str, Any]], int]:
    """获取推荐职位
    
    Returns:
      tuple: (职位列表, totalCount总数)
    """
    logger.info(f"获取推荐职位: 页码={page_no}, 页大小={page_size}, 地区={job_area}, 工作类型={function_type}")
    
    timestamp = int(time.time())
    
    # 请求参数
    data = {
      "pageNo": page_no,
      "pageSize": page_size,
      "specialPageCode": True,
      "isTouristMode": True,
      "type": "recommend",
      "jobArea": self._normalize_job_area(job_area),
      "personAsLabel": "",
      "functionType": self._normalize_function_type(function_type)
    }
    
    # 构建URL路径
    api_path = "open/noauth/recommend/job-tab-dynamic-wx-mini"
    url_path = f"/{api_path}?api_key={self.api_key}&timestamp={timestamp}"
    full_url = f"{self.base_url}{url_path}"
    
    # 生成签名
    signature = self.signature_generator.generate_signature(url_path, data)
    
    # 动态生成property字段
    property_value = self.build_property(page_code="home|hotjob|jobfxlist")
    
    # 添加签名和动态property到请求头
    headers = self.base_headers.copy()
    headers["sign"] = signature
    headers["property"] = property_value

    # 将布尔值转换为字符串
    for key, value in data.items():
      if isinstance(value, bool):
        data[key] = "true" if value else "false"
    # 先尝试 GET，再回退 POST
    signature_get = self.signature_generator.generate_signature(url_path)
    headers_get = self.base_headers.copy()
    headers_get["sign"] = signature_get
    headers_get["property"] = property_value
    headers_get["Content-Type"] = "application/x-www-form-urlencoded"
    response_data = self._make_request(full_url, None, headers_get, method="GET")
    if not (response_data and response_data.get("status") in ['1', 1]):
      response_data = self._make_request(full_url, data, headers, method="POST")
    if not response_data:
      return [], 0
    
    # 解析响应
    try:
      if response_data.get("status") in [1, "1"] and response_data.get("resultbody"):
        job_list = response_data["resultbody"].get("jobList", {})
        jobs = job_list.get("items", [])
        # 提取totalCount
        total_count_raw = job_list.get("totalCount") or job_list.get("totalcount")
        total_count = 0
        if total_count_raw is not None:
          if isinstance(total_count_raw, str):
            try:
              total_count = int(total_count_raw)
            except (ValueError, TypeError):
              total_count = 0
          elif isinstance(total_count_raw, (int, float)):
            total_count = int(total_count_raw)
          else:
            total_count = 0
        
        all_jobs = jobs
        # 获取城市名称和职能名称用于日志
        city_name = self.get_city_name_by_code(str(job_area))
        function_name = self.get_function_name_by_code(str(function_type))
        
        # 🔍 调试：打印每页返回的前5个 jobId，验证分页是否正确
        job_ids_preview = [get_job_id(job) for job in jobs[:5]]
        logger.info(f"🔍 第 {page_no} 页 jobIds 预览: {job_ids_preview}")
        
        # 内存去重：过滤掉已爬取的职位
        if seen_job_ids is not None:
          unique_jobs = []
          duplicate_count = 0
          for job in all_jobs:
            job_id = get_job_id(job)
            if job_id and job_id in seen_job_ids:
              duplicate_count += 1
              continue
            if job_id:
              seen_job_ids.add(job_id)
            unique_jobs.append(job)
          all_jobs = unique_jobs
          if duplicate_count > 0:
            logger.warning(f"第 {page_no} 页发现 {duplicate_count} 个重复职位（已跳过）")
        
        logger.info(f"第 {page_no} 页成功获取 {len(jobs)} 个职位（去重后: {len(all_jobs)}） | API返回totalCount: {total_count} (原始值: {total_count_raw}) | 地区: {city_name}({job_area}) | 职能: {function_name}({function_type})")
        
        # 保存每条职位记录到文件
        if all_jobs:
          for job in all_jobs:
            save_job_record(job, job_area, function_type, page_no)
        
        # 上报数据（只上报去重后的数据）
        if all_jobs:
          jobs_with_details = []
          for i, job in enumerate(all_jobs, 1):
            try:
              job_id = job.get('jobId') or job.get('id') or job.get('job_id')
              company_id = job.get('coId') or job.get('companyId')
              target_job = job.copy()
              city_pinyin = self.get_city_pinyin_by_code(str(job_area))
              target_job["jobHref"] = f"https://jobs.51job.com/{city_pinyin}/{job_id}.html"
              
              # 获取职位详情
              if job_id:
                try:
                  job_detail = self.get_job_detail(str(job_id))
                  if job_detail:
                    target_job['jobDescribe'] = job_detail.get('detailJobInfo', {}).get("jobDescribe")
                    if not company_id:
                      company_id = job_detail.get('coId') or job_detail.get('companyId')
                except Exception as e:
                  logger.warning(f"获取职位详情失败 job_id={job_id}: {e}")
                
                # 获取公司信息
                if company_id:
                  try:
                    company_info = self.get_company_info(str(company_id))
                    if company_info:
                      target_job['company_info'] = company_info
                      target_job['company_desc'] = company_info.get("coinfo",{}).get("coinfo")
                      target_job['companyHref'] = company_info.get("share",{}).get("weixinshareurl")
                  except Exception as e:
                    logger.warning(f"获取公司信息失败 company_id={company_id}: {e}")
              
              jobs_with_details.append(target_job)
              
              # 在获取详情之间添加延迟（最后一个元素不需要延迟）
              if i < len(all_jobs):
                sleep_random_between()
            except Exception as e:
              logger.error(f"处理职位数据失败: {e}")
              # 即使处理失败，也继续处理下一条
              continue
          
          if self.remote_reporter and jobs_with_details:
            logger.info(f"准备上报第 {page_no} 页数据，包含详细信息的数据条数: {len(jobs_with_details)}")
            self.remote_reporter.report_data(jobs_with_details, "job")
        
        return jobs, total_count
      else:
        logger.warning(f"API返回错误: {response_data}")
        return [], 0
        
    except Exception as e:
      import traceback
      traceback.print_exc()
      logger.error(f"解析响应失败: {e}")
      return [], 0
  
  def get_company_jobs(self, 
            page_no: int = 1, 
            page_size: int = 30,
            co_id: str = "",
            job_area: str = "",
            function: str = "",
            salary_type: str = "") -> List[Dict[str, Any]]:
    """获取公司职位"""
    url = "https://cupid.51job.com/open/noauth/jobs/company"
    
    params = {
      "api_key": "51job",
      "timestamp": int(time.time())
    }
    
    data = {
      "pageNum": page_no,
      "pageSize": page_size,
      "coId": co_id,
      "jobArea": job_area,
      "function": function,
      "salaryType": salary_type,
      "scene": 14,
      "requestId": ""
    }
    
    # 构建完整URL
    full_url = f"{url}?api_key={params['api_key']}&timestamp={params['timestamp']}"
    
    # 生成签名（包含查询参数）
    url_with_params = f"/open/noauth/jobs/company?api_key={params['api_key']}&timestamp={params['timestamp']}"
    signature = self.signature_generator.generate_signature(url_with_params, data)
    
    # 构建property字段
    property_value = self.build_property(page_code="companyDetail|job|joblist")
    
    # 添加签名和动态property到请求头
    headers = self.base_headers.copy()
    headers["sign"] = signature
    headers["property"] = property_value
    
    try:
      response_data = self._make_request(full_url, data, headers)
      if not response_data:
        signature_get = self.signature_generator.generate_signature(url_with_params)
        headers_get = self.base_headers.copy()
        headers_get["sign"] = signature_get
        headers_get["property"] = property_value
        response_data = self._make_request(full_url, None, headers_get, method="GET")
      
      if response_data and response_data.get('status') in ['1', 1]:
        result_list = response_data.get('resultbody', {}).get('items', [])
        all_jobs = result_list
        if result_list:
          logger.info(f"成功获取 {len(result_list)} 个公司职位")
          
        else:
          logger.warning("API返回成功但职位列表为空")
        
        return result_list
      else:
        logger.error(f"获取公司职位失败: {response_data}")
        return []
        
    except Exception as e:
      logger.error(f"获取公司职位时发生异常: {str(e)}")
      
      return []
  
  def get_job_detail(self, job_id: str) -> Dict[str, Any]:
    """获取职位详情"""

    
    timestamp = int(time.time())
    
    # 构建URL
    api_path = f"open/noauth/jobs/detail/base/{job_id}"
    url_path = f"/{api_path}?api_key={self.api_key}&timestamp={timestamp}"
    full_url = f"{self.base_url}{url_path}"
    
    # 生成签名
    signature = self.signature_generator.generate_signature(url_path)
    
    # 构建property字段
    property_value = self.build_property(page_code="pages/jobs/jobdetail/jobdetail")
    
    # 添加签名和动态property到请求头
    headers = self.base_headers.copy()
    headers["sign"] = signature
    headers["property"] = property_value
    headers["Content-Type"] = "application/x-www-form-urlencoded"
    # 可选登录头，开放接口无需
    
    try:
      # 使用GET请求获取职位详情
      response_data = self._make_request(full_url, None, headers, method="GET")
      
      if response_data and response_data.get('status') in ['1', 1]:
        job_detail = response_data.get('resultbody', {})
        
        if job_detail:

          return job_detail
        else:
          logger.warning(f"职位详情为空: {job_id}")
          return {}
      else:
        logger.error(f"获取职位详情失败: {job_id}, 响应: {response_data}")
        return {}
        
    except Exception as e:
      logger.error(f"获取职位详情时发生异常: {job_id}, 错误: {str(e)}")
      return {}
  
  def get_company_info(self, company_id: str) -> Dict[str, Any]:
    """获取公司详情信息"""
    
    timestamp = int(time.time())
    
    # 构建URL和参数
    api_path = "open/noauth/company-info/info-data"
    params = {
      "companyId": company_id,
      "colorOne": "#ffffff", 
      "colorTwo": "#ffffffcc"
    }
    
    # 构建完整URL路径（包含查询参数）
    url_path = f"/{api_path}?api_key={self.api_key}&timestamp={timestamp}"
    
    # 添加业务参数到URL
    param_str = "&".join([f"{k}={quote(str(v))}" for k, v in params.items()])
    url_path += f"&{param_str}"
    full_url = f"{self.base_url}{url_path}"
    
    # 生成签名（GET请求，签名只包含URL路径）
    signature = self.signature_generator.generate_signature(url_path)
    
    # 构建property字段
    property_value = self.build_property(page_code="companyDetail|company|companyinfo")
    
    # 添加签名和动态property到请求头
    headers = self.base_headers.copy()
    headers["sign"] = signature
    headers["property"] = property_value
    headers["Content-Type"] = "application/x-www-form-urlencoded"
    
    try:
      # 使用GET请求获取公司详情
      response_data = self._make_request(full_url, None, headers, method="GET")
      
      if response_data and response_data.get('status') in ['1', 1]:
        company_info = response_data.get('resultbody', {})
        
        if company_info:

          return company_info
        else:
          logger.warning(f"公司详情为空: {company_id}")
          return {}
      else:
        logger.error(f"获取公司详情失败: {response_data}")
        return {}
        
    except Exception as e:
      logger.error(f"获取公司详情时发生异常: {str(e)}")
      return {}
    
  def crawl_multiple_pages(self, 
             max_pages: int = 5,
             page_size: int = 10,
             job_area: str = "020000",
             function_type: str = "A0N7",
             delay: float = 1.0) -> List[Dict[str, Any]]:
    """爬取指定页数的数据，根据totalCount动态计算页数"""
    all_jobs = []
    total_count = None
    calculated_max_pages = max_pages
    page = 1
    actual_pages_crawled = 0  # 记录实际爬取的页数
    
    # 内存去重：维护已爬取的job_id集合，避免重复获取详情和上报
    seen_job_ids = set()
    
    # 获取城市名称和职能名称用于日志
    city_name = self.get_city_name_by_code(str(job_area))
    function_name = self.get_function_name_by_code(str(function_type))
    
    logger.info("=" * 80)
    logger.info(f"开始爬取推荐职位 | 最大页数: {max_pages} | 每页大小: {page_size} | 地区: {city_name}({job_area}) | 职能: {function_name}({function_type})")
    logger.info("=" * 80)

    while page <= calculated_max_pages:
      logger.info(f"\n>>> 正在爬取第 {page}/{calculated_max_pages} 页...")
      try:
        self._init_client()
      except Exception:
        pass
      
      jobs, current_total_count = self.get_recommend_jobs(
        page_no=page,
        page_size=page_size,
        job_area=job_area,
        function_type=function_type,
        seen_job_ids=seen_job_ids  # 传入已爬取的job_id集合，用于去重
      )
      
      # 记录实际爬取的页数
      actual_pages_crawled += 1
      
      # 首次获取时，根据totalCount动态计算需要爬取的页数
      if total_count is None and current_total_count > 0:
        total_count = current_total_count
        # 根据totalCount计算实际需要的页数（向上取整）
        calculated_max_pages = min(max_pages, (total_count + page_size - 1) // page_size)
        logger.info(f"检测到API返回totalCount: {total_count} | 计算需要爬取: {calculated_max_pages} 页 (每页 {page_size} 条) | 预计最大数据: {calculated_max_pages * page_size} 条")
      
      if jobs:
        all_jobs.extend(jobs)
        logger.info(f"第 {page} 页获取到 {len(jobs)} 个职位 | 累计: {len(all_jobs)} 个")
      else:
        logger.warning(f"第 {page} 页没有获取到职位数据，提前结束")
        # 如果某页没有数据，提前结束
        break
      
      # 如果已获取的数据量达到或超过totalCount，提前结束
      if total_count and len(all_jobs) >= total_count:
        logger.info(f"已获取所有数据 ({len(all_jobs)}/{total_count})，提前结束爬取")
        break
      
      # 延迟避免被封（在每页请求之间添加休眠）
      if page < calculated_max_pages:
        sleep_time = sleep_random_between()
        logger.info(f"休眠 {sleep_time:.2f} 秒后继续...")
      
      page += 1
    
    # 统计和去重
    total_crawled = len(all_jobs)
    unique_jobs = deduplicate_jobs(all_jobs)
    unique_count = len(unique_jobs)
    duplicate_count = total_crawled - unique_count
    
    # 打印详细统计信息
    logger.info("\n" + "=" * 80)
    logger.info("爬取统计信息:")
    logger.info(f"  总抓取数量: {total_crawled} 条")
    logger.info(f"  去重后数量: {unique_count} 条")
    logger.info(f"  重复数量: {duplicate_count} 条")
    logger.info(f"  API返回totalCount: {total_count if total_count is not None else '未知'} 条")
    logger.info(f"  实际爬取页数: {actual_pages_crawled} 页")
    logger.info(f"  地区: {city_name}({job_area})")
    logger.info(f"  职能: {function_name}({function_type})")
    if total_count and total_count > 0:
      coverage_rate = (unique_count / total_count * 100)
      logger.info(f"  覆盖率: {coverage_rate:.2f}% ({unique_count}/{total_count})")
      if unique_count < total_count:
        logger.warning(f" 注意: 实际抓取数量({unique_count}) < API返回总数({total_count})，可能存在分页问题或数据过滤")
    elif total_count == 0:
      logger.warning(f"注意: API返回totalCount为0，可能是该地区/职能组合下没有数据")
    logger.info("=" * 80 + "\n")
    
    # 保存统计信息到文件
    save_crawl_stats(
      total_count=total_crawled,
      unique_count=unique_count,
      total_count_api=total_count,
      job_area=job_area,
      function_type=function_type
    )
    
    # 记录保存的文件路径
    records_dir = os.path.dirname(os.path.abspath(__file__))
    records_path = os.path.join(records_dir, "crawl_jobs_records.log")
    logger.info(f"职位记录已保存到: {records_path} (共 {unique_count} 条记录)")
    
    return unique_jobs
  
  def crawl_company_jobs_multiple_pages(self, 
                    co_id: str,
                    max_pages: int = 5,
                    page_size: int = 30,
                    job_area: str = "",
                    function: str = "",
                    salary_type: str = "",
                    delay: float = 1.0) -> List[Dict[str, Any]]:
    """爬取指定页数的公司职位数据"""
    all_jobs = []
    
    for page in range(1, max_pages + 1):
      logger.info(f"正在爬取公司 {co_id} 第 {page} 页职位...")
      try:
        self._init_client()
      except Exception:
        pass
      
      jobs = self.get_company_jobs(
        page_no=page,
        page_size=page_size,
        co_id=co_id,
        job_area=job_area,
        function=function,
        salary_type=salary_type
      )
      
      if jobs:
        all_jobs.extend(jobs)
        logger.info(f"第 {page} 页获取到 {len(jobs)} 个职位")
      else:
        logger.warning(f"第 {page} 页没有获取到职位数据")
      
      # 延迟避免被封
      if page < max_pages:
        sleep_random_between()
    
    logger.info(f"总共获取到 {len(all_jobs)} 个公司职位")
     
    return all_jobs
  

  def close(self):
    """关闭资源"""
    # 关闭httpx客户端
    if self.client and not self.client.is_closed:
      self.client.close()
      logger.info("httpx客户端已关闭")
    
    # 关闭远程报告器
    if self.remote_reporter:
      self.remote_reporter.close()


def crawl_recommend_jobs_main(max_pages: int = 10, page_size: int = 15, job_area: str = None, function_type: str = None, delay: float = 2.0):
  """推荐职位爬取主函数
  
  Args:
    max_pages: 最大爬取页数
    page_size: 每页数量
    job_area: 地区代码，为None时随机选择
    function_type: 工作类型代码，为None时随机选择
    delay: 页面间隔时间
    
  Returns:
    List[Dict]: 职位列表
  """
  # 1. 配置代理（可选）
  proxy_manager = ProxyManager()
  username = os.getenv('PROXY_USERNAME', 't13319619426654')
  password = os.getenv('PROXY_PASSWORD', 'ln8aj9nl')
  tunnel = os.getenv('PROXY_TUNNEL', 's432.kdltps.com:15818')
  
  # 解析隧道地址和端口
  if tunnel and ':' in tunnel:
    host, port_str = tunnel.split(':')
    port = int(port_str)
    # 使用环境变量中的代理信息创建代理配置
    proxy_config = ProxyConfig(host, port, username, password)
    proxy_manager.add_proxy(proxy_config)
    logger.info(f"已配置代理: {proxy_config.to_url()}")
  else:
    logger.warning("未找到有效的隧道配置，将使用本地代理")
    proxy_manager.add_proxy(ProxyConfig("127.0.0.1", 8080)) # 添加本地代理作为备选
  
  # 2. 配置远程上报（可选）
  
  # 3. 创建爬虫实例
  remote_reporter = RemoteReporter(
    report_url=API_BASE_URL,
    # proxy_url=proxy_manager.get_current_proxy()
  )
  crawler = JobCrawler(
    proxy_manager=proxy_manager,
    remote_reporter=remote_reporter
  )
  
  try:
    svc = fetch_service_params()
    if svc:
      job_area = svc["job_area"]
      code = crawler.find_function_code_by_name(svc["job_name"]) if function_type is None else function_type
      if code:
        function_type = code
    # 回退随机选择
    if job_area is None:
      random_city = crawler.get_random_city()
      job_area = random_city['id']
      logger.info(f"随机选择城市: {random_city['value']} ({random_city['id']})")
    if function_type is None:
      random_work_type = crawler.get_random_work_type()
      function_type = random_work_type['code']
      logger.info(f"随机选择工作类型: {random_work_type['value']} ({random_work_type['code']})")
    
    logger.info(f"开始爬取推荐职位，最多 {max_pages} 页")
    
    all_jobs = crawler.crawl_multiple_pages(
      max_pages=max_pages,
      page_size=page_size,
      job_area=job_area,
      function_type=function_type,
      delay=delay
    )
    
    # 显示结果
    if all_jobs:
      logger.info(f"推荐职位爬取完成！共找到 {len(all_jobs)} 个职位")
    else:
      logger.warning("未找到任何推荐职位信息")
    
    return all_jobs
    
  except KeyboardInterrupt:
    logger.info("用户中断推荐职位爬取")
    return []
  except Exception as e:
    logger.error(f"推荐职位爬取过程中发生错误: {e}")
    return []
  finally:
    crawler.close()


def crawl_company_jobs_main(co_id: str, max_pages: int = 5, page_size: int = 30, job_area: str = "", function: str = "", salary_type: str = "", delay: float = 2.0):
  """公司职位爬取主函数
  
  Args:
    co_id: 公司ID
    max_pages: 最大爬取页数
    page_size: 每页数量
    job_area: 地区代码
    function: 职能代码
    salary_type: 薪资类型
    delay: 页面间隔时间
    
  Returns:
    List[Dict]: 职位列表
  """
  # 1. 配置代理（可选）
  proxy_manager = ProxyManager()
  # proxy_manager.add_proxy(ProxyConfig("127.0.0.1", 8080)) # 添加代理
  
  # 2. 配置远程上报（可选）
  remote_reporter = RemoteReporter(
    report_url=API_BASE_URL,
    proxy_url=proxy_manager.get_current_proxy()
  )
  
  # 3. 创建爬虫实例
  crawler = JobCrawler(
    proxy_manager=proxy_manager,
    remote_reporter = remote_reporter
  )
  
  try:
    logger.info(f"开始爬取公司 {co_id} 的职位，最多 {max_pages} 页")
    
    all_jobs = crawler.crawl_company_jobs_multiple_pages(
      co_id=co_id,
      max_pages=max_pages,
      page_size=page_size,
      job_area=job_area,
      function=function,
      salary_type=salary_type,
      delay=delay
    )
    return all_jobs
    
  except KeyboardInterrupt:
    logger.info("用户中断公司职位爬取")
    return []
  except Exception as e:
    logger.error(f"公司职位爬取过程中发生错误: {e}")
    return []
  finally:
    crawler.close()


def main():
  """主函数示例 - 演示两个爬取函数的用法"""
  # 使用更大的默认值，根据totalCount会动态调整实际爬取的页数
  recommend_jobs = crawl_recommend_jobs_main(max_pages=50, page_size=20)

if __name__ == "__main__":
  while True:
    main()
    sleep_random_between()
