From 42280f8bed2b9ad4944181f32aaa0656b44908da Mon Sep 17 00:00:00 2001 From: win Date: Tue, 24 Mar 2026 01:49:31 +0800 Subject: [PATCH] ip --- aaa.json | 1175 +++++++++++++----------- app/services/ingest/remote_push.py | 2 + docker-compose.spider.yml | 40 + ecs_full_pipeline.log | 50 + spider-entrypoint.sh | 55 ++ spider.Dockerfile | 41 + spiderJobs/__init__.py | 3 + spiderJobs/aaa.txt | 55 ++ spiderJobs/platforms/boss/main.py | 14 +- spiderJobs/platforms/job51/main.py | 14 +- spiderJobs/platforms/zhilian/client.py | 8 +- spiderJobs/platforms/zhilian/main.py | 9 +- 12 files changed, 886 insertions(+), 580 deletions(-) create mode 100644 docker-compose.spider.yml create mode 100644 spider-entrypoint.sh create mode 100644 spider.Dockerfile create mode 100644 spiderJobs/aaa.txt diff --git a/aaa.json b/aaa.json index 27e493f..474c5e6 100644 --- a/aaa.json +++ b/aaa.json @@ -1,589 +1,650 @@ { - "source_type": "智联招聘", - "name": "深圳市安保医疗科技股份有限公司", - "common_name": "深圳市安保医疗科技股份有限公司", - "title": "质量体系工程师", - "title_addr": "质量体系工程师", - "description": "岗位职责：1、负责公司质量管理体系的策划，运营，维护和合规性的管理；2、负责公司各类医疗器械质量管理体系（国内GMP、ISO13485、QSR820、MDSAP、 MDR、ISO9001、 职业健康和安全，环境管理体系等）的运营、维护与升级； 3、负责公司各类外部体系审核工作（国内飞检、CE审核、飞检等）的内部协调与组织工作； 4、负责生产、经营数据上报，及负责监控不良事件上报； 5、负责相关行业...", + "source_type": "Boss直聘", + "name": "华为云计算", + "common_name": "华为云计算", + "title": "软件开发工程师", + "title_addr": "软件开发工程师", + "description": "岗位职责：\n1、负责国标接入，私有协议接入，设备管理等特性方案优化，特性开发，问题定位，问题修改等工作。\n2、看护模块特性，定位并处理相关现网问题，并将处理经验作为输入进行负向改进。\n\n岗位要求：\n1、具备丰富的软件开发经验，深入理解并掌握C/C++/Java/Go编程语言中的一种或多种；\n2、了解视频接入国标协议者优先\n3、对软件工程技术有浓厚兴趣和热情，求知欲和学习能力强，积极探索新技术新方法。具备良好的团队合作意识和良好的跨团队沟通和协作能力；", "education": "本科", - "skill": "医疗器械质量管理体系,ISO认证,GMP认证,FDA认证,二三类医械经验", + "skill": "Golang,Java,C++,MySQL,接受无前端经验/技能,Redis,Spring,后端开发", "welfare": "", - "years": "3-5年", - "salary": "1.2-1.8万", - "location": "深圳宝安", - "position": "深圳宝安", + "years": "1-3年", + "salary": "18-35", + "location": "深圳", + "position": "深圳", "job_type": "全职", - "size": "500-999人", - "employer_type": "民营", - "industry": "医疗设备/器械", + "size": "10000人以上", + "employer_type": "全职", + "industry": "计算机服务", "job_1st_class": "", "job_2nd_class": "", "job_3rd_class": "", "job_4th_class": "", - "date": "2026-01-22 11:23:37", + "date": "", "start_date": "", "end_date": "", "age": "", "sex": "", - "number": "1", - "url": "http://jobs.zhaopin.com/CC219609310J40829101009.htm", - "company_id": "21960931", - "company_name": "深圳市安保医疗科技股份有限公司", - "company_url": "http://company.zhaopin.com/CC219609310.htm", - "company_desc": "

深圳市安保医疗科技股份有限公司（简称“安保医疗”）成立于2001年，是国家级高新技术企业、国家级专精特新重点“小巨人”企业，深耕急救与生命支持领域二十余年，致力于为全球医疗系统提供一体化综合解决方案。


作为国内首家危急重症一体化专业制造商，公司构建了完善的研发创新体系，拥有广东省及深圳市急救和生命支持类医疗设备工程技术研究中心、高端急危重症医疗设备广东省工程研究中心、深圳市高端医疗设备中小试基地、深圳市博士后创新实践基地等多层次科研平台。全面掌握了融合按压、气道管理、机械通气、电除颤等核心技术，实现了关键技术与核心部件的自主可控，推动高端医疗装备的国产化进程。


目前，安保医疗已上市40余款生命支持设备，涵盖心肺复苏机、急救转运呼吸机、除颤仪等关键产品，拥有近1000项专利技术，多项集成化设备达到国际先进水平，部分技术全球领先。公司牵头承担了十余项国家及省市级重大科研项目，与50余家顶级医院、30余所高校建立深度协同，构建了产学研用一体化的创新生态。


公司凭借卓越的技术实力与创新能力，先后荣获2023年深圳市科技进步奖一等奖、2018年广东省科技进步奖二等奖、2023年及2024年中国专利奖优秀奖、2024年广东省及深圳市制造业单项冠军等多项荣誉，成为中国高端医疗装备自主创新与国产替代的中坚力量。

", + "number": "", + "url": "https://www.zhipin.com/job_detail/326f178f0497dfcb03R92tW_EVVT.html", + "company_id": "08fbb99a8f9314e21nF73Nq9E1E~", + "company_name": "华为云计算", + "company_url": "https://www.zhipin.com/gongsi/08fbb99a8f9314e21nF73Nq9E1E~.html", + "company_desc": "华为云持续践行“一切皆服务”战略，通过基础设施即服务、技术即服务和经验即服务，帮助客户加速释放数字生产力。入选Gartner® 2022年云基础设施和平台服务(CIPS)魔力象限。全球开发者数量超过400万。", "base_data": { - "abroadFlag": 2, - "abroadTipInfo": { - "abroadTips": [], - "icon": "", - "title": "" - }, - "adResponse": None, - "aiPositionRecommendLevel": "", - "aiPositionRecommendReason": "", - "alreadyCallPhone": False, - "applyType": "1", - "campusBestCompany": { - "bestCompanyUrl": "", - "homepageType": 0, - "logoTagUrl": "", - "state": 0 - }, - "campusJobDetail": None, - "campusJobMatchData": None, - "campusPositionCardTagInfo": None, - "campusRootOrgInfo": None, - "canBeRegular": False, - "canRemoteInternship": False, - "cardCustomJson": "{"address":"深圳 宝安 石岩","companyName":"深圳市安保医疗","locationType":"1","salary60":"1.2-1.8万","strengthLabel":"A轮"}", - "cardType": 1, - "chatWindow": 1, - "cityDistrict": "宝安", - "cityId": "765", - "commercialLabel": [], - "commonTrack": { - "trackCommercialFeature": "", - "trackSocialSearchEmergencyFeature": False - }, - "companyId": 21960931, - "companyLogo": "https://rd5-public.zhaopin.cn/imgs/company/043dbec10b9ee8b8104f537b2280e6c4.jpg", - "companyName": "深圳市安保医疗科技股份有限公司", - "companyNumber": "CZ219609310", - "companyRootId": 21960931, - "companyScaleTypeTagsNew": [], - "companySize": "500-999人", - "companyUrl": "http://company.zhaopin.com/CC219609310.htm", - "complainFlag": False, - "deliveryPath": "", - "displayPhoneNumber": False, - "distance": 0.0, - "distanceFormat": "", - "distanceText": "", - "education": "本科", - "experimentInfo": None, - "extend": None, - "extensions": None, - "featureServer": { - "jdViews3d": "38", - "lastReplyTime": 1773997359829, - "lastReplyTimeText": "", - "staffAvgFirstResponseTime7d": 1332, - "staffAvgHandleResumeTime30d": 3681, - "staffHandleResumeCnts30d": 191, - "staffReplyRate30d": 0.48, - "todayReplyNum": 0, - "todayReplyNumText": "" - }, - "feedOperation": None, - "feedPosition": None, - "financingStage": { - "code": 3, - "name": "A轮" - }, - "firstPublishTime": "2026-01-22 11:23:37", - "hasAppliedPosition": False, - "industryCompanyTags": [ - "500030000", - "1200030000" - ], - "industryName": "医疗设备/器械", - "industryTags": [ - "500210000" - ], - "innerBusinessInfo": { - "customIndustryList": [ - { - "itemCode": 500210000, - "name": "医疗设备/器械", - "standard": True - } + "pageType": 0, + "securityId": "SqtdEDmqivtfe-i11hbsG5Vns8UH4D7KioJSlTda6xkyUnOmDMxckHHQg2xu-aekF3XqS30WmHClXSDt1m_vnqPKqB1SNLaaih0dSh14r173siugh81YdHCHWwo5SboyIZ7yXZxA1p7WQ_MjNxOuvmegfC4MuMlhBtexVgQD5cQ-6rEcYtKwEGI97bsZU8wJS8mMzRMTUV_2LFZuhA~~", + "bossBaseInfoVO": { + "encryptBossId": "2aa57f8865baa8291XR-3Nu_FFM~", + "bossId": 0, + "securityId": "sV0jXiokoR5rb-R1GnCYNjp6MIb6MDctVtlwVCXhrSi8IQZwH-C9zqs6l_QdxCIjAI9NiW7R8wth0h27TMQ5PtLl3Yp-z6-0NYNQ72XABTyj5eHiIhdiIBuOgvdhB7chAGeByKoTIs4R99Te_rels6zrxcQ95zof3eaOtNTwj-3Gi404REp1zo2AsXk~", + "name": "李先生", + "tinyAvatar": "https://img.bosszhipin.com/boss/avatar/avatar_6.png", + "title": "Android开发工程师", + "activeTimeDesc": "3月内活跃", + "activeTime": 1768990978000, + "bossOnline": False, + "brandName": "华为云计算", + "disableBossInfo": False, + "dimission": False, + "bossBehaviorLabels": [ + "3月内活跃" ] }, - "internshipMonths": 0, - "isNewPosition": 0, - "jdCardType": 2, - "jobDetailData": { - "company": { - "base": None, - "companyAuditNature": None, - "companyComment": None, - "companyInterview": None, - "jumpDetail": None, - "orgBestRanking": None, - "orgReliableCompany": None, - "other": None, - "state": None - }, - "companyProxy": { - "companyAddress": "", - "companyImage": "", - "companyName": "", - "companySize": "", - "entryCompanyTitle": "" - }, - "customAttributeInfo": { - "platformRemind": "", - "reportItems": [], - "welfareItems": [], - "workTimeItems": [] - }, - "debug": {}, - "experimentInfo": { - "blueCollarJobTitleExperimentInfo": None - }, - "featureServer": None, - "imSection": None, - "internship": [], - "live": { - "liveItems": [], - "liveQuickFocusChecked": 0, - "liveQuickFocusState": 0, - "recommendLiveList": [], - "state": 0 - }, - "operationSection": { - "topJobBannerArea": None - }, - "partTime": [], - "position": { - "base": { - "deliveryPath": "", - "education": "本科", - "educationCode": "", - "maxSalary": "", - "minSalary": "", - "positionId": 40829101009, - "positionName": "质量体系工程师", - "positionNumber": "CC219609310J40829101009", - "positionUrl": "", - "positionWorkingExp": "3-5年", - "positionWorkingExpCode": "", - "propertyType": "", - "salary": "1.2-1.8万", - "salaryReal": "", - "workType": "全职" - }, - "date": { - "dateEnd": "", - "dateStart": "", - "firstPublishTime": "", - "positionPublishTime": "", - "positionUpdateTime": "", - "positionUpdateTimeText": "" - }, - "desc": { - "description": "岗位职责：\n1、负责公司质量管理体系的策划，运营，维护和合规性的管理；\n2、负责公司各类医疗器械质量管理体系（国内GMP、ISO13485、QSR820、MDSAP、 MDR、ISO9001、 职业健康和安全，环境管理体系等）的运营、维护与升级； \n3、负责公司各类外部体系审核工作（国内飞检、CE审核、飞检等）的内部协调与组织工作； \n4、负责生产、经营数据上报，及负责监控不良事件上报； \n5、负责相关行业法律法规的收集、整理、受控及内部传递；\n6、协助质量经理做好其他部门工作。\n岗位要求：\n1.本科及以上学历，大学英文4级以上的阅读能力，专业不限；\n2.至少三年以上二类有源或三类有源医疗器械质量体系管理经验；\n3.有医疗器械内审员资格证，有GCP资格证优先；\n4.熟悉MDSAP，或MDR2017/745优先。", - "descriptionHighlight": "", - "highlightLabels": [], - "labels": [ - "医疗器械质量管理体系", - "ISO认证", - "GMP认证", - "FDA认证", - "二三类医械经验", - "医疗设备/器械" + "jobBaseInfoVO": { + "jobId": 407182173, + "encryptJobId": "326f178f0497dfcb03R92tW_EVVT", + "expectId": 0, + "jobValidStatus": 1, + "deleted": 0, + "position": 100101, + "positionName": "软件开发工程师", + "positionCategory": "Java", + "location": 101280600, + "locationName": "深圳", + "areaDistrict": "龙岗区", + "businessDistrict": "坂田", + "locationDesc": "深圳·龙岗区·坂田", + "experienceName": "1-3年", + "degreeName": "本科", + "lowSalary": 18, + "highSalary": 35, + "jobDesc": "岗位职责：\n1、负责国标接入，私有协议接入，设备管理等特性方案优化，特性开发，问题定位，问题修改等工作。\n2、看护模块特性，定位并处理相关现网问题，并将处理经验作为输入进行负向改进。\n\n岗位要求：\n1、具备丰富的软件开发经验，深入理解并掌握C/C++/Java/Go编程语言中的一种或多种；\n2、了解视频接入国标协议者优先\n3、对软件工程技术有浓厚兴趣和热情，求知欲和学习能力强，积极探索新技术新方法。具备良好的团队合作意识和良好的跨团队沟通和协作能力；", + "requiredSkills": [ + "Golang", + "Java", + "C++", + "MySQL", + "接受无前端经验/技能", + "Redis", + "Spring", + "后端开发" + ], + "address": "深圳龙岗区天安云谷一期1栋A座10楼", + "encryptAddressId": "3ed90011c6ff552e1nZ-2tu-E1FZxoi2UfOd", + "longitude": 114.06822, + "latitude": 22.65904, + "staticMapUrl": "https://img.bosszhipin.com/beijin/upload/amap_proxy/20230424/48ba41acc9cef1bfd24cbf912f096cc8f3a620ada92e7e796bb61e3b7bce0931da574d19d1d82c88.jpg", + "pcStaticMapUrl": "https://img.bosszhipin.com/beijin/upload/amap_proxy/20230518/48ba41acc9cef1bf0b1f35b756e351497fc5eb9ce999036a6bb61e3b7bce0931da574d19d1d82c88.jpg", + "baiduStaticMapUrl": "", + "baiduPcStaticMapUrl": "", + "addressIcons": None, + "salaryMonth": 0, + "performance": "", + "salarySchemeDesc": None, + "salaryWelfareInfo": None, + "jobType": 0, + "jobSource": 0, + "daysPerWeekDesc": "", + "leastMonthDesc": "", + "salaryDesc": "18-35K·16薪", + "salaryDescAppend": None, + "salaryDetail": None, + "afterNameIcon": None, + "afterNameIcons": [], + "proxyJob": 0, + "proxyType": 0, + "anonymous": 0, + "salaryType": 0, + "newAddressTypeDesc": "", + "recruitNumDesc": "", + "jobSkills": None, + "jobTemplateModule": None, + "blueCollarPosition": False, + "graduateYearDesc": None, + "recruitEndTimeDesc": None, + "partTimeJobDesc": None, + "overseasAddressInfo": None, + "overseasInfo": None, + "overseasWelfareList": None + }, + "brandComInfoVO": { + "encryptBrandId": "08fbb99a8f9314e21nF73Nq9E1E~", + "brandName": "华为云计算", + "proxyBrandName": None, + "customerBrandName": None, + "logo": "https://img.bosszhipin.com/beijin/upload/admin/20241009/7a57a5a743894a0e61db0c9f94e55da3789b92c168491f5575a8c6cfd5eed090fc41aa91be88d106.png.webp", + "industryName": "计算机服务", + "stageName": "不需要融资", + "scaleName": "10000人以上", + "brandIntroduce": "华为云持续践行“一切皆服务”战略，通过基础设施即服务、技术即服务和经验即服务，帮助客户加速释放数字生产力。入选Gartner® 2022年云基础设施和平台服务(CIPS)魔力象限。全球开发者数量超过400万。", + "comName": "华为云计算技术有限公司", + "legalPerson": "赵明路", + "startDate": "2019-12-06", + "regCapital": "500000万人民币", + "statusDesc": "存续", + "regAddress": "贵州省贵安新区黔中大道交兴功路华为云数据中心", + "businessScope": "法律、法规、国务院决定规定禁止的不得经营；法律、法规、国务院决定规定应当许可（审批）的，经审批机关批准后凭许可（审批）文件经营；法律、法规、国务院决定规定无需许可（审批）的，市场主体自主选择经营。（一般项目：云计算装备技术服务；云计算设备制造；云计算设备销售；软件开发；软件销售；软件外包服务；技术服务、技术开发、技术咨询、技术交流、技术转让、技术推广；信息技术咨询服务；企业管理咨询；人工智能基础软件开发；人工智能基础资源与技术平台；人工智能应用软件开发；区块链技术相关软件和服务；物联网技术服务；物联网应用服务；量子计算技术服务；大数据服务；互联网数据服务；信息系统集成服务；信息系统运行维护服务；商用密码产品生产；计算机软硬件及外围设备制造；计算机软硬件及辅助设备批发；计算机软硬件及辅助设备零售；计算机及通讯设备租赁；非居住房地产租赁；业务培训（不含教育培训、职业技能培训等需取得许可的培训）；广告制作；广告发布；互联网销售（除销售需要许可的商品）；销售代理；技术进出口；货物进出口；集成电路设计；数字内容制作服务（不含出版发行）；网络技术服务；会议及展览服务（除许可业务外，可自主依法经营法律法规非禁止或限制的项目）许可项目：第一类增值电信业务；第二类增值电信业务；互联网信息服务；网络文化经营；互联网直播技术服务；信息网络传播视听节目；互联网域名注册服务（依法须经批准的项目，经相关部门批准后方可开展经营活动））", + "srcFromDesc": "", + "srcUrl": None, + "brandWelfares": [], + "canJumpToBrandPage": True, + "introduce": "华为云持续践行“一切皆服务”战略，通过基础设施即服务、技术即服务和经验即服务，帮助客户加速释放数字生产力。入选Gartner® 2022年云基础设施和平台服务(CIPS)魔力象限。全球开发者数量超过400万。", + "encryptComId": "54a167afc64130921X170tm9GFo~" + }, + "relationInfoVO": { + "interested": False, + "applyJobDirectlyStyle": 0, + "applyJobDirectlyType": 0, + "friend": False + }, + "relatedJobInfo": { + "title": "该公司其他岗位", + "jobType": 0, + "relatedJobList": [ + { + "jobName": "留学生招聘-区块链工程师", + "jobValidStatus": 1, + "jobExperience": "在校/应届", + "jobDegree": "硕士", + "cityName": "深圳", + "districtName": "龙岗区", + "businessName": "坂田", + "distance": "", + "lid": "7wHHKOw8Wwq.search.1", + "performance": "", + "salaryDesc": "15-30K", + "jobLabels": [ + "在校/应届", + "硕士", + "Golang", + "Java", + "计算机相关专业" ], - "performanceBonus": "", - "welfareLabel": [], - "welfareTags": [] - }, - "jobType": { - "jobType": "", - "jobTypeLevel": "15000400000000", - "jobTypeLevelName": "", - "subJobType": "", - "subJobTypeLevel": "15000400170000", - "subJobTypeLevelName": "" - }, - "onlineCarHailingExtend": { - "gray": False, - "promiseGuarantee": "" - }, - "onlineCarInfo": [], - "other": { - "customJobGroup": "DEFAULT", - "deliveredPreviouslyTip": "", - "jobKeyword": { - "keywords": [] + "highlightJobLabels": [ + { + "content": "在校/应届", + "highlight": False + }, + { + "content": "硕士", + "highlight": False + }, + { + "content": "Golang", + "highlight": False + }, + { + "content": "Java", + "highlight": False + }, + { + "content": "计算机相关专业", + "highlight": False + } + ], + "jobType": 5, + "encryptJobId": "63608fff5ee3aa8b1HR_0967FFdV", + "afterNameIcon": None, + "afterNameIcons": [ + "https://img.bosszhipin.com/beijin/mcs/banner/4d77f815083a7850d8c5188f10b2ff0dcfcd208495d565ef66e7dff9f98764da.png" + ], + "beforeNameIcons": [], + "picVideoList": [], + "positionName": None, + "commuteInfo": None, + "optimalTag": None, + "jobSource": 0, + "iconUrl": "", + "jobDesc": None, + "brandName": "华为云计算", + "brandLogo": "https://img.bosszhipin.com/beijin/upload/admin/20241009/7a57a5a743894a0e61db0c9f94e55da3789b92c168491f5575a8c6cfd5eed090fc41aa91be88d106.png.webp", + "brandStage": "不需要融资", + "brandScale": "10000人以上", + "comName": None, + "bossName": "奉先生", + "bossTitle": "开发工程师", + "bossAvatar": "https://img.bosszhipin.com/beijin/upload/avatar/20230818/607f1f3d68754fd0c728075c6087f79e9a2b5f4fce32b0e04f1bd0cb7cabd77a9f2b1cc10c32ff98_s.jpg.webp", + "bossCert": 0, + "activeMsg": "", + "bossOnline": None, + "proxyJob": 0, + "skills": [ + "Golang", + "Java", + "计算机相关专业", + "云服务开发", + "区块链", + "web 3" + ], + "welfares": [], + "securityId": "e1NT-74GVDTFO-K1LmCr5CvkX_PLzUQXaS5dnIZpJ_CdnXn4winPNzC2yK0md_EjYwQV7Q7ZDecswKE_rls5M3DlJQB1U78oLXWL_w5avmouCcgjS7uxesDeL8MJHdpw_rbhIg8DIw_kNfnYUcfkuaOs4cKJ2zOXCgv-LPKsW_gpO-_J1iKr6iRstQP5", + "sessionId": "", + "replayRate": "", + "jobNameLineNumber": None, + "bottomRcdReason": None, + "gps": { + "longitude": 114.059362, + "latitude": 22.65073 }, - "jobSkillTags": [], - "jobTypeIsBlueCollar": False, - "overseasList": [], - "pageStyle": 0, - "positionCommercialLabel": [], - "positionHighlight": "", - "propertyTypeUrl": "", - "rpoProxyDisplayOrgName": "", - "urgentRecruitmentUrl": "" - }, - "preferredHrInfo": { - "icon": "", - "introduce": "", - "jumpUrl": "", - "preferredHr": False - }, - "todayInterview": None, - "workLocation": { - "address": "工作地点：宝安区 · 石岩", - "addressType": 0, - "latitude": "22.6397107496406", - "longitude": "113.92158006824309", - "positionCityDistrict": "", - "positionCityDistrictCode": "", - "positionCityId": "765", - "positionWorkCity": "", - "showMap": True, - "showMultiAddressesTip": "", - "staticMapUrl": "https://storage-public.zhaopin.cn/job/share/1772075366278481083/6e4958065143498cad2bbfc1aef40d83", - "streetName": "", - "tradingArea": "", - "travelMode": "bus", - "verifyTheTruthUrl": "", - "workAddress": "深圳宝安创维创新谷-8号楼" - } - }, - "proxyWarning": None, - "recommender": { - "avatar": "", - "name": "", - "state": 0, - "text": "", - "title": "" - }, - "secure": { - "abroadLabel": "", - "abroadTipInfo": None, - "safeCenter": None, - "safetyReminder": None - }, - "staff": { - "activityLevel": [ - "高回复率" - ], - "auditNaturePrompt": None, - "authenticationState": 0, - "avatar": "https://storage-public.zhaopin.cn/zp/clouddisk/1771900452818918625/fee00116-221c-49df-a42a-6fb6f9cfcd09.png?x-oss-process=image/resize,l_240/rotate,0", - "companyName": "深圳市安保医疗科技股份有限公司", - "goldMedalInterviewer": None, - "greeting": "", - "greetingHasDelivery": "", - "hrJob": "HR", - "hrOnlineIocState": 0, - "hrOnlineState": "", - "hrResumeOperationState": "", - "hrStateInfo": "", - "id": 1168020727, - "lastOnlineTime": 0, - "lastOnlineTimeText": "", - "modularState": 1, - "other": { - "freeTag": None, - "tagUrl": "" - }, - "positionDetailStaffQuickReply": None, - "staffName": "陈女士" - }, - "stateInfo": { - "deliveryAfterGuide": None, - "imSessionInfoDetail": { - "imChatStatus": 0, - "imChatStatusForChatBeforeDelivery": 0, - "imDeliveryTitle": "", - "referType": -1 - }, - "positionBehaviorState": { - "deliveryState": 0, - "favoriteState": 0, - "followHrState": 0, - "imReplyState": -1, - "negativeState": 0, - "sessionChatState": 0 - }, - "signUpStatusInfo": {}, - "state": { - "abroadFlag": 2, - "applyType": "", - "callProcess": "", - "hasAppliedPosition": False, - "positionDeliveryType": "", - "positionSourceType": 1, - "workMode": "ONSITE", - "workModeDesc": "" - }, - "useNewAfterDeliveryStyle": False - }, - "verifyTheTruth": None, - "verifyTrueFeedback": None - }, - "jobHitReason": "", - "jobHitReasonHighlights": [], - "jobId": 40829101009, - "jobKeyword": { - "keywords": [ - { - "itemValue": "" + "highSalary": 30, + "salaryType": 0, + "businessCode": 92, + "dividerInfo": None, + "contact": False, + "communicateType": 0, + "communicateText": None, + "contactCommunicateText": None, + "applyJobDirectlyType": 0 }, { - "itemValue": "" + "jobName": "Java开发工程师", + "jobValidStatus": 1, + "jobExperience": "经验不限", + "jobDegree": "本科", + "cityName": "深圳", + "districtName": "龙岗区", + "businessName": "坂田", + "distance": "", + "lid": "7wHHKOw8Wwq.search.2", + "performance": "", + "salaryDesc": "18-35K·14薪", + "jobLabels": [ + "经验不限", + "本科", + "Java", + "Spring", + "MySQL" + ], + "highlightJobLabels": [ + { + "content": "经验不限", + "highlight": False + }, + { + "content": "本科", + "highlight": False + }, + { + "content": "Java", + "highlight": False + }, + { + "content": "Spring", + "highlight": False + }, + { + "content": "MySQL", + "highlight": False + } + ], + "jobType": 0, + "encryptJobId": "3e383b21b9b1722503Ry3d20GVBT", + "afterNameIcon": None, + "afterNameIcons": [], + "beforeNameIcons": [], + "picVideoList": [], + "positionName": None, + "commuteInfo": None, + "optimalTag": None, + "jobSource": 0, + "iconUrl": "", + "jobDesc": None, + "brandName": "华为云计算", + "brandLogo": "https://img.bosszhipin.com/beijin/upload/admin/20241009/7a57a5a743894a0e61db0c9f94e55da3789b92c168491f5575a8c6cfd5eed090fc41aa91be88d106.png.webp", + "brandStage": "不需要融资", + "brandScale": "10000人以上", + "comName": None, + "bossName": "杨先生", + "bossTitle": "开发", + "bossAvatar": "https://img.bosszhipin.com/beijin/upload/avatar/20190710/4e71d55095afc32e17ba427045fc318dd325c4a519489376765d4cee5675e2df_s.png", + "bossCert": 0, + "activeMsg": "", + "bossOnline": None, + "proxyJob": 0, + "skills": [ + "Java", + "Spring", + "MySQL", + "PostgreSQL" + ], + "welfares": [], + "securityId": "AE6A4cCTJT04W-y1IjYfaOEVkhw75_8QcigWkSJw3-IOh5QPkSa97xA-0809k_TzDgdJ5fqIUWAZsEoUimbU4_VYH--tUraaZsy5I9OH7yX91wD5WJQfLzymnYq8PCGjfssRfgnSeOCBHxiF06UwDDZ1gJvdWf-4ooaF1bIBKW0yfS_X5obIGFHnC9E~", + "sessionId": "", + "replayRate": "", + "jobNameLineNumber": None, + "bottomRcdReason": None, + "gps": { + "longitude": 114.059803, + "latitude": 22.650574 + }, + "highSalary": 35, + "salaryType": 0, + "businessCode": 92, + "dividerInfo": None, + "contact": False, + "communicateType": 0, + "communicateText": None, + "contactCommunicateText": None, + "applyJobDirectlyType": 0 }, { - "itemValue": "" + "jobName": "数据库POC交付顾问、DBA", + "jobValidStatus": 1, + "jobExperience": "10年以上", + "jobDegree": "本科", + "cityName": "深圳", + "districtName": None, + "businessName": None, + "distance": "", + "lid": "7wHHKOw8Wwq.search.3", + "performance": "", + "salaryDesc": "30-60K", + "jobLabels": [ + "10年以上", + "本科", + "MySQL", + "Oracle", + "PostgreSQL" + ], + "highlightJobLabels": [ + { + "content": "10年以上", + "highlight": False + }, + { + "content": "本科", + "highlight": False + }, + { + "content": "MySQL", + "highlight": False + }, + { + "content": "Oracle", + "highlight": False + }, + { + "content": "PostgreSQL", + "highlight": False + } + ], + "jobType": 0, + "encryptJobId": "39b8eb3338261ad71HR909u9FVpR", + "afterNameIcon": None, + "afterNameIcons": [], + "beforeNameIcons": [], + "picVideoList": [], + "positionName": None, + "commuteInfo": None, + "optimalTag": None, + "jobSource": 0, + "iconUrl": "", + "jobDesc": None, + "brandName": "华为云计算", + "brandLogo": "https://img.bosszhipin.com/beijin/upload/admin/20241009/7a57a5a743894a0e61db0c9f94e55da3789b92c168491f5575a8c6cfd5eed090fc41aa91be88d106.png.webp", + "brandStage": "不需要融资", + "brandScale": "10000人以上", + "comName": None, + "bossName": "高先生", + "bossTitle": "软件开发工程师", + "bossAvatar": "https://img.bosszhipin.com/beijin/upload/avatar/20210111/5a220f0a8b632c4d26139629999ae595b16d293c0340651d052d153134f8c135_s.png", + "bossCert": 0, + "activeMsg": "", + "bossOnline": None, + "proxyJob": 0, + "skills": [ + "MySQL", + "Oracle", + "PostgreSQL", + "计算机相关专业", + "DBA工作经验" + ], + "welfares": [], + "securityId": "SchrttPYjwevG-G1LCg9fGqSCQ8n5VnpyX0n38YkLwUuUAZipvf2QfEXtxKe2E7CZ2mhEXatzPu3Vb7Sm1TYB7YHv4BTLY64ClvQ4B_5L7p-S0xe0RM4UP_Gu89lAuVsCjK9XgdmgivOMDerPwkl2OU9nN3TTKBvz-GZ-7bYckPOt6K02HeILnFuUIrZ", + "sessionId": "", + "replayRate": "", + "jobNameLineNumber": None, + "bottomRcdReason": None, + "gps": { + "longitude": 114.059362, + "latitude": 22.65073 + }, + "highSalary": 60, + "salaryType": 0, + "businessCode": 0, + "dividerInfo": None, + "contact": False, + "communicateType": 0, + "communicateText": None, + "contactCommunicateText": None, + "applyJobDirectlyType": 0 }, { - "itemValue": "" + "jobName": "AI交付工程师", + "jobValidStatus": 1, + "jobExperience": "10年以上", + "jobDegree": "本科", + "cityName": "深圳", + "districtName": "龙岗区", + "businessName": "坂田", + "distance": "", + "lid": "7wHHKOw8Wwq.search.4", + "performance": "", + "salaryDesc": "25-50K", + "jobLabels": [ + "10年以上", + "本科", + "大模型算法", + "图像算法", + "自然语言处理算法" + ], + "highlightJobLabels": [ + { + "content": "10年以上", + "highlight": False + }, + { + "content": "本科", + "highlight": False + }, + { + "content": "大模型算法", + "highlight": False + }, + { + "content": "图像算法", + "highlight": False + }, + { + "content": "自然语言处理算法", + "highlight": False + } + ], + "jobType": 0, + "encryptJobId": "16f877ce43f922b91HZ80tm_E1RT", + "afterNameIcon": None, + "afterNameIcons": [], + "beforeNameIcons": [], + "picVideoList": [], + "positionName": None, + "commuteInfo": None, + "optimalTag": None, + "jobSource": 0, + "iconUrl": "", + "jobDesc": None, + "brandName": "华为云计算", + "brandLogo": "https://img.bosszhipin.com/beijin/upload/admin/20241009/7a57a5a743894a0e61db0c9f94e55da3789b92c168491f5575a8c6cfd5eed090fc41aa91be88d106.png.webp", + "brandStage": "不需要融资", + "brandScale": "10000人以上", + "comName": None, + "bossName": "徐先生", + "bossTitle": "AI工程师", + "bossAvatar": "https://img.bosszhipin.com/beijin/upload/avatar/20201228/7b760d8168e09caef6ad2fd10ed09dd9b2c6b8f046ab0353d624d868879abc74_s.jpg", + "bossCert": 0, + "activeMsg": "", + "bossOnline": None, + "proxyJob": 0, + "skills": [ + "大模型算法", + "图像算法", + "自然语言处理算法", + "多模态算法", + "Python", + "Shell", + "算法工程化经验", + "团队管理经验" + ], + "welfares": [], + "securityId": "vCNEiRilHcqpS-X1KdAQgJJ73dDzkAi9YY0X_j8kxliKH9569pxHa5CY0-g21EOsDo5uvVt96teOFes3ay8_sBAZmhTG3koiySm0r3Jryt3m4BRaJDHJGolbdS3HVFpIjQijspE_NBG7dW3PAE6iA8qhV98UF5klsN7SvrYkC-f15eCOnmWAoQIlPmsa", + "sessionId": "", + "replayRate": "", + "jobNameLineNumber": None, + "bottomRcdReason": None, + "gps": { + "longitude": 114.059803, + "latitude": 22.650574 + }, + "highSalary": 50, + "salaryType": 0, + "businessCode": 92, + "dividerInfo": None, + "contact": False, + "communicateType": 0, + "communicateText": None, + "contactCommunicateText": None, + "applyJobDirectlyType": 0 }, { - "itemValue": "" - }, - { - "itemValue": "" - }, - { - "itemValue": "" - }, - { - "itemValue": "" - }, - { - "itemValue": "" - }, - { - "itemValue": "" - }, - { - "itemValue": "" - }, - { - "itemValue": "" + "jobName": "高级后端工程师", + "jobValidStatus": 1, + "jobExperience": "经验不限", + "jobDegree": "本科", + "cityName": "深圳", + "districtName": "龙岗区", + "businessName": "坂田", + "distance": "", + "lid": "7wHHKOw8Wwq.search.5", + "performance": "", + "salaryDesc": "30-50K", + "jobLabels": [ + "经验不限", + "本科", + "Java", + "C", + "Python" + ], + "highlightJobLabels": [ + { + "content": "经验不限", + "highlight": False + }, + { + "content": "本科", + "highlight": False + }, + { + "content": "Java", + "highlight": False + }, + { + "content": "C", + "highlight": False + }, + { + "content": "Python", + "highlight": False + } + ], + "jobType": 0, + "encryptJobId": "587e4f740b5dd3fb1HN53d2_FlJU", + "afterNameIcon": None, + "afterNameIcons": [], + "beforeNameIcons": [], + "picVideoList": [], + "positionName": None, + "commuteInfo": None, + "optimalTag": None, + "jobSource": 0, + "iconUrl": "", + "jobDesc": None, + "brandName": "华为云计算", + "brandLogo": "https://img.bosszhipin.com/beijin/upload/admin/20241009/7a57a5a743894a0e61db0c9f94e55da3789b92c168491f5575a8c6cfd5eed090fc41aa91be88d106.png.webp", + "brandStage": "不需要融资", + "brandScale": "10000人以上", + "comName": None, + "bossName": "黎女士", + "bossTitle": "HR", + "bossAvatar": "https://img.bosszhipin.com/beijin/upload/avatar/20230302/607f1f3d68754fd0575392c2dfa3d3eaf571c5d252feb32d9dbc31137d0c19f702517649a85c3775_s.jpg", + "bossCert": 0, + "activeMsg": "", + "bossOnline": None, + "proxyJob": 0, + "skills": [ + "Java", + "C", + "Python" + ], + "welfares": [], + "securityId": "1w9YjXv2tVewC-21FaMhYKnjhQptZ51EmmQJO7lcw7NrDrg2dtkMCA8Gbb9ulSo5-pKoaX8u0dQStZ6al4OL2ibFHKCaAy82mFILT5YH7KtWyG3ff9qb2qPWvNArwf3Pzn3lwelbC63mHyoRX9yH9CCINpgfVGzTU3M_AlmENE-XpEc3vJDAe-vO38ig", + "sessionId": "", + "replayRate": "", + "jobNameLineNumber": None, + "bottomRcdReason": None, + "gps": { + "longitude": 114.068154, + "latitude": 22.660299 + }, + "highSalary": 50, + "salaryType": 0, + "businessCode": 92, + "dividerInfo": None, + "contact": False, + "communicateType": 0, + "communicateText": None, + "contactCommunicateText": None, + "applyJobDirectlyType": 0 } ] }, - "jobKnowledgeWelfareFeatures": [], - "jobPostingTime": 1769052217250, - "jobRootOrgInfo": { - "cityName": "深圳", - "reviewOrgNature": 1 + "safeTipInfo": { + "title": "BOSS安全提示", + "safeReport": { + "name": "BOSS直聘严禁用人单位和招聘者用户做出任何损害求职者合法权益的违法违规行为，包括但不限于扣押求职者证件、收取求职者财物、向求职者集资、让求职者入股、诱导求职者异地入职、异地参加培训、违法违规使用求职者简历等，您一旦发现此类行为，请立即举报", + "code": "BOSS直聘严禁用人单位和招聘者用户做出任何损害求职者合法权益的违法违规行为，包括但不限于扣押求职者证件、收取求职者财物、向求职者集资、让求职者入股、诱导求职者异地入职、异地参加培训、违法违规使用求职者简历等，您一旦发现此类行为，", + "hlname": "请立即举报", + "highlightList": [ + { + "startIndex": 115, + "endIndex": 120 + } + ] + }, + "riskTip": None, + "safeGuidTip": None, + "safeGuidUrl": None }, - "jobSkillTags": [ - { - "id": 19824127, - "name": "医疗器械质量管理体系", - "standard": False - }, - { - "id": 19381048, - "name": "ISO认证", - "standard": False - }, - { - "id": 18958610, - "name": "GMP认证", - "standard": False - }, - { - "id": 19366784, - "name": "FDA认证", - "standard": False - }, - { - "id": 488082095, - "name": "二三类医械经验", - "standard": False - } - ], - "jobSummary": "岗位职责：1、负责公司质量管理体系的策划，运营，维护和合规性的管理；2、负责公司各类医疗器械质量管理体系（国内GMP、ISO13485、QSR820、MDSAP、 MDR、ISO9001、 职业健康和安全，环境管理体系等）的运营、维护与升级； 3、负责公司各类外部体系审核工作（国内飞检、CE审核、飞检等）的内部协调与组织工作； 4、负责生产、经营数据上报，及负责监控不良事件上报； 5、负责相关行业...", - "liveCard": { - "icon": "", - "liveState": 0, - "liveTips": "", - "roomId": 0, - "startTimeFormat": "", - "videoUrl": "" + "workEnvironment": None, + "bannerList": None, + "commitment": None, + "homeAddressInfo": None, + "handicappedInfo": None, + "highlightInfo": None, + "atsOnlineApplicationInfo": None, + "appendixInfo": { + "douYinIllegalJob": False }, - "matchInfo": { - "icon": "", - "matched": 0, - "tagState": 0 - }, - "menVipLevel": 0, - "name": "质量体系工程师", - "needMajor": [], - "number": "CC219609310J40829101009", - "orgBestEmployerFlag": 1, - "orgCommercialTags": [], - "orgPayedFlag": 1, - "payload": { - "name": "", - "partition": "", - "score": "", - "weight": "" - }, - "positionCommercialLabel": [], - "positionExpandCardData": "", - "positionExpandCardType": 0, - "positionHighlight": "", - "positionOfNlp": 1, - "positionSourceType": 1, - "positionSourceTypeUrl": "", - "positionURL": "http://jobs.zhaopin.com/CC219609310J40829101009.htm", - "positionUrl": "http://jobs.zhaopin.com/CC219609310J40829101009.htm", - "property": "民营", - "propertyCode": "5", - "propertyName": "民营", - "propertyType": "", - "propertyTypeUrl": "", - "provideInternshipCertificate": False, - "proxyModel": { - "proxiedOrgName": "", - "proxiedOrgSize": "", - "recruitPosition": 0 - }, - "publishTime": "2026-03-05 09:51:08", - "recallSign": { - "gMethod": "config-position_search-position_mbscore-ANONYMOUS-welfare-words", - "gParam": "query-ps-mbscore-3", - "gQuery": "query-ps-mbscore-3", - "gSort": "query-ps-mbscore-3", - "gSource": "solr.source_position_query", - "gWeight": 0 - }, - "recruitNumber": 1, - "redirectUrl": "", - "redirectable": False, - "rootCompanyNumber": "CZ219609310", - "rpoProxied": False, - "rpoProxy": False, - "salary60": "1.2-1.8万", - "salaryCount": "", - "salaryReal": "12001-18000", - "salaryType": 1, - "searchTagList": [], - "securityAddressLabel": "", - "settlementType": "", - "showDistance": 0, - "showSkillTags": [ - { - "highlightBackGroundColor": "", - "highlightWordColor": "", - "tag": "3-5年" - }, - { - "highlightBackGroundColor": "", - "highlightWordColor": "", - "tag": "本科" - }, - { - "highlightBackGroundColor": "", - "highlightWordColor": "", - "tag": "医疗器械质量管理体系" - }, - { - "highlightBackGroundColor": "", - "highlightWordColor": "", - "tag": "ISO认证" - }, - { - "highlightBackGroundColor": "", - "highlightWordColor": "", - "tag": "GMP认证" - }, - { - "highlightBackGroundColor": "", - "highlightWordColor": "", - "tag": "FDA认证" - }, - { - "highlightBackGroundColor": "", - "highlightWordColor": "", - "tag": "二三类医械经验" - }, - { - "highlightBackGroundColor": "", - "highlightWordColor": "", - "tag": "医疗设备/器械" - } - ], - "skillLabel": [ - { - "state": 0, - "value": "医疗器械质量管理体系" - }, - { - "state": 0, - "value": "ISO认证" - }, - { - "state": 0, - "value": "GMP认证" - }, - { - "state": 0, - "value": "FDA认证" - }, - { - "state": 0, - "value": "二三类医械经验" - } - ], - "skillLabelPersonality": "", - "staffCard": { - "authenticationState": 0, - "avatar": "https://storage-public.zhaopin.cn/zp/clouddisk/1771900452818918625/fee00116-221c-49df-a42a-6fb6f9cfcd09.png?x-oss-process=image/resize,l_240/rotate,0", - "goldMedalInterviewer": { - "goldMedalInterviewer": False, - "interviewerImageUrl": "", - "interviewerTitle": "" - }, - "hrCompanyName": "", - "hrJob": "HR", - "hrOnlineIocState": 0, - "hrOnlineState": "三日内活跃", - "hrStateInfo": "高回复率", - "id": 1168020727, - "lastOnlineTime": 1774084085367, - "lastOnlineTimeText": "", - "staffName": "陈女士" - }, - "streetId": 44030605, - "streetName": "石岩", - "subJobTypeLevel": "15000400170000", - "subJobTypeLevelName": "质量体系工程师", - "subways": [], - "tagABC": "", - "tagList": [], - "todayInterview": False, - "todayInterviewImageUrl": "", - "topLabel": None, - "tradingArea": "", - "volcanoMeterial": None, - "weeklyInternshipDays": 0, - "welfareLabel": [], - "welfareTagList": [], - "workCity": "深圳", - "workDateType": "", - "workMode": "", - "workType": "全职", - "workTypeCode": "2", - "workingExp": "3-5年" + "certMaterials": None, + "showGroupTip": False } } \ No newline at end of file diff --git a/app/services/ingest/remote_push.py b/app/services/ingest/remote_push.py index 4ce62dc..ba11fb9 100644 --- a/app/services/ingest/remote_push.py +++ b/app/services/ingest/remote_push.py @@ -59,6 +59,8 @@ async def push_to_remote(data: Dict[str, Any]) -> bool: source_type = data.get("source_type", "未知平台") title = data.get("title", "未知职位") company = data.get("company_name", data.get("name", "未知公司")) + logger.info(f"上报数据: [{source_type}] {title} - {company}") + print(data) try: url = _build_auth_url() diff --git a/docker-compose.spider.yml b/docker-compose.spider.yml new file mode 100644 index 0000000..d47ebc7 --- /dev/null +++ b/docker-compose.spider.yml @@ -0,0 +1,40 @@ +version: "3.8" + +x-spider-common: &spider-common + build: + context: . + dockerfile: spider.Dockerfile + restart: unless-stopped + environment: &spider-env + API_BASE_URL: ${API_BASE_URL:-http://124.222.106.226:9999} + API_TOKEN: ${API_TOKEN:-dev} + SLEEP_MIN_SECONDS: ${SLEEP_MIN_SECONDS:-5} + SLEEP_MAX_SECONDS: ${SLEEP_MAX_SECONDS:-12} + MAX_PAGES: "100" + INLINE_COMPANY: "0" + # 代理已内置，设 PROXY_TUNNEL=none 可禁用 + +services: + # ── Boss直聘 ── + spider-boss: + <<: *spider-common + environment: + <<: *spider-env + PLATFORM: boss + INSTANCES: "3" + + # ── 前程无忧 ── + spider-qcwy: + <<: *spider-common + environment: + <<: *spider-env + PLATFORM: qcwy + INSTANCES: "3" + + # ── 智联招聘 ── + spider-zhilian: + <<: *spider-common + environment: + <<: *spider-env + PLATFORM: zhilian + INSTANCES: "3" diff --git a/ecs_full_pipeline.log b/ecs_full_pipeline.log index b0e9ab6..eeadd0b 100644 --- a/ecs_full_pipeline.log +++ b/ecs_full_pipeline.log @@ -36,3 +36,53 @@ https://api.aliyun.com/troubleshoot?q=InvalidAccountStatus.NotEnoughBalance&prod 未获得实例ID，终止 [定时] 开始执行 pipeline：2026-03-22T18:30:00.007107 +[main] start clearing instances with prefix launch-advisor-20251123 +当前地域无实例或无匹配实例，无需清理 +[main] clearing completed +[创建] 正在提交创建实例请求 +InvalidAccountStatus.NotEnoughBalance +code: 403, Your account does not have enough balance to order postpaid product. request id: AFA9356D-151E-5F85-82CC-DA00A0A5D2DF +https://api.aliyun.com/troubleshoot?q=InvalidAccountStatus.NotEnoughBalance&product=Ecs&requestId=AFA9356D-151E-5F85-82CC-DA00A0A5D2DF +未获得实例ID，终止 + +[定时] 开始执行 pipeline：2026-03-23T00:30:00.005997 +[main] start clearing instances with prefix launch-advisor-20251123 +当前地域无实例或无匹配实例，无需清理 +[main] clearing completed +[创建] 正在提交创建实例请求 +InvalidAccountStatus.NotEnoughBalance +code: 403, Your account does not have enough balance to order postpaid product. request id: 8831C224-BA9F-5E84-9AA2-8BDCA00C6B21 +https://api.aliyun.com/troubleshoot?q=InvalidAccountStatus.NotEnoughBalance&product=Ecs&requestId=8831C224-BA9F-5E84-9AA2-8BDCA00C6B21 +未获得实例ID，终止 + +[定时] 开始执行 pipeline：2026-03-23T06:30:00.006451 +[main] start clearing instances with prefix launch-advisor-20251123 +当前地域无实例或无匹配实例，无需清理 +[main] clearing completed +[创建] 正在提交创建实例请求 +InvalidAccountStatus.NotEnoughBalance +code: 403, Your account does not have enough balance to order postpaid product. request id: 60394E0C-C0A4-5D7D-8380-49A7D5EC37A5 +https://api.aliyun.com/troubleshoot?q=InvalidAccountStatus.NotEnoughBalance&product=Ecs&requestId=60394E0C-C0A4-5D7D-8380-49A7D5EC37A5 +未获得实例ID，终止 + +[定时] 开始执行 pipeline：2026-03-23T12:30:00.003733 +[main] start clearing instances with prefix launch-advisor-20251123 +当前地域无实例或无匹配实例，无需清理 +[main] clearing completed +[创建] 正在提交创建实例请求 +InvalidAccountStatus.NotEnoughBalance +code: 403, Your account does not have enough balance to order postpaid product. request id: 79E1B24A-4D5A-533D-97CB-71616BB5336B +https://api.aliyun.com/troubleshoot?q=InvalidAccountStatus.NotEnoughBalance&product=Ecs&requestId=79E1B24A-4D5A-533D-97CB-71616BB5336B +未获得实例ID，终止 + +[定时] 开始执行 pipeline：2026-03-23T18:30:00.002226 +[main] start clearing instances with prefix launch-advisor-20251123 +当前地域无实例或无匹配实例，无需清理 +[main] clearing completed +[创建] 正在提交创建实例请求 +InvalidAccountStatus.NotEnoughBalance +code: 403, Your account does not have enough balance to order postpaid product. request id: 309194E6-612F-5921-96A1-C6620E0BE99B +https://api.aliyun.com/troubleshoot?q=InvalidAccountStatus.NotEnoughBalance&product=Ecs&requestId=309194E6-612F-5921-96A1-C6620E0BE99B +未获得实例ID，终止 + +[定时] 开始执行 pipeline：2026-03-24T00:30:00.004105 diff --git a/spider-entrypoint.sh b/spider-entrypoint.sh new file mode 100644 index 0000000..9c6cc40 --- /dev/null +++ b/spider-entrypoint.sh @@ -0,0 +1,55 @@ +#!/bin/bash +set -e + +PLATFORM="${PLATFORM:-boss}" +INSTANCES="${INSTANCES:-1}" + +echo "==========================================" +echo " Spider Container" +echo " Platform: ${PLATFORM}" +echo " Instances: ${INSTANCES}" +echo " API: ${API_BASE_URL}" +echo " Delay: ${SLEEP_MIN_SECONDS}-${SLEEP_MAX_SECONDS}s" +echo " Inline Co: ${INLINE_COMPANY}" +echo "==========================================" + +# 平台 -> Python 模块映射 +case "${PLATFORM}" in + boss) MODULE="spiderJobs.platforms.boss.main" ;; + qcwy) MODULE="spiderJobs.platforms.job51.main" ;; + zhilian) MODULE="spiderJobs.platforms.zhilian.main" ;; + *) + echo "[ERROR] Unknown PLATFORM: ${PLATFORM}" + echo " Supported: boss, qcwy, zhilian" + exit 1 + ;; +esac + +# 启动多个实例（最后一个前台运行） +PIDS=() +for i in $(seq 1 $((INSTANCES - 1))); do + echo "[spider] Starting ${PLATFORM} instance ${i}/${INSTANCES} (background)..." + python -m "${MODULE}" & + PIDS+=($!) + sleep 2 # 错开启动，避免同时抢同一个关键词 +done + +echo "[spider] Starting ${PLATFORM} instance ${INSTANCES}/${INSTANCES} (foreground)..." +python -m "${MODULE}" & +PIDS+=($!) + +# 等待所有子进程，任一退出则全部退出 +wait_and_cleanup() { + echo "[spider] Shutting down all instances..." + for pid in "${PIDS[@]}"; do + kill "$pid" 2>/dev/null || true + done + wait + echo "[spider] All instances stopped." +} + +trap wait_and_cleanup SIGTERM SIGINT + +# 等任意子进程退出 +wait -n +wait_and_cleanup diff --git a/spider.Dockerfile b/spider.Dockerfile new file mode 100644 index 0000000..4f8412b --- /dev/null +++ b/spider.Dockerfile @@ -0,0 +1,41 @@ +FROM python:3.11-slim-bullseye + +WORKDIR /opt/spider + +# 时区 + 基础工具 +RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \ + && echo "Asia/Shanghai" > /etc/timezone \ + && apt-get update \ + && apt-get install -y --no-install-recommends curl \ + && rm -rf /var/lib/apt/lists/* + +# Python 依赖（先复制依赖文件，利用 Docker 缓存） +COPY crawler_core/pyproject.toml /opt/spider/crawler_core/pyproject.toml +RUN pip install --no-cache-dir \ + requests_go==1.0.9 \ + tenacity>=8.0 \ + requests \ + PySocks \ + six \ + -i https://pypi.tuna.tsinghua.edu.cn/simple + +# 复制代码 +COPY crawler_core/ /opt/spider/crawler_core/ +COPY spiderJobs/ /opt/spider/spiderJobs/ + +# 默认环境变量 +ENV API_BASE_URL=http://124.222.106.226:9999 \ + API_TOKEN=dev \ + PLATFORM=boss \ + SLEEP_MIN_SECONDS=5 \ + SLEEP_MAX_SECONDS=12 \ + MAX_PAGES=100 \ + INLINE_COMPANY=0 \ + PYTHONUNBUFFERED=1 +# 代理已内置到代码中（spiderJobs/__init__.py） +# 设 PROXY_TUNNEL=none 可禁用代理 + +COPY spider-entrypoint.sh /opt/spider/entrypoint.sh +RUN chmod +x /opt/spider/entrypoint.sh + +ENTRYPOINT ["/opt/spider/entrypoint.sh"] diff --git a/spiderJobs/__init__.py b/spiderJobs/__init__.py index e69de29..f6e8b64 100644 --- a/spiderJobs/__init__.py +++ b/spiderJobs/__init__.py @@ -0,0 +1,3 @@ +# 默认隧道代理（快代理），每次请求自动换 IP +# 环境变量 PROXY_TUNNEL 可覆盖；设为 "none" 则禁用代理 +DEFAULT_TUNNEL_PROXY = "http://t13319619426654:ln8aj9nl@s432.kdltps.com:15818" diff --git a/spiderJobs/aaa.txt b/spiderJobs/aaa.txt new file mode 100644 index 0000000..82da385 --- /dev/null +++ b/spiderJobs/aaa.txt @@ -0,0 +1,55 @@ + 容器运行正常，成功获取到关键词并开始爬取。 + + 创建完成，3 个文件 + + ┌───────────────────────────┬─────────────────────────────────┐ + │ 文件 │ 用途 │ + ├───────────────────────────┼─────────────────────────────────┤ + │ spider.Dockerfile │ 爬虫专用镜像，轻量无前端/nginx │ + ├───────────────────────────┼─────────────────────────────────┤ + │ spider-entrypoint.sh │ 支持单容器多进程（INSTANCES=3） │ + ├───────────────────────────┼─────────────────────────────────┤ + │ docker-compose.spider.yml │ 一键启动三个平台 │ + └───────────────────────────┴─────────────────────────────────┘ + + 使用方式 + + 单机快速启动（3 个平台各 3 个实例 = 9 个爬虫）： + docker compose -f docker-compose.spider.yml up -d + + 多台电脑部署： + # 1. 构建并推送到你的镜像仓库 + docker build -f spider.Dockerfile -t your-registry/jobdata-spider:latest . + docker push your-registry/jobdata-spider:latest + + # 2. 每台电脑上拉取运行 + docker run -d --name spider-boss \ + -e PLATFORM=boss \ + -e INSTANCES=3 \ + -e API_BASE_URL=http://124.222.106.226:9999 \ + -e SLEEP_MIN_SECONDS=5 \ + -e SLEEP_MAX_SECONDS=12 \ + -e INLINE_COMPANY=0 \ + your-registry/jobdata-spider:latest + + 调参： + # 加速：降低延迟，多开实例 + -e SLEEP_MIN_SECONDS=3 -e SLEEP_MAX_SECONDS=8 -e INSTANCES=5 + + # 需要代理 + -e PROXY_TUNNEL=proxy.example.com:8080 -e PROXY_USERNAME=xxx -e PROXY_PASSWORD=xxx + + # Boss 需要 Token（会自动从后端 API 获取，也可手动指定） + -e BOSS_MPT=xxx -e BOSS_WT2=xxx + + 默认配置已优化：延迟 5-12 秒（比之前 10-20 + 快一倍），INLINE_COMPANY=0（关掉内联公司爬取）。3 台电脑每台 INSTANCES=3，Boss + 吞吐量可以从现在的 5 个/小时 → ~270 个/小时。 + +✻ Cooked for 2m 16s + +─────────────────────────────────────────────────────────────────────────────────────── +❯  +─────────────────────────────────────────────────────────────────────────────────────── + Opus 4.6 (1M context) │ JobData █░░░░░░░░░ 14% + ⏵⏵ accept edits on (shift+tab to cycle) diff --git a/spiderJobs/platforms/boss/main.py b/spiderJobs/platforms/boss/main.py index 8666810..59fff0d 100644 --- a/spiderJobs/platforms/boss/main.py +++ b/spiderJobs/platforms/boss/main.py @@ -141,15 +141,11 @@ def main(): signer = BossSign(mpt=mpt, wt2=wt2) client_kwargs["signer"] = signer - tunnel = os.environ.get("PROXY_TUNNEL", "") - if tunnel: - scheme = os.environ.get("PROXY_SCHEME", "http") - username = os.environ.get("PROXY_USERNAME", "") - password = os.environ.get("PROXY_PASSWORD", "") - if username and password: - client_kwargs["tunnel_proxy"] = f"{scheme}://{username}:{password}@{tunnel}" - else: - client_kwargs["tunnel_proxy"] = f"{scheme}://{tunnel}" + from spiderJobs import DEFAULT_TUNNEL_PROXY + tunnel_proxy = os.environ.get("PROXY_TUNNEL", DEFAULT_TUNNEL_PROXY) + if tunnel_proxy and tunnel_proxy.lower() != "none": + client_kwargs["tunnel_proxy"] = tunnel_proxy + print(f"[boss] 隧道代理: {tunnel_proxy.split('@')[-1] if '@' in tunnel_proxy else tunnel_proxy}") run_crawl_loop( platform="boss", diff --git a/spiderJobs/platforms/job51/main.py b/spiderJobs/platforms/job51/main.py index d036257..3ae27ab 100644 --- a/spiderJobs/platforms/job51/main.py +++ b/spiderJobs/platforms/job51/main.py @@ -90,15 +90,11 @@ def create_company_fetcher(company_id: str, http_client: Job51Client) -> BaseFet def main(): client_kwargs = {} - tunnel = os.environ.get("PROXY_TUNNEL", "") - if tunnel: - scheme = os.environ.get("PROXY_SCHEME", "http") - username = os.environ.get("PROXY_USERNAME", "") - password = os.environ.get("PROXY_PASSWORD", "") - if username and password: - client_kwargs["tunnel_proxy"] = f"{scheme}://{username}:{password}@{tunnel}" - else: - client_kwargs["tunnel_proxy"] = f"{scheme}://{tunnel}" + from spiderJobs import DEFAULT_TUNNEL_PROXY + tunnel_proxy = os.environ.get("PROXY_TUNNEL", DEFAULT_TUNNEL_PROXY) + if tunnel_proxy and tunnel_proxy.lower() != "none": + client_kwargs["tunnel_proxy"] = tunnel_proxy + print(f"[qcwy] 隧道代理: {tunnel_proxy.split('@')[-1] if '@' in tunnel_proxy else tunnel_proxy}") run_crawl_loop( platform="qcwy", diff --git a/spiderJobs/platforms/zhilian/client.py b/spiderJobs/platforms/zhilian/client.py index 5c1a026..d9c492a 100644 --- a/spiderJobs/platforms/zhilian/client.py +++ b/spiderJobs/platforms/zhilian/client.py @@ -51,6 +51,7 @@ class ZhilianClient(HTTPClient): self, base_url: str = CGATE_BASE_URL, signer: Optional[ZhilianSign] = None, + tunnel_proxy: Optional[str] = None, proxy: Optional[str] = None, proxy_pool: Optional[list[str]] = None, timeout: int = 10, @@ -58,6 +59,7 @@ class ZhilianClient(HTTPClient): super().__init__( base_url=base_url, default_headers=ZHILIAN_HEADERS, + tunnel_proxy=tunnel_proxy, proxy=proxy, proxy_pool=proxy_pool, timeout=timeout, @@ -81,18 +83,20 @@ class ZhilianClient(HTTPClient): def create_cgate_client( signer: Optional[ZhilianSign] = None, + tunnel_proxy: Optional[str] = None, proxy: Optional[str] = None, proxy_pool: Optional[list[str]] = None, ) -> ZhilianClient: """创建 cgate 客户端""" - return ZhilianClient(base_url=CGATE_BASE_URL, signer=signer, proxy=proxy, proxy_pool=proxy_pool) + return ZhilianClient(base_url=CGATE_BASE_URL, signer=signer, tunnel_proxy=tunnel_proxy, proxy=proxy, proxy_pool=proxy_pool) def create_capi_client( signer: Optional[ZhilianSign] = None, + tunnel_proxy: Optional[str] = None, proxy: Optional[str] = None, proxy_pool: Optional[list[str]] = None, ) -> ZhilianClient: """创建 capi 客户端""" - return ZhilianClient(base_url=CAPI_BASE_URL, signer=signer, proxy=proxy, proxy_pool=proxy_pool) + return ZhilianClient(base_url=CAPI_BASE_URL, signer=signer, tunnel_proxy=tunnel_proxy, proxy=proxy, proxy_pool=proxy_pool) diff --git a/spiderJobs/platforms/zhilian/main.py b/spiderJobs/platforms/zhilian/main.py index 58fb8a6..9f7a2ff 100644 --- a/spiderJobs/platforms/zhilian/main.py +++ b/spiderJobs/platforms/zhilian/main.py @@ -91,9 +91,12 @@ def create_company_fetcher(company_id: str, http_client: ZhilianClient) -> BaseF def main(): client_kwargs = {} - proxy = os.environ.get("PROXY_URL", "") - if proxy: - client_kwargs["proxy"] = proxy + + from spiderJobs import DEFAULT_TUNNEL_PROXY + tunnel_proxy = os.environ.get("PROXY_TUNNEL", DEFAULT_TUNNEL_PROXY) + if tunnel_proxy and tunnel_proxy.lower() != "none": + client_kwargs["tunnel_proxy"] = tunnel_proxy + print(f"[zhilian] 隧道代理: {tunnel_proxy.split('@')[-1] if '@' in tunnel_proxy else tunnel_proxy}") run_crawl_loop( platform="zhilian",