[{"data":1,"prerenderedAt":983},["ShallowReactive",2],{"header-counts":3,"footer-counts":6,"wiki-token":9},{"tools":4,"reviews":5},65,7,{"tools":4,"reviews":5,"playbooks":7,"news":8},10,8,{"id":10,"title":11,"body":12,"category":965,"description":37,"extension":966,"meta":967,"navigation":451,"path":968,"published":969,"relatedModels":970,"relatedTools":973,"seo":974,"slug":975,"stem":976,"summary":977,"tags":978,"updated":969,"__hash__":982},"wiki\u002Fwiki\u002Ftoken.md","Token（令牌）",{"type":13,"value":14,"toc":941},"minimark",[15,20,24,27,38,42,47,50,56,60,63,130,134,137,141,144,201,204,210,213,235,239,242,324,334,338,341,362,365,554,581,584,587,666,672,675,678,732,735,738,820,823,827,880,883,887,894,898,906,910,913,916,937],[16,17,19],"h2",{"id":18},"什么是-token","什么是 Token",[21,22,23],"p",{},"Token 是大语言模型处理文本的基本单位。模型不是逐字符或逐词处理文本，而是把文本切分成一个个 token。",[21,25,26],{},"对于英文：1 个 token ≈ 0.75 个单词（4 个字符）\n对于中文：1 个 token ≈ 1-2 个汉字",[28,29,34],"pre",{"className":30,"code":32,"language":33},[31],"language-text","英文: \"Hello world\"        → [\"Hello\", \" world\"]           → 2 tokens\n中文: \"你好世界\"           → [\"你好\", \"世界\"]               → 2-4 tokens\n代码: \"function add(a,b)\"  → [\"function\", \" add\", \"(\", \"a\", \",\", \"b\", \")\"] → 7 tokens\n","text",[35,36,32],"code",{"__ignoreMap":37},"",[16,39,41],{"id":40},"为什么-token-重要","为什么 Token 重要",[43,44,46],"h3",{"id":45},"_1-决定-api-费用","1. 决定 API 费用",[21,48,49],{},"所有 LLM API 都按 token 计费。Token 越多，费用越高。",[28,51,54],{"className":52,"code":53,"language":33},[31],"100 万 token ≈ 75 万英文单词 ≈ 50-100 万汉字\n",[35,55,53],{"__ignoreMap":37},[43,57,59],{"id":58},"_2-决定上下文窗口","2. 决定上下文窗口",[21,61,62],{},"模型能处理的上下文长度以 token 为单位：",[64,65,66,82],"table",{},[67,68,69],"thead",{},[70,71,72,76,79],"tr",{},[73,74,75],"th",{},"模型",[73,77,78],{},"上下文窗口",[73,80,81],{},"约等于",[83,84,85,97,108,119],"tbody",{},[70,86,87,91,94],{},[88,89,90],"td",{},"GPT-4o",[88,92,93],{},"128K",[88,95,96],{},"一本中篇小说",[70,98,99,102,105],{},[88,100,101],{},"Claude Sonnet 4",[88,103,104],{},"200K",[88,106,107],{},"一本长篇小说",[70,109,110,113,116],{},[88,111,112],{},"GPT-5",[88,114,115],{},"400K",[88,117,118],{},"两部长篇小说",[70,120,121,124,127],{},[88,122,123],{},"Gemini 2.5 Pro",[88,125,126],{},"1M",[88,128,129],{},"一套百科全书",[43,131,133],{"id":132},"_3-决定速度","3. 决定速度",[21,135,136],{},"Token 越多，推理时间越长。生成 1000 token 大约需要 3-10 秒（取决于模型）。",[16,138,140],{"id":139},"input-output-cache-三种价格","Input \u002F Output \u002F Cache 三种价格",[21,142,143],{},"API 计费早就不是\"按 token 一口价\"了，常见至少分三档：",[64,145,146,159],{},[67,147,148],{},[70,149,150,153,156],{},[73,151,152],{},"类别",[73,154,155],{},"含义",[73,157,158],{},"相对价格（粗略）",[83,160,161,175,188],{},[70,162,163,169,172],{},[88,164,165],{},[166,167,168],"strong",{},"Input",[88,170,171],{},"你发送给模型的 prompt（含 system + history + query）",[88,173,174],{},"1x",[70,176,177,182,185],{},[88,178,179],{},[166,180,181],{},"Output",[88,183,184],{},"模型生成的内容",[88,186,187],{},"3-5x Input",[70,189,190,195,198],{},[88,191,192],{},[166,193,194],{},"Cached Input",[88,196,197],{},"命中 prompt cache 的输入部分",[88,199,200],{},"0.1-0.5x Input",[21,202,203],{},"举例：Claude Sonnet 4（数量级，会变动）：",[28,205,208],{"className":206,"code":207,"language":33},[31],"Input:        $3 \u002F M token\nOutput:      $15 \u002F M token\nCache write: $3.75 \u002F M（写一次缓存，5 分钟有效，可续）\nCache read:  $0.30 \u002F M（命中后 -90%）\n",[35,209,207],{"__ignoreMap":37},[21,211,212],{},"设计应用时三句话原则：",[214,215,216,223,229],"ol",{},[217,218,219,222],"li",{},[166,220,221],{},"能少 output 就少 output","——Output 比 Input 贵 3-5 倍。让模型只返回必要内容、不要重复用户的话。",[217,224,225,228],{},[166,226,227],{},"能复用 input 就开缓存","——长 system prompt、tool definitions、知识库文档放最前面，缓存一次反复用。",[217,230,231,234],{},[166,232,233],{},"能批量就批量","——OpenAI \u002F Anthropic 都有 Batch API，24h 内出结果，价格 -50%。",[16,236,238],{"id":237},"prompt-cache-计费机制","Prompt Cache 计费机制",[21,240,241],{},"各家细节不同，但模式接近：",[64,243,244,260],{},[67,245,246],{},[70,247,248,251,254,257],{},[73,249,250],{},"平台",[73,252,253],{},"触发方式",[73,255,256],{},"TTL",[73,258,259],{},"折扣",[83,261,262,280,294,311],{},[70,263,264,267,274,277],{},[88,265,266],{},"Anthropic",[88,268,269,270,273],{},"显式 ",[35,271,272],{},"cache_control"," 标记",[88,275,276],{},"5 分钟（可续到 1 小时）",[88,278,279],{},"Read -90%",[70,281,282,285,288,291],{},[88,283,284],{},"OpenAI",[88,286,287],{},"自动缓存 ≥1024 token 的前缀",[88,289,290],{},"~10 分钟",[88,292,293],{},"Read -50%",[70,295,296,299,305,308],{},[88,297,298],{},"Gemini",[88,300,269,301,304],{},[35,302,303],{},"cachedContent"," API",[88,306,307],{},"默认 1 小时（可设）",[88,309,310],{},"Read -75% + 按存储时长收费",[70,312,313,316,319,322],{},[88,314,315],{},"DeepSeek",[88,317,318],{},"自动缓存",[88,320,321],{},"~ 数小时",[88,323,279],{},[21,325,326,329,330,333],{},[166,327,328],{},"生效条件","：必须前缀完全一致。哪怕在 system prompt 开头加一个时间戳，整个缓存就废了。所以",[166,331,332],{},"动态内容要严格放在 prompt 末尾","。",[16,335,337],{"id":336},"token-计数工具","Token 计数工具",[43,339,340],{"id":340},"在线工具",[342,343,344,354],"ul",{},[217,345,346,353],{},[347,348,352],"a",{"href":349,"rel":350},"https:\u002F\u002Fplatform.openai.com\u002Ftokenizer",[351],"nofollow","OpenAI Tokenizer"," — GPT 系列分词器",[217,355,356,361],{},[347,357,360],{"href":358,"rel":359},"https:\u002F\u002Ftiktokenizer.vercel.app\u002F",[351],"Tiktokenizer"," — 支持 GPT \u002F Claude \u002F Llama \u002F DeepSeek 等多家分词器对比",[43,363,364],{"id":364},"代码计数",[28,366,370],{"className":367,"code":368,"language":369,"meta":37,"style":37},"language-python shiki shiki-themes github-light github-dark","# OpenAI 的 tiktoken 库\nimport tiktoken\nenc = tiktoken.encoding_for_model(\"gpt-4o\")\ntokens = enc.encode(\"你好世界，Hello world\")\nprint(len(tokens))  # 输出 token 数\n\n# Anthropic：通过 SDK 直接调\nfrom anthropic import Anthropic\nclient = Anthropic()\ncount = client.messages.count_tokens(\n    model=\"claude-sonnet-4\",\n    messages=[{\"role\": \"user\", \"content\": \"你好世界\"}]\n)\nprint(count.input_tokens)\n","python",[35,371,372,381,392,411,427,446,453,458,471,482,492,507,541,546],{"__ignoreMap":37},[373,374,377],"span",{"class":375,"line":376},"line",1,[373,378,380],{"class":379},"sJ8bj","# OpenAI 的 tiktoken 库\n",[373,382,384,388],{"class":375,"line":383},2,[373,385,387],{"class":386},"szBVR","import",[373,389,391],{"class":390},"sVt8B"," tiktoken\n",[373,393,395,398,401,404,408],{"class":375,"line":394},3,[373,396,397],{"class":390},"enc ",[373,399,400],{"class":386},"=",[373,402,403],{"class":390}," tiktoken.encoding_for_model(",[373,405,407],{"class":406},"sZZnC","\"gpt-4o\"",[373,409,410],{"class":390},")\n",[373,412,414,417,419,422,425],{"class":375,"line":413},4,[373,415,416],{"class":390},"tokens ",[373,418,400],{"class":386},[373,420,421],{"class":390}," enc.encode(",[373,423,424],{"class":406},"\"你好世界，Hello world\"",[373,426,410],{"class":390},[373,428,430,434,437,440,443],{"class":375,"line":429},5,[373,431,433],{"class":432},"sj4cs","print",[373,435,436],{"class":390},"(",[373,438,439],{"class":432},"len",[373,441,442],{"class":390},"(tokens))  ",[373,444,445],{"class":379},"# 输出 token 数\n",[373,447,449],{"class":375,"line":448},6,[373,450,452],{"emptyLinePlaceholder":451},true,"\n",[373,454,455],{"class":375,"line":5},[373,456,457],{"class":379},"# Anthropic：通过 SDK 直接调\n",[373,459,460,463,466,468],{"class":375,"line":8},[373,461,462],{"class":386},"from",[373,464,465],{"class":390}," anthropic ",[373,467,387],{"class":386},[373,469,470],{"class":390}," Anthropic\n",[373,472,474,477,479],{"class":375,"line":473},9,[373,475,476],{"class":390},"client ",[373,478,400],{"class":386},[373,480,481],{"class":390}," Anthropic()\n",[373,483,484,487,489],{"class":375,"line":7},[373,485,486],{"class":390},"count ",[373,488,400],{"class":386},[373,490,491],{"class":390}," client.messages.count_tokens(\n",[373,493,495,499,501,504],{"class":375,"line":494},11,[373,496,498],{"class":497},"s4XuR","    model",[373,500,400],{"class":386},[373,502,503],{"class":406},"\"claude-sonnet-4\"",[373,505,506],{"class":390},",\n",[373,508,510,513,515,518,521,524,527,530,533,535,538],{"class":375,"line":509},12,[373,511,512],{"class":497},"    messages",[373,514,400],{"class":386},[373,516,517],{"class":390},"[{",[373,519,520],{"class":406},"\"role\"",[373,522,523],{"class":390},": ",[373,525,526],{"class":406},"\"user\"",[373,528,529],{"class":390},", ",[373,531,532],{"class":406},"\"content\"",[373,534,523],{"class":390},[373,536,537],{"class":406},"\"你好世界\"",[373,539,540],{"class":390},"}]\n",[373,542,544],{"class":375,"line":543},13,[373,545,410],{"class":390},[373,547,549,551],{"class":375,"line":548},14,[373,550,433],{"class":432},[373,552,553],{"class":390},"(count.input_tokens)\n",[28,555,559],{"className":556,"code":557,"language":558,"meta":37,"style":37},"language-javascript shiki shiki-themes github-light github-dark","\u002F\u002F 浏览器端\nimport { encode } from 'gpt-tokenizer'\nconst tokens = encode('你好世界，Hello world')\nconsole.log(tokens.length)\n","javascript",[35,560,561,566,571,576],{"__ignoreMap":37},[373,562,563],{"class":375,"line":376},[373,564,565],{},"\u002F\u002F 浏览器端\n",[373,567,568],{"class":375,"line":383},[373,569,570],{},"import { encode } from 'gpt-tokenizer'\n",[373,572,573],{"class":375,"line":394},[373,574,575],{},"const tokens = encode('你好世界，Hello world')\n",[373,577,578],{"class":375,"line":413},[373,579,580],{},"console.log(tokens.length)\n",[16,582,583],{"id":583},"不同模型的分词差异",[21,585,586],{},"不同模型使用不同的分词器，同样的文本 token 数可能差异巨大：",[64,588,589,605],{},[67,590,591],{},[70,592,593,596,598,601,603],{},[73,594,595],{},"文本",[73,597,90],{},[73,599,600],{},"Claude",[73,602,298],{},[73,604,315],{},[83,606,607,621,634,650],{},[70,608,609,612,615,617,619],{},[88,610,611],{},"\"Hello world\"",[88,613,614],{},"2",[88,616,614],{},[88,618,614],{},[88,620,614],{},[70,622,623,625,628,630,632],{},[88,624,537],{},[88,626,627],{},"4",[88,629,614],{},[88,631,627],{},[88,633,614],{},[70,635,636,639,642,645,648],{},[88,637,638],{},"1000 行 Python 代码",[88,640,641],{},"~8000",[88,643,644],{},"~7500",[88,646,647],{},"~9000",[88,649,644],{},[70,651,652,655,658,661,663],{},[88,653,654],{},"1000 字中文文档",[88,656,657],{},"~1700",[88,659,660],{},"~1200",[88,662,657],{},[88,664,665],{},"~1100",[21,667,668,671],{},[166,669,670],{},"结论","：中文场景下 Claude \u002F DeepSeek \u002F Qwen 这类有中文优化的 tokenizer 比 GPT 省 30-50% token。同样的对话，GPT 跑下来可能比 Claude 贵不少——尤其是中文 input 多的场景。",[16,673,674],{"id":674},"实用估算",[21,676,677],{},"快速估算 token 数量的经验法则：",[64,679,680,690],{},[67,681,682],{},[70,683,684,687],{},[73,685,686],{},"文本类型",[73,688,689],{},"1K token ≈",[83,691,692,700,708,716,724],{},[70,693,694,697],{},[88,695,696],{},"英文文本",[88,698,699],{},"750 单词",[70,701,702,705],{},[88,703,704],{},"中文文本",[88,706,707],{},"500-700 汉字",[70,709,710,713],{},[88,711,712],{},"代码",[88,714,715],{},"30-50 行",[70,717,718,721],{},[88,719,720],{},"JSON",[88,722,723],{},"100-200 行",[70,725,726,729],{},[88,727,728],{},"Markdown",[88,730,731],{},"500-700 字",[16,733,734],{"id":734},"上下文压缩策略",[21,736,737],{},"长对话 \u002F Agent 长任务里 token 涨得最快。常见压缩手段：",[64,739,740,753],{},[67,741,742],{},[70,743,744,747,750],{},[73,745,746],{},"策略",[73,748,749],{},"适用",[73,751,752],{},"副作用",[83,754,755,768,781,794,807],{},[70,756,757,762,765],{},[88,758,759],{},[166,760,761],{},"滑动窗口",[88,763,764],{},"多轮对话保留最近 N 轮",[88,766,767],{},"早期上下文丢失",[70,769,770,775,778],{},[88,771,772],{},[166,773,774],{},"历史总结",[88,776,777],{},"把旧对话用 LLM 总结成几百 token",[88,779,780],{},"多一次 LLM 调用、细节丢失",[70,782,783,788,791],{},[88,784,785],{},[166,786,787],{},"工具输出截断",[88,789,790],{},"Agent 调工具返回大量数据时只保留摘要 + 引用 ID",[88,792,793],{},"模型可能再次请求完整数据",[70,795,796,801,804],{},[88,797,798],{},[166,799,800],{},"检索式压缩",[88,802,803],{},"旧消息存向量库，按需检索",[88,805,806],{},"工程复杂度高",[70,808,809,814,817],{},[88,810,811],{},[166,812,813],{},"Map-Reduce",[88,815,816],{},"长文档先并行小段总结、再合并",[88,818,819],{},"跨段语义可能被切断",[21,821,822],{},"Claude Code \u002F Cursor 长会话时都内置了自动总结——当上下文超过阈值，会把前面对话压缩成几百 token 的摘要保留。",[16,824,826],{"id":825},"省-token-技巧","省 Token 技巧",[214,828,829,835,841,847,853,862,868,874],{},[217,830,831,834],{},[166,832,833],{},"精简 prompt"," — 删除冗余措辞，用简洁的指令",[217,836,837,840],{},[166,838,839],{},"压缩历史"," — 长对话用摘要替代完整历史",[217,842,843,846],{},[166,844,845],{},"缓存重复内容"," — 使用 prompt caching（Anthropic \u002F OpenAI \u002F Gemini \u002F DeepSeek 都支持）",[217,848,849,852],{},[166,850,851],{},"选对模型"," — 简单任务用 Haiku \u002F Flash \u002F Mini 而非 Sonnet \u002F Pro \u002F GPT-5",[217,854,855,858,859],{},[166,856,857],{},"控制输出长度"," — 明确要求\"用 100 字以内回答\"或设 ",[35,860,861],{},"max_tokens",[217,863,864,867],{},[166,865,866],{},"少用 Few-shot"," — 推理模型不需要太多示例",[217,869,870,873],{},[166,871,872],{},"批量化"," — 同一类请求合并到 Batch API，价格直接 -50%",[217,875,876,879],{},[166,877,878],{},"结构化输出"," — 让模型只输出 JSON 而不是\"自然语言 + JSON\"，Output 少一半",[16,881,882],{"id":882},"常见误区",[43,884,886],{"id":885},"误区-1-我用-gpt-比-claude-便宜因为单价低","误区 1: \"我用 GPT 比 Claude 便宜，因为单价低\"",[21,888,889,890,893],{},"不一定。GPT tokenizer 在中文场景下 token 数显著更多，单价低但总账不一定低。",[166,891,892],{},"算账要按\"完成同样任务的总成本\"","，不是单 token 价。",[43,895,897],{"id":896},"误区-2-200k-上下文我就尽量塞满","误区 2: \"200K 上下文我就尽量塞满\"",[21,899,900,901,905],{},"参考 ",[347,902,904],{"href":903},"\u002Fwiki\u002Fcontext-engineering.html","Context Engineering","——超过 80% 上下文窗口后模型质量明显下降。塞满还多花了钱，得不偿失。",[43,907,909],{"id":908},"误区-3-output-反正不多没必要省","误区 3: \"Output 反正不多没必要省\"",[21,911,912],{},"Output 单价是 Input 的 3-5 倍。让模型回答\"以下是详细分析：...\"这类填充话，每次都在烧钱。",[16,914,915],{"id":915},"延伸阅读",[342,917,918,923,930],{},[217,919,920,921],{},"上下文窗口管理：",[347,922,904],{"href":903},[217,924,925,926],{},"采样参数（影响 Output 长度）：",[347,927,929],{"href":928},"\u002Fwiki\u002Ftemperature-top-p.html","Temperature 与 Top-P",[217,931,932,933],{},"输出结构化（省 token）：",[347,934,936],{"href":935},"\u002Fwiki\u002Ffunction-calling.html","Function Calling",[938,939,940],"style",{},"html pre.shiki code .sJ8bj, html code.shiki .sJ8bj{--shiki-default:#6A737D;--shiki-dark:#6A737D}html pre.shiki code .szBVR, html code.shiki .szBVR{--shiki-default:#D73A49;--shiki-dark:#F97583}html pre.shiki code .sVt8B, html code.shiki .sVt8B{--shiki-default:#24292E;--shiki-dark:#E1E4E8}html pre.shiki code .sZZnC, html code.shiki .sZZnC{--shiki-default:#032F62;--shiki-dark:#9ECBFF}html pre.shiki code .sj4cs, html code.shiki .sj4cs{--shiki-default:#005CC5;--shiki-dark:#79B8FF}html pre.shiki code .s4XuR, html code.shiki .s4XuR{--shiki-default:#E36209;--shiki-dark:#FFAB70}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}",{"title":37,"searchDepth":394,"depth":394,"links":942},[943,944,949,950,951,955,956,957,958,959,964],{"id":18,"depth":383,"text":19},{"id":40,"depth":383,"text":41,"children":945},[946,947,948],{"id":45,"depth":394,"text":46},{"id":58,"depth":394,"text":59},{"id":132,"depth":394,"text":133},{"id":139,"depth":383,"text":140},{"id":237,"depth":383,"text":238},{"id":336,"depth":383,"text":337,"children":952},[953,954],{"id":340,"depth":394,"text":340},{"id":364,"depth":394,"text":364},{"id":583,"depth":383,"text":583},{"id":674,"depth":383,"text":674},{"id":734,"depth":383,"text":734},{"id":825,"depth":383,"text":826},{"id":882,"depth":383,"text":882,"children":960},[961,962,963],{"id":885,"depth":394,"text":886},{"id":896,"depth":394,"text":897},{"id":908,"depth":394,"text":909},{"id":915,"depth":383,"text":915},"concept","md",{},"\u002Fwiki\u002Ftoken","2026-06-21",[971,972],"claude-sonnet-4","gpt-5",null,{"title":11,"description":37},"token","wiki\u002Ftoken","大模型处理文本的最小单位。一个 Token 约等于 0.75 个英文单词或 1-2 个汉字。Token 数量决定 API 费用和上下文窗口占用。",[979,980,981],"Token","基础概念","定价","gbHEyHh8pBEr1Saq2lobbZq4ogKbMzLTlLlaRX-Vvec",1782316491228]