From de73a9174565780a603b389debad6f2519ec737e Mon Sep 17 00:00:00 2001 From: benank Date: Mon, 16 Jun 2025 10:32:16 -0700 Subject: [PATCH 1/4] Add Groq to provider docs --- docs/inference-providers/_toctree.yml | 2 + docs/inference-providers/index.md | 1 + docs/inference-providers/providers/groq.md | 39 +++++++++++++++++++ .../inference-providers/scripts/generate.ts | 2 + .../templates/providers/groq.handlebars | 19 +++++++++ 5 files changed, 63 insertions(+) create mode 100644 docs/inference-providers/providers/groq.md create mode 100644 scripts/inference-providers/templates/providers/groq.handlebars diff --git a/docs/inference-providers/_toctree.yml b/docs/inference-providers/_toctree.yml index 26d7fa0040..9faf4ffcdc 100644 --- a/docs/inference-providers/_toctree.yml +++ b/docs/inference-providers/_toctree.yml @@ -21,6 +21,8 @@ title: Fal AI - local: providers/fireworks-ai title: Fireworks + - local: providers/groq + title: Groq - local: providers/hyperbolic title: Hyperbolic - local: providers/hf-inference diff --git a/docs/inference-providers/index.md b/docs/inference-providers/index.md index 4cec406b4c..02975df725 100644 --- a/docs/inference-providers/index.md +++ b/docs/inference-providers/index.md @@ -19,6 +19,7 @@ Here is the complete list of partners integrated with Inference Providers, and t | [Cohere](./providers/cohere) | ✅ | ✅ | | | | | [Fal AI](./providers/fal-ai) | | | | ✅ | ✅ | | [Fireworks](./providers/fireworks-ai) | ✅ | ✅ | | | | +| [Groq](./providers/groq) | ✅ | | | | | | [HF Inference](./providers/hf-inference) | ✅ | ✅ | ✅ | ✅ | | | [Hyperbolic](./providers/hyperbolic) | ✅ | ✅ | | | | | [Nebius](./providers/nebius) | ✅ | ✅ | ✅ | ✅ | | diff --git a/docs/inference-providers/providers/groq.md b/docs/inference-providers/providers/groq.md new file mode 100644 index 0000000000..df30c4c9ae --- /dev/null +++ b/docs/inference-providers/providers/groq.md @@ -0,0 +1,39 @@ + + +# Groq + +
+ + + + +
+ +
+ + + + +
+ +Groq is fast AI inference. Their groundbreaking Language Processing Unit (LPU) technology delivers industry-leading speed and performance for large language models, achieving much faster performance than traditional GPU-based solutions. With custom chips specifically designed for AI inference workloads and a deterministic, software-first approach, Groq eliminates the bottlenecks of conventional hardware to enable real-time AI applications with predictable latency and exceptional throughput for developers building the next generation of AI-powered solutions. + +## Supported tasks + diff --git a/scripts/inference-providers/scripts/generate.ts b/scripts/inference-providers/scripts/generate.ts index 90ed6f49a3..86a46220c3 100644 --- a/scripts/inference-providers/scripts/generate.ts +++ b/scripts/inference-providers/scripts/generate.ts @@ -37,6 +37,7 @@ const PROVIDERS_HUB_ORGS: Record = { cohere: "CohereLabs", "fal-ai": "fal", "fireworks-ai": "fireworks-ai", + groq: "groq", "hf-inference": "hf-inference", hyperbolic: "Hyperbolic", nebius: "nebius", @@ -52,6 +53,7 @@ const PROVIDERS_URLS: Record = { cohere: "https://cohere.com/", "fal-ai": "https://fal.ai/", "fireworks-ai": "https://fireworks.ai/", + groq: "https://groq.com/", "hf-inference": "https://huggingface.co/", hyperbolic: "https://hyperbolic.xyz/", nebius: "https://nebius.com/", diff --git a/scripts/inference-providers/templates/providers/groq.handlebars b/scripts/inference-providers/templates/providers/groq.handlebars new file mode 100644 index 0000000000..d25d4c4c7d --- /dev/null +++ b/scripts/inference-providers/templates/providers/groq.handlebars @@ -0,0 +1,19 @@ +# Groq + +{{{logoSection}}} + +{{{followUsSection}}} + +Groq is fast AI inference. Their groundbreaking LPU technology delivers record-setting performance and efficiency for GenAI models. With custom chips specifically designed for AI inference workloads and a deterministic, software-first approach, Groq eliminates the bottlenecks of conventional hardware to enable real-time AI applications with predictable latency and exceptional throughput so developers can build fast. + +For latest pricing, visit our [pricing page](https://groq.com/pricing/). + +## Resources + - **Website**: https://groq.com/ + - **Documentation**: https://console.groq.com/docs + - **Community Forum**: https://community.groq.com/ + - **X**: [@GroqInc](https://x.com/GroqInc) + - **LinkedIn**: [Groq](https://www.linkedin.com/company/groq/) + - **YouTube**: [Groq](https://www.youtube.com/@GroqInc) + +{{{tasksSection}}} \ No newline at end of file From 1510ab7c30f21cd3b406e77269b5729633097b99 Mon Sep 17 00:00:00 2001 From: SBrandeis Date: Mon, 16 Jun 2025 21:17:00 +0200 Subject: [PATCH 2/4] update supported tasks --- docs/inference-providers/providers/groq.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/inference-providers/providers/groq.md b/docs/inference-providers/providers/groq.md index df30c4c9ae..a4c0a19a4b 100644 --- a/docs/inference-providers/providers/groq.md +++ b/docs/inference-providers/providers/groq.md @@ -37,3 +37,23 @@ Groq is fast AI inference. Their groundbreaking Language Processing Unit (LPU) t ## Supported tasks + +### Chat Completion (LLM) + +Find out more about Chat Completion (LLM) [here](../tasks/chat-completion). + + + + +### Chat Completion (VLM) + +Find out more about Chat Completion (VLM) [here](../tasks/chat-completion). + + + From a5c80ff717539f079ce69efeb4d17e443ee61b5c Mon Sep 17 00:00:00 2001 From: SBrandeis Date: Mon, 16 Jun 2025 21:22:04 +0200 Subject: [PATCH 3/4] update md file --- docs/inference-providers/providers/cohere.md | 2 +- docs/inference-providers/providers/groq.md | 12 ++- .../providers/hf-inference.md | 8 +- docs/inference-providers/providers/nscale.md | 2 +- .../providers/replicate.md | 2 +- .../inference-providers/providers/together.md | 4 +- .../tasks/chat-completion.md | 4 +- .../tasks/image-classification.md | 2 +- .../tasks/summarization.md | 3 +- .../tasks/text-classification.md | 2 +- .../tasks/text-generation.md | 2 +- .../tasks/text-to-video.md | 2 +- .../inference-providers/scripts/generate.ts | 100 +++++++++--------- 13 files changed, 78 insertions(+), 67 deletions(-) diff --git a/docs/inference-providers/providers/cohere.md b/docs/inference-providers/providers/cohere.md index 28ee4c9745..8edb2601ad 100644 --- a/docs/inference-providers/providers/cohere.md +++ b/docs/inference-providers/providers/cohere.md @@ -56,6 +56,6 @@ Find out more about Chat Completion (VLM) [here](../tasks/chat-completion). diff --git a/docs/inference-providers/providers/groq.md b/docs/inference-providers/providers/groq.md index a4c0a19a4b..9e326230c3 100644 --- a/docs/inference-providers/providers/groq.md +++ b/docs/inference-providers/providers/groq.md @@ -33,7 +33,17 @@ For more details, check out the `generate.ts` script: https://github.com/hugging -Groq is fast AI inference. Their groundbreaking Language Processing Unit (LPU) technology delivers industry-leading speed and performance for large language models, achieving much faster performance than traditional GPU-based solutions. With custom chips specifically designed for AI inference workloads and a deterministic, software-first approach, Groq eliminates the bottlenecks of conventional hardware to enable real-time AI applications with predictable latency and exceptional throughput for developers building the next generation of AI-powered solutions. +Groq is fast AI inference. Their groundbreaking LPU technology delivers record-setting performance and efficiency for GenAI models. With custom chips specifically designed for AI inference workloads and a deterministic, software-first approach, Groq eliminates the bottlenecks of conventional hardware to enable real-time AI applications with predictable latency and exceptional throughput so developers can build fast. + +For latest pricing, visit our [pricing page](https://groq.com/pricing/). + +## Resources + - **Website**: https://groq.com/ + - **Documentation**: https://console.groq.com/docs + - **Community Forum**: https://community.groq.com/ + - **X**: [@GroqInc](https://x.com/GroqInc) + - **LinkedIn**: [Groq](https://www.linkedin.com/company/groq/) + - **YouTube**: [Groq](https://www.youtube.com/@GroqInc) ## Supported tasks diff --git a/docs/inference-providers/providers/hf-inference.md b/docs/inference-providers/providers/hf-inference.md index fff7ae27ef..4ec33aae05 100644 --- a/docs/inference-providers/providers/hf-inference.md +++ b/docs/inference-providers/providers/hf-inference.md @@ -65,7 +65,7 @@ Find out more about Chat Completion (VLM) [here](../tasks/chat-completion). @@ -95,7 +95,7 @@ Find out more about Image Classification [here](../tasks/image_classification). @@ -135,7 +135,7 @@ Find out more about Summarization [here](../tasks/summarization). @@ -155,7 +155,7 @@ Find out more about Text Classification [here](../tasks/text_classification). diff --git a/docs/inference-providers/providers/nscale.md b/docs/inference-providers/providers/nscale.md index 5da7783ecd..8aad210292 100644 --- a/docs/inference-providers/providers/nscale.md +++ b/docs/inference-providers/providers/nscale.md @@ -46,7 +46,7 @@ Find out more about Chat Completion (LLM) [here](../tasks/chat-completion). diff --git a/docs/inference-providers/providers/replicate.md b/docs/inference-providers/providers/replicate.md index 657cee3f5c..c1cd26b938 100644 --- a/docs/inference-providers/providers/replicate.md +++ b/docs/inference-providers/providers/replicate.md @@ -54,6 +54,6 @@ Find out more about Text To Video [here](../tasks/text_to_video). diff --git a/docs/inference-providers/providers/together.md b/docs/inference-providers/providers/together.md index 8ccce4cfa9..426de47444 100644 --- a/docs/inference-providers/providers/together.md +++ b/docs/inference-providers/providers/together.md @@ -44,7 +44,7 @@ Find out more about Chat Completion (LLM) [here](../tasks/chat-completion). @@ -64,7 +64,7 @@ Find out more about Text Generation [here](../tasks/text_generation). diff --git a/docs/inference-providers/tasks/chat-completion.md b/docs/inference-providers/tasks/chat-completion.md index 0a28600675..2f46da5657 100644 --- a/docs/inference-providers/tasks/chat-completion.md +++ b/docs/inference-providers/tasks/chat-completion.md @@ -61,7 +61,7 @@ The API supports: @@ -71,7 +71,7 @@ conversational /> diff --git a/docs/inference-providers/tasks/image-classification.md b/docs/inference-providers/tasks/image-classification.md index 6dd81d8384..cc68b01fd6 100644 --- a/docs/inference-providers/tasks/image-classification.md +++ b/docs/inference-providers/tasks/image-classification.md @@ -35,7 +35,7 @@ Explore all available models and find the one that suits you best [here](https:/ diff --git a/docs/inference-providers/tasks/summarization.md b/docs/inference-providers/tasks/summarization.md index 001d1878ce..6d39944062 100644 --- a/docs/inference-providers/tasks/summarization.md +++ b/docs/inference-providers/tasks/summarization.md @@ -24,6 +24,7 @@ For more details about the `summarization` task, check out its [dedicated page]( ### Recommended models +- [facebook/bart-large-cnn](https://huggingface.co/facebook/bart-large-cnn): A strong summarization model trained on English news articles. Excels at generating factual summaries. - [Falconsai/medical_summarization](https://huggingface.co/Falconsai/medical_summarization): A summarization model trained on medical articles. Explore all available models and find the one that suits you best [here](https://huggingface.co/models?inference=warm&pipeline_tag=summarization&sort=trending). @@ -33,7 +34,7 @@ Explore all available models and find the one that suits you best [here](https:/ diff --git a/docs/inference-providers/tasks/text-classification.md b/docs/inference-providers/tasks/text-classification.md index 48934e7587..fd2827abbf 100644 --- a/docs/inference-providers/tasks/text-classification.md +++ b/docs/inference-providers/tasks/text-classification.md @@ -34,7 +34,7 @@ Explore all available models and find the one that suits you best [here](https:/ diff --git a/docs/inference-providers/tasks/text-generation.md b/docs/inference-providers/tasks/text-generation.md index 1e350e2fe3..0f2a409fd9 100644 --- a/docs/inference-providers/tasks/text-generation.md +++ b/docs/inference-providers/tasks/text-generation.md @@ -41,7 +41,7 @@ Explore all available models and find the one that suits you best [here](https:/ diff --git a/docs/inference-providers/tasks/text-to-video.md b/docs/inference-providers/tasks/text-to-video.md index 733b16be44..5a29e371d0 100644 --- a/docs/inference-providers/tasks/text-to-video.md +++ b/docs/inference-providers/tasks/text-to-video.md @@ -35,7 +35,7 @@ Explore all available models and find the one that suits you best [here](https:/ diff --git a/scripts/inference-providers/scripts/generate.ts b/scripts/inference-providers/scripts/generate.ts index 86a46220c3..889618e883 100644 --- a/scripts/inference-providers/scripts/generate.ts +++ b/scripts/inference-providers/scripts/generate.ts @@ -128,7 +128,7 @@ await Promise.all( } } } - }) + }), ); //////////////////////// @@ -144,7 +144,7 @@ const TASKS_DOCS_DIR = path.join(DOCS_DIR, "inference-providers", "tasks"); const PROVIDERS_DOCS_DIR = path.join( DOCS_DIR, "inference-providers", - "providers" + "providers", ); const NBSP = " "; // non-breaking space @@ -152,12 +152,12 @@ const TABLE_INDENT = NBSP.repeat(8); function readTemplate( templateName: string, - namespace: string + namespace: string, ): Promise { const templatePath = path.join( TEMPLATE_DIR, namespace, - `${templateName}.handlebars` + `${templateName}.handlebars`, ); console.log(` 🔍 Reading ${templateName}.handlebars`); return fs.readFile(templatePath, { encoding: "utf-8" }); @@ -171,13 +171,13 @@ function writeTaskDoc(templateName: string, content: string): Promise { return fs .mkdir(TASKS_DOCS_DIR, { recursive: true }) .then(() => - fs.writeFile(taskDocPath, contentWithHeader, { encoding: "utf-8" }) + fs.writeFile(taskDocPath, contentWithHeader, { encoding: "utf-8" }), ); } function writeProviderDoc( templateName: string, - content: string + content: string, ): Promise { const providerDocPath = path.join(PROVIDERS_DOCS_DIR, `${templateName}.md`); console.log(` 💾 Saving to ${providerDocPath}`); @@ -186,7 +186,7 @@ function writeProviderDoc( return fs .mkdir(TASKS_DOCS_DIR, { recursive: true }) .then(() => - fs.writeFile(providerDocPath, contentWithHeader, { encoding: "utf-8" }) + fs.writeFile(providerDocPath, contentWithHeader, { encoding: "utf-8" }), ); } @@ -206,13 +206,13 @@ const TASKS_DATA = (await authFetchJson(TASKS_API_URL)) as any; type SpecNameType = "input" | "output" | "stream_output"; const SPECS_URL_TEMPLATE = Handlebars.compile( - `https://raw.githubusercontent.com/huggingface/huggingface.js/${SPECS_REVISION}/packages/tasks/src/tasks/{{task}}/spec/{{name}}.json` + `https://raw.githubusercontent.com/huggingface/huggingface.js/${SPECS_REVISION}/packages/tasks/src/tasks/{{task}}/spec/{{name}}.json`, ); const COMMON_DEFINITIONS_URL = `https://raw.githubusercontent.com/huggingface/huggingface.js/${SPECS_REVISION}/packages/tasks/src/tasks/common-definitions.json`; async function fetchOneSpec( task: PipelineType, - name: SpecNameType + name: SpecNameType, ): Promise { const url = SPECS_URL_TEMPLATE({ task, name }); console.log(` 🕸️ Fetching ${task} ${name} specs`); @@ -220,7 +220,7 @@ async function fetchOneSpec( } async function fetchSpecs( - task: PipelineType + task: PipelineType, ): Promise< Record<"input" | "output" | "stream_output", JsonObject | undefined> > { @@ -258,7 +258,7 @@ function processPayloadSchema(schema: any): JsonObject[] { key: string, value: any, required: boolean, - parentPrefix: string + parentPrefix: string, ): void { const isRequired = required; let type = value.type || "unknown"; @@ -322,9 +322,9 @@ function processPayloadSchema(schema: any): JsonObject[] { nestedKey, nestedValue, nestedRequired, - parentPrefix + TABLE_INDENT + parentPrefix + TABLE_INDENT, ); - } + }, ); } else if (isArray) { // Process array items @@ -342,7 +342,7 @@ function processPayloadSchema(schema: any): JsonObject[] { `${NBSP}(#${index + 1})`, subSchema, false, - parentPrefix + TABLE_INDENT + parentPrefix + TABLE_INDENT, ); }); } @@ -384,32 +384,32 @@ For more details about the \`{{task}}\` task, check out its [dedicated page](htt `); const TIP_LIST_MODELS_LINK_TEMPLATE = Handlebars.compile( - `Explore all available models and find the one that suits you best [here](https://huggingface.co/models?inference=warm&pipeline_tag={{task}}&sort=trending).` + `Explore all available models and find the one that suits you best [here](https://huggingface.co/models?inference=warm&pipeline_tag={{task}}&sort=trending).`, ); const SPECS_HEADERS = await readTemplate("specs-headers", "common"); const PAGE_HEADER = Handlebars.compile( - await readTemplate("page-header", "common") + await readTemplate("page-header", "common"), ); const PROVIDER_PAGE_HEADER = Handlebars.compile( - await readTemplate("provider-header", "common") + await readTemplate("provider-header", "common"), ); const SNIPPETS_TEMPLATE = Handlebars.compile( - await readTemplate("snippets-template", "common") + await readTemplate("snippets-template", "common"), ); const SPECS_PAYLOAD_TEMPLATE = Handlebars.compile( - await readTemplate("specs-payload", "common") + await readTemplate("specs-payload", "common"), ); const SPECS_OUTPUT_TEMPLATE = Handlebars.compile( - await readTemplate("specs-output", "common") + await readTemplate("specs-output", "common"), ); const PROVIDER_TASKS_TEMPLATE = Handlebars.compile( - await readTemplate("provider-tasks", "common") + await readTemplate("provider-tasks", "common"), ); const PROVIDER_LOGO_TEMPLATE = Handlebars.compile( - await readTemplate("provider-logo", "common") + await readTemplate("provider-logo", "common"), ); const FOLLOW_US_BUTTON_TEMPLATE = Handlebars.compile( - await readTemplate("follow-us-button", "common") + await readTemplate("follow-us-button", "common"), ); //////////////////// @@ -479,15 +479,15 @@ await Promise.all( const modelData = await authFetchJson(url); model.inference = modelData.inference; model.tags = modelData.tags; - } - ) + }, + ), ); - }) + }), ); async function fetchWarmModels( task: PipelineType, - conversational: boolean = false + conversational: boolean = false, ): Promise< { modelId: string; @@ -509,7 +509,7 @@ async function fetchWarmModels( await Promise.all( providers.map(async (provider) => { console.log( - ` ⚡ Fetching most popular warm model for ${task} from ${provider}` + ` ⚡ Fetching most popular warm model for ${task} from ${provider}`, ); const url = `https://huggingface.co/api/models?pipeline_tag=${task}&inference_provider=${provider}&sort=likes30d&expand[]=inferenceProviderMapping&expand[]=tags&limit=5`; const modelsData = (await authFetchJson(url)) as { @@ -520,7 +520,7 @@ async function fetchWarmModels( }[]; if (modelsData.length === 0) { console.warn( - ` ⚠️ No warm model found for ${task} from ${provider}` + ` ⚠️ No warm model found for ${task} from ${provider}`, ); return; } @@ -538,7 +538,7 @@ async function fetchWarmModels( return; } const providerData = providerMapping.filter( - (mapping) => mapping.provider === provider + (mapping) => mapping.provider === provider, )[0]; return { modelId: topModelData.id, @@ -547,7 +547,7 @@ async function fetchWarmModels( providerTask: providerData.task, tags: topModelData.tags, }; - }) + }), ) ).filter((model) => model !== undefined); } @@ -556,13 +556,13 @@ async function fetchWarmModels( await Promise.all( TASKS.map(async (task) => { DATA.perProviderWarmModels[task] = await fetchWarmModels(task); - }) + }), ); // Filter recommended models (i.e. recommended + warm) TASKS.forEach((task) => { DATA.recommendedModels[task] = TASKS_DATA[task].models.filter( - (model: { inference: string }) => model.inference === "warm" + (model: { inference: string }) => model.inference === "warm", ); }); @@ -573,7 +573,7 @@ function buildProviderMapping( providerModelId: string; providerTask: string; tags: string[]; - }[] + }[], ): Record { return models.reduce( (acc, item) => { @@ -583,14 +583,14 @@ function buildProviderMapping( }; return acc; }, - {} as Record + {} as Record, ); } // Generate snippets TASKS.forEach((task) => { const providersMapping = buildProviderMapping( - DATA.perProviderWarmModels[task] + DATA.perProviderWarmModels[task], ); DATA.snippets[task] = SNIPPETS_TEMPLATE({ @@ -621,7 +621,7 @@ await Promise.all( }) : undefined, }; - }) + }), ); // Render tips @@ -637,7 +637,7 @@ TASKS.forEach((task) => { async function fetchChatCompletion() { // Conversational text-generation console.log( - " ⚡ Prepare data for chat-completion (conversational text-generation)" + " ⚡ Prepare data for chat-completion (conversational text-generation)", ); DATA.recommendedModels["chat-completion"] = DATA.recommendedModels[ "text-generation" @@ -645,10 +645,10 @@ async function fetchChatCompletion() { DATA.perProviderWarmModels["chat-completion"] = await fetchWarmModels( "text-generation", - true + true, ); const providersMappingChatCompletion = buildProviderMapping( - DATA.perProviderWarmModels["chat-completion"] + DATA.perProviderWarmModels["chat-completion"], ); DATA.snippets["chat-completion"] = SNIPPETS_TEMPLATE({ task: "text-generation", @@ -661,18 +661,18 @@ async function fetchChatCompletion() { // Conversational image-text-to-text console.log( - " ⚡ Prepare data for chat-completion (conversational image-text-to-text)" + " ⚡ Prepare data for chat-completion (conversational image-text-to-text)", ); DATA.recommendedModels["conversational-image-text-to-text"] = DATA.recommendedModels["image-text-to-text"].filter((model) => - model.tags?.includes("conversational") + model.tags?.includes("conversational"), ); DATA.perProviderWarmModels["image-text-to-text"] = await fetchWarmModels( "image-text-to-text", - true + true, ); const providersMappingImageTextToText = buildProviderMapping( - DATA.perProviderWarmModels["image-text-to-text"] + DATA.perProviderWarmModels["image-text-to-text"], ); DATA.snippets["conversational-image-text-to-text"] = SNIPPETS_TEMPLATE({ @@ -711,7 +711,7 @@ Object.entries(DATA.perProviderWarmModels).forEach(([task, models]) => { PER_PROVIDER_TASKS[model.provider] = []; } let conversational = ["chat-completion", "image-text-to-text"].includes( - task + task, ); let title = conversational ? task == "image-text-to-text" @@ -744,7 +744,7 @@ Object.entries(DATA.perProviderWarmModels).forEach(([task, models]) => { // sort tasks by title Object.entries(PER_PROVIDER_TASKS).forEach(([provider, tasks]) => { PER_PROVIDER_TASKS[provider] = tasks.sort((a, b) => - a.title.localeCompare(b.title) + a.title.localeCompare(b.title), ); }); @@ -755,11 +755,11 @@ Object.entries(PER_PROVIDER_TASKS).forEach(([provider, tasks]) => { async function renderTemplate( templateName: string, namespace: string, - data: JsonObject + data: JsonObject, ): Promise { console.log(`🎨 Rendering ${templateName} (${namespace})`); const template = Handlebars.compile( - await readTemplate(templateName, namespace) + await readTemplate(templateName, namespace), ); return template(data); } @@ -772,7 +772,7 @@ await Promise.all( // @ts-ignore const rendered = await renderTemplate(task, "task", DATA); await writeTaskDoc(task, rendered); - }) + }), ); await Promise.all( @@ -790,7 +790,7 @@ await Promise.all( }), }); await writeProviderDoc(provider, rendered); - }) + }), ); console.log("✅ All done!"); From 4da800f507a849b5c88917acd2ef52773ad2045b Mon Sep 17 00:00:00 2001 From: SBrandeis Date: Mon, 16 Jun 2025 21:24:55 +0200 Subject: [PATCH 4/4] last update --- docs/inference-providers/providers/groq.md | 2 +- .../tasks/chat-completion.md | 294 +++++++++--------- 2 files changed, 154 insertions(+), 142 deletions(-) diff --git a/docs/inference-providers/providers/groq.md b/docs/inference-providers/providers/groq.md index 9e326230c3..c8c6605578 100644 --- a/docs/inference-providers/providers/groq.md +++ b/docs/inference-providers/providers/groq.md @@ -54,7 +54,7 @@ Find out more about Chat Completion (LLM) [here](../tasks/chat-completion). diff --git a/docs/inference-providers/tasks/chat-completion.md b/docs/inference-providers/tasks/chat-completion.md index ec9e59a069..4f20bfefcf 100644 --- a/docs/inference-providers/tasks/chat-completion.md +++ b/docs/inference-providers/tasks/chat-completion.md @@ -25,6 +25,7 @@ This is a subtask of [`text-generation`](https://huggingface.co/docs/inference-p - [deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B](https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B): Smaller variant of one of the most powerful models. - [meta-llama/Meta-Llama-3.1-8B-Instruct](https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct): Very powerful text generation model trained to follow instructions. - [microsoft/phi-4](https://huggingface.co/microsoft/phi-4): Powerful text generation model by Microsoft. +- [Qwen/Qwen2.5-7B-Instruct-1M](https://huggingface.co/Qwen/Qwen2.5-7B-Instruct-1M): Strong conversational model that supports very long instructions. - [Qwen/Qwen2.5-Coder-32B-Instruct](https://huggingface.co/Qwen/Qwen2.5-Coder-32B-Instruct): Text generation model used to write code. - [deepseek-ai/DeepSeek-R1](https://huggingface.co/deepseek-ai/DeepSeek-R1): Powerful reasoning based open large language model. @@ -51,171 +52,182 @@ Access the Inference UI Playground and start exploring: [https://huggingface.co/ The API supports: -- Using the chat completion API compatible with the OpenAI SDK. -- Using grammars, constraints, and tools. -- Streaming the output +* Using the chat completion API compatible with the OpenAI SDK. +* Using grammars, constraints, and tools. +* Streaming the output #### Code snippet example for conversational LLMs + + + #### Code snippet example for conversational VLMs + + + ### API specification #### Request -| Headers | | | -| :---------------- | :------- | :---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Headers | | | +| :--- | :--- | :--- | | **authorization** | _string_ | Authentication header in the form `'Bearer: hf_****'` when `hf_****` is a personal user access token with "Inference Providers" permission. You can generate one from [your settings page](https://huggingface.co/settings/tokens/new?ownUserPermissions=inference.serverless.write&tokenType=fineGrained). | -| Payload | | | -| :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :--------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **frequency_penalty** | _number_ | Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far, decreasing the model's likelihood to repeat the same line verbatim. | -| **logprobs** | _boolean_ | Whether to return log probabilities of the output tokens or not. If true, returns the log probabilities of each output token returned in the content of message. | -| **max_tokens** | _integer_ | The maximum number of tokens that can be generated in the chat completion. | -| **messages\*** | _object[]_ | A list of messages comprising the conversation so far. | -| **         (#1)** | _unknown_ | One of the following: | -| **                 (#1)** | _object_ | | -| **                        content\*** | _unknown_ | One of the following: | -| **                                 (#1)** | _string_ | | -| **                                 (#2)** | _object[]_ | | -| **                                         (#1)** | _object_ | | -| **                                                text\*** | _string_ | | -| **                                                type\*** | _enum_ | Possible values: text. | -| **                                         (#2)** | _object_ | | -| **                                                image_url\*** | _object_ | | -| **                                                        url\*** | _string_ | | -| **                                                type\*** | _enum_ | Possible values: image_url. | -| **                 (#2)** | _object_ | | -| **                        tool_calls\*** | _object[]_ | | -| **                                function\*** | _object_ | | -| **                                        parameters\*** | _unknown_ | | -| **                                        description** | _string_ | | -| **                                        name\*** | _string_ | | -| **                                id\*** | _string_ | | -| **                                type\*** | _string_ | | -| **         (#2)** | _object_ | | -| **                name** | _string_ | | -| **                role\*** | _string_ | | -| **presence_penalty** | _number_ | Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far, increasing the model's likelihood to talk about new topics | -| **response_format** | _unknown_ | One of the following: | -| **         (#1)** | _object_ | | -| **                type\*** | _enum_ | Possible values: json. | -| **                value\*** | _unknown_ | A string that represents a [JSON Schema](https://json-schema.org/). JSON Schema is a declarative language that allows to annotate JSON documents with types and descriptions. | -| **         (#2)** | _object_ | | -| **                type\*** | _enum_ | Possible values: regex. | -| **                value\*** | _string_ | | -| **         (#3)** | _object_ | | -| **                type\*** | _enum_ | Possible values: json_schema. | -| **                value\*** | _object_ | | -| **                        name** | _string_ | Optional name identifier for the schema | -| **                        schema\*** | _unknown_ | The actual JSON schema definition | -| **seed** | _integer_ | | -| **stop** | _string[]_ | Up to 4 sequences where the API will stop generating further tokens. | -| **stream** | _boolean_ | | -| **stream_options** | _object_ | | -| **        include_usage** | _boolean_ | If set, an additional chunk will be streamed before the data: [DONE] message. The usage field on this chunk shows the token usage statistics for the entire request, and the choices field will always be an empty array. All other chunks will also include a usage field, but with a null value. | -| **temperature** | _number_ | What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. We generally recommend altering this or `top_p` but not both. | -| **tool_choice** | _unknown_ | One of the following: | -| **         (#1)** | _enum_ | Possible values: auto. | -| **         (#2)** | _enum_ | Possible values: none. | -| **         (#3)** | _enum_ | Possible values: required. | -| **         (#4)** | _object_ | | -| **                function\*** | _object_ | | -| **                        name\*** | _string_ | | -| **tool_prompt** | _string_ | A prompt to be appended before the tools | -| **tools** | _object[]_ | A list of tools the model may call. Currently, only functions are supported as a tool. Use this to provide a list of functions the model may generate JSON inputs for. | -| **        function\*** | _object_ | | -| **                parameters\*** | _unknown_ | | -| **                description** | _string_ | | -| **                name\*** | _string_ | | -| **        type\*** | _string_ | | -| **top_logprobs** | _integer_ | An integer between 0 and 5 specifying the number of most likely tokens to return at each token position, each with an associated log probability. logprobs must be set to true if this parameter is used. | -| **top_p** | _number_ | An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered. | + +| Payload | | | +| :--- | :--- | :--- | +| **frequency_penalty** | _number_ | Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far, decreasing the model's likelihood to repeat the same line verbatim. | +| **logprobs** | _boolean_ | Whether to return log probabilities of the output tokens or not. If true, returns the log probabilities of each output token returned in the content of message. | +| **max_tokens** | _integer_ | The maximum number of tokens that can be generated in the chat completion. | +| **messages*** | _object[]_ | A list of messages comprising the conversation so far. | +| **         (#1)** | _unknown_ | One of the following: | +| **                 (#1)** | _object_ | | +| **                        content*** | _unknown_ | One of the following: | +| **                                 (#1)** | _string_ | | +| **                                 (#2)** | _object[]_ | | +| **                                         (#1)** | _object_ | | +| **                                                text*** | _string_ | | +| **                                                type*** | _enum_ | Possible values: text. | +| **                                         (#2)** | _object_ | | +| **                                                image_url*** | _object_ | | +| **                                                        url*** | _string_ | | +| **                                                type*** | _enum_ | Possible values: image_url. | +| **                 (#2)** | _object_ | | +| **                        tool_calls*** | _object[]_ | | +| **                                function*** | _object_ | | +| **                                        parameters*** | _unknown_ | | +| **                                        description** | _string_ | | +| **                                        name*** | _string_ | | +| **                                id*** | _string_ | | +| **                                type*** | _string_ | | +| **         (#2)** | _object_ | | +| **                name** | _string_ | | +| **                role*** | _string_ | | +| **presence_penalty** | _number_ | Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far, increasing the model's likelihood to talk about new topics | +| **response_format** | _unknown_ | One of the following: | +| **         (#1)** | _object_ | | +| **                type*** | _enum_ | Possible values: json. | +| **                value*** | _unknown_ | A string that represents a [JSON Schema](https://json-schema.org/). JSON Schema is a declarative language that allows to annotate JSON documents with types and descriptions. | +| **         (#2)** | _object_ | | +| **                type*** | _enum_ | Possible values: regex. | +| **                value*** | _string_ | | +| **         (#3)** | _object_ | | +| **                type*** | _enum_ | Possible values: json_schema. | +| **                value*** | _object_ | | +| **                        name** | _string_ | Optional name identifier for the schema | +| **                        schema*** | _unknown_ | The actual JSON schema definition | +| **seed** | _integer_ | | +| **stop** | _string[]_ | Up to 4 sequences where the API will stop generating further tokens. | +| **stream** | _boolean_ | | +| **stream_options** | _object_ | | +| **        include_usage** | _boolean_ | If set, an additional chunk will be streamed before the data: [DONE] message. The usage field on this chunk shows the token usage statistics for the entire request, and the choices field will always be an empty array. All other chunks will also include a usage field, but with a null value. | +| **temperature** | _number_ | What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. We generally recommend altering this or `top_p` but not both. | +| **tool_choice** | _unknown_ | One of the following: | +| **         (#1)** | _enum_ | Possible values: auto. | +| **         (#2)** | _enum_ | Possible values: none. | +| **         (#3)** | _enum_ | Possible values: required. | +| **         (#4)** | _object_ | | +| **                function*** | _object_ | | +| **                        name*** | _string_ | | +| **tool_prompt** | _string_ | A prompt to be appended before the tools | +| **tools** | _object[]_ | A list of tools the model may call. Currently, only functions are supported as a tool. Use this to provide a list of functions the model may generate JSON inputs for. | +| **        function*** | _object_ | | +| **                parameters*** | _unknown_ | | +| **                description** | _string_ | | +| **                name*** | _string_ | | +| **        type*** | _string_ | | +| **top_logprobs** | _integer_ | An integer between 0 and 5 specifying the number of most likely tokens to return at each token position, each with an associated log probability. logprobs must be set to true if this parameter is used. | +| **top_p** | _number_ | An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered. | + #### Response Output type depends on the `stream` input parameter. If `stream` is `false` (default), the response will be a JSON object with the following fields: -| Body | | -| :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :--------- | :-------------------- | -| **choices** | _object[]_ | | -| **        finish_reason** | _string_ | | -| **        index** | _integer_ | | -| **        logprobs** | _object_ | | -| **                content** | _object[]_ | | -| **                        logprob** | _number_ | | -| **                        token** | _string_ | | -| **                        top_logprobs** | _object[]_ | | -| **                                logprob** | _number_ | | -| **                                token** | _string_ | | -| **        message** | _unknown_ | One of the following: | -| **                 (#1)** | _object_ | | -| **                        content** | _string_ | | -| **                        role** | _string_ | | -| **                        tool_call_id** | _string_ | | -| **                 (#2)** | _object_ | | -| **                        role** | _string_ | | -| **                        tool_calls** | _object[]_ | | -| **                                function** | _object_ | | -| **                                        arguments** | _string_ | | -| **                                        description** | _string_ | | -| **                                        name** | _string_ | | -| **                                id** | _string_ | | -| **                                type** | _string_ | | -| **created** | _integer_ | | -| **id** | _string_ | | -| **model** | _string_ | | -| **system_fingerprint** | _string_ | | -| **usage** | _object_ | | -| **        completion_tokens** | _integer_ | | -| **        prompt_tokens** | _integer_ | | -| **        total_tokens** | _integer_ | | +| Body | | +| :--- | :--- | :--- | +| **choices** | _object[]_ | | +| **        finish_reason** | _string_ | | +| **        index** | _integer_ | | +| **        logprobs** | _object_ | | +| **                content** | _object[]_ | | +| **                        logprob** | _number_ | | +| **                        token** | _string_ | | +| **                        top_logprobs** | _object[]_ | | +| **                                logprob** | _number_ | | +| **                                token** | _string_ | | +| **        message** | _unknown_ | One of the following: | +| **                 (#1)** | _object_ | | +| **                        content** | _string_ | | +| **                        role** | _string_ | | +| **                        tool_call_id** | _string_ | | +| **                 (#2)** | _object_ | | +| **                        role** | _string_ | | +| **                        tool_calls** | _object[]_ | | +| **                                function** | _object_ | | +| **                                        arguments** | _string_ | | +| **                                        description** | _string_ | | +| **                                        name** | _string_ | | +| **                                id** | _string_ | | +| **                                type** | _string_ | | +| **created** | _integer_ | | +| **id** | _string_ | | +| **model** | _string_ | | +| **system_fingerprint** | _string_ | | +| **usage** | _object_ | | +| **        completion_tokens** | _integer_ | | +| **        prompt_tokens** | _integer_ | | +| **        total_tokens** | _integer_ | | + If `stream` is `true`, generated tokens are returned as a stream, using Server-Sent Events (SSE). For more information about streaming, check out [this guide](https://huggingface.co/docs/text-generation-inference/conceptual/streaming). -| Body | | -| :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | :--------- | :-------------------- | -| **choices** | _object[]_ | | -| **        delta** | _unknown_ | One of the following: | -| **                 (#1)** | _object_ | | -| **                        content** | _string_ | | -| **                        role** | _string_ | | -| **                        tool_call_id** | _string_ | | -| **                 (#2)** | _object_ | | -| **                        role** | _string_ | | -| **                        tool_calls** | _object[]_ | | -| **                                function** | _object_ | | -| **                                        arguments** | _string_ | | -| **                                        name** | _string_ | | -| **                                id** | _string_ | | -| **                                index** | _integer_ | | -| **                                type** | _string_ | | -| **        finish_reason** | _string_ | | -| **        index** | _integer_ | | -| **        logprobs** | _object_ | | -| **                content** | _object[]_ | | -| **                        logprob** | _number_ | | -| **                        token** | _string_ | | -| **                        top_logprobs** | _object[]_ | | -| **                                logprob** | _number_ | | -| **                                token** | _string_ | | -| **created** | _integer_ | | -| **id** | _string_ | | -| **model** | _string_ | | -| **system_fingerprint** | _string_ | | -| **usage** | _object_ | | -| **        completion_tokens** | _integer_ | | -| **        prompt_tokens** | _integer_ | | -| **        total_tokens** | _integer_ | | +| Body | | +| :--- | :--- | :--- | +| **choices** | _object[]_ | | +| **        delta** | _unknown_ | One of the following: | +| **                 (#1)** | _object_ | | +| **                        content** | _string_ | | +| **                        role** | _string_ | | +| **                        tool_call_id** | _string_ | | +| **                 (#2)** | _object_ | | +| **                        role** | _string_ | | +| **                        tool_calls** | _object[]_ | | +| **                                function** | _object_ | | +| **                                        arguments** | _string_ | | +| **                                        name** | _string_ | | +| **                                id** | _string_ | | +| **                                index** | _integer_ | | +| **                                type** | _string_ | | +| **        finish_reason** | _string_ | | +| **        index** | _integer_ | | +| **        logprobs** | _object_ | | +| **                content** | _object[]_ | | +| **                        logprob** | _number_ | | +| **                        token** | _string_ | | +| **                        top_logprobs** | _object[]_ | | +| **                                logprob** | _number_ | | +| **                                token** | _string_ | | +| **created** | _integer_ | | +| **id** | _string_ | | +| **model** | _string_ | | +| **system_fingerprint** | _string_ | | +| **usage** | _object_ | | +| **        completion_tokens** | _integer_ | | +| **        prompt_tokens** | _integer_ | | +| **        total_tokens** | _integer_ | | + +