ollama/model/renderers/qwen3vl.go

package renderers

import (
	"encoding/json"
	"strings"

	"github.com/ollama/ollama/api"
)

var imageCount int
var videoCount int

func marshalWithSpaces(v any) ([]byte, error) {
	b, err := json.Marshal(v)
	if err != nil {
		return nil, err
	}

	out := make([]byte, 0, len(b)+len(b)/8)
	inStr, esc := false, false
	for _, c := range b {
		if inStr {
			out = append(out, c)
			if esc {
				esc = false
				continue
			}
			if c == '\\' {
				esc = true
				continue
			}
			if c == '"' {
				inStr = false
			}
			continue
		}
		switch c {
		case '"':
			inStr = true
			out = append(out, c)
		case ':':
			out = append(out, ':', ' ')
		case ',':
			out = append(out, ',', ' ')
		default:
			out = append(out, c)
		}
	}
	return out, nil
}

type Qwen3VLRenderer struct {
	isThinking bool
}

// func renderContent(content api.Message, doVisionCount bool) string {
func (r *Qwen3VLRenderer) renderContent(content api.Message, doVisionCount bool) string {
	// This assumes all images are at the front of the message - same assumption as ollama/ollama/runner.go
	var subSb strings.Builder
	for _ = range content.Images {
		if doVisionCount {
			imageCount++
		}
		subSb.WriteString("<|vision_start|><|image_pad|><|vision_end|>")
	}
	// TODO: support videos

	subSb.WriteString(content.Content)
	return subSb.String()
}

// func Qwen3VLRenderer(messages []api.Message, tools []api.Tool, _ *api.ThinkValue) (string, error) {
func (r *Qwen3VLRenderer) Render(messages []api.Message, tools []api.Tool, _ *api.ThinkValue) (string, error) {
	var sb strings.Builder
	// r.isThinking = false

	if len(tools) > 0 {
		sb.WriteString(imStartTag + "system\n")
		if len(messages) > 0 && messages[0].Role == "system" {
			sb.WriteString(messages[0].Content + "\n\n")
		}
		sb.WriteString("# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>")
		for _, tool := range tools {
			sb.WriteString("\n")
			if b, err := marshalWithSpaces(tool); err == nil {
				sb.Write(b)
			}
		}
		sb.WriteString("\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n")
	} else if len(messages) > 0 && messages[0].Role == "system" {
		sb.WriteString("<|im_start|>system\n" + messages[0].Content + "<|im_end|>\n")
	}
	multiStepTool := true
	lastQueryIndex := len(messages) - 1

	for i := len(messages) - 1; i >= 0; i-- {
		message := messages[i]
		if multiStepTool && message.Role == "user" {
			// Check if content starts with <tool_response> and ends with </tool_response>
			content := message.Content
			if !(strings.HasPrefix(content, "<tool_response>") && strings.HasSuffix(content, "</tool_response>")) {
				multiStepTool = false
				lastQueryIndex = i
			}
		}
	}

	for i, message := range messages {
		content := r.renderContent(message, true)

		if message.Role == "user" || message.Role == "system" && i != 0 {
			sb.WriteString("<|im_start|>" + message.Role + "\n" + content + "<|im_end|>\n")
		} else if message.Role == "assistant" {
			contentReasoning := ""

			// here we need to reconstruct
			if r.isThinking { // we only do this if its a thinking model (i.e contentReasoning != "" if its a thinking model)
				if message.Thinking != "" {
					contentReasoning = message.Thinking
				} else if strings.Contains(content, "</think>") {
					contentReasoning = strings.Split(content, "</think>")[0]
					contentReasoning = strings.TrimRight(contentReasoning, "\n")

					contentReasoningSplit := strings.Split(contentReasoning, "<think>")
					contentReasoning = contentReasoningSplit[len(contentReasoningSplit)-1]

					contentReasoning = strings.TrimLeft(contentReasoning, "\n")

					contentSplit := strings.Split(content, "</think>")
					content = contentSplit[len(contentSplit)-1]
					content = strings.TrimLeft(content, "\n")
				}
			}
			// reconstruct the content

			// isThinking && i > lastQueryIndex
			if r.isThinking && i > lastQueryIndex { // if it is a thinking model
				if i == len(messages)-1 || contentReasoning != "" {
					sb.WriteString("<|im_start|>" + message.Role + "\n<think>\n" + strings.Trim(contentReasoning, "\n") + "\n</think>\n\n" + strings.TrimLeft(content, "\n"))
				} else {
					sb.WriteString("<|im_start|>" + message.Role + "\n" + content)
				}
			} else {
				sb.WriteString("<|im_start|>" + message.Role + "\n" + content)
			}

			if len(message.ToolCalls) > 0 {
				for j, toolCall := range message.ToolCalls {
					if j > 0 || content != "" {
						sb.WriteString("\n")
					}

					sb.WriteString("<tool_call>\n{\"name\": \"" + toolCall.Function.Name + "\", \"arguments\": ")
					if b, err := marshalWithSpaces(toolCall.Function.Arguments); err == nil {
						sb.Write(b)
					}
					sb.WriteString("}\n</tool_call>")
				}
			}
			sb.WriteString("<|im_end|>\n")
		} else if message.Role == "tool" {
			if i == 0 || messages[i-1].Role != "tool" {
				sb.WriteString("<|im_start|>user")
			}
			sb.WriteString("\n<tool_response>\n" + message.Content + "\n</tool_response>")
			if i == len(messages)-1 || messages[i+1].Role != "tool" {
				sb.WriteString("<|im_end|>\n")
			}
		}

	}

	sb.WriteString("<|im_start|>assistant\n")
	if r.isThinking {
		sb.WriteString("<think>\n") // Thinking models end with <|im_start|>assistant\n<think>\n
	}

	return sb.String(), nil

}