From fdd4d479a3cbaff1a7fe849e38a145652f87d611 Mon Sep 17 00:00:00 2001 From: Daniel Hiltgen Date: Thu, 22 May 2025 09:12:32 -0700 Subject: [PATCH 01/54] integration: add qwen2.5-vl (#10815) Replace the older llava model with qwen2.5 for vision tests Skip split-batch test on small VRAM systems to avoid excessive test time --- integration/llm_image_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/integration/llm_image_test.go b/integration/llm_image_test.go index b9726c8f0..bbd031a93 100644 --- a/integration/llm_image_test.go +++ b/integration/llm_image_test.go @@ -19,7 +19,7 @@ func TestVisionModels(t *testing.T) { } testCases := []testCase{ { - model: "llava:7b", + model: "qwen2.5vl", }, { model: "llama3.2-vision", @@ -60,6 +60,7 @@ func TestVisionModels(t *testing.T) { } func TestIntegrationSplitBatch(t *testing.T) { + skipUnderMinVRAM(t, 6) image, err := base64.StdEncoding.DecodeString(imageEncoding) require.NoError(t, err) req := api.GenerateRequest{ From fbe6ae285a23baddb14c5bbce26d4fcb837503e4 Mon Sep 17 00:00:00 2001 From: Bruce MacDonald Date: Thu, 22 May 2025 10:48:08 -0700 Subject: [PATCH 02/54] server: improve tensor quantization fallback logic (#10806) Fall back to alternative quantization types when a tensor's dimensions aren't divisible by the block size required for the original desired quantization type. If retried quantization types fail, the system ultimately falls back to F16 (half-precision floating point) which has a block size of 1 and can handle any tensor dimension. --- server/quantization.go | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/server/quantization.go b/server/quantization.go index adfc948ec..e57e8a4da 100644 --- a/server/quantization.go +++ b/server/quantization.go @@ -120,14 +120,30 @@ func getTensorNewType(kv fsggml.KV, qs *quantizeState, newType fsggml.TensorType if newType.IsQuantized() { nx := shape[0] - ny := uint64(1) - if len(shape) > 1 { - ny = shape[1] - } qk_k := newType.BlockSize() + + // Check if first dimension is divisible by block size if nx%qk_k != 0 { - slog.Warn(fmt.Sprintf("tensor cols %d x %d are not divisible by %d, required for %s. Falling back to quantization %s", nx, ny, qk_k, newType.String(), fsggml.TensorTypeF16.String())) - newType = fsggml.TensorTypeF16 + // Store the original type for logging + originalType := newType + + // Select appropriate fallback based on original type + switch newType { + case fsggml.TensorTypeQ4_K: + newType = fsggml.TensorTypeQ5_0 + case fsggml.TensorTypeQ5_K: + newType = fsggml.TensorTypeQ5_1 + case fsggml.TensorTypeQ6_K: + newType = fsggml.TensorTypeQ8_0 + } + + // Final check - if still incompatible, fall back to F16 + if nx%newType.BlockSize() != 0 { + newType = fsggml.TensorTypeF16 + } + + slog.Warn(fmt.Sprintf("tensor cols %d are not divisible by %d, required for %s - using fallback quantization %s", + nx, qk_k, originalType.String(), newType.String())) } } return newType From adff143bcda0c7ab4ca3a85dc3db5a81552368c7 Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Thu, 22 May 2025 11:30:49 -0700 Subject: [PATCH 03/54] fix: mllama quality (#10807) * fix mllama convert - transform attn_gate and ffn_gate - swap attention heads for vision models * fix mllama the mlp gate which was applied in the wrong place --- convert/convert_mllama.go | 65 +++++++++++++++++++---------- model/models/mllama/model_vision.go | 22 ++-------- 2 files changed, 45 insertions(+), 42 deletions(-) diff --git a/convert/convert_mllama.go b/convert/convert_mllama.go index 12478be71..69d7f5882 100644 --- a/convert/convert_mllama.go +++ b/convert/convert_mllama.go @@ -94,7 +94,9 @@ func (m *mllamaModel) Tensors(ts []Tensor) []*ggml.Tensor { var out []*ggml.Tensor var text []Tensor for _, t := range ts { - if t.Name() == "v.position_embd.gate" { + if !strings.HasPrefix(t.Name(), "v.") && !strings.HasPrefix(t.Name(), "mm.") { + text = append(text, t) + } else if t.Name() == "v.position_embd.gate" { for _, name := range []string{"v.position_embd.gate", "v.tile_position_embd.gate"} { tt := t.Clone() tt.SetRepacker(m.repack(name)) @@ -105,23 +107,21 @@ func (m *mllamaModel) Tensors(ts []Tensor) []*ggml.Tensor { WriterTo: tt, }) } - } else if t.Name() == "v.pre_tile_position_embd.gate" || t.Name() == "v.post_tile_position_embd.gate" { - t.SetRepacker(m.repack(t.Name())) - out = append(out, &ggml.Tensor{ - Name: t.Name(), - Kind: t.Kind(), - Shape: t.Shape(), - WriterTo: t, - }) - } else if strings.HasPrefix(t.Name(), "v.") || strings.HasPrefix(t.Name(), "mm.") { - out = append(out, &ggml.Tensor{ - Name: t.Name(), - Kind: t.Kind(), - Shape: t.Shape(), - WriterTo: t, - }) } else { - text = append(text, t) + if t.Name() == "v.pre_tile_position_embd.gate" || t.Name() == "v.post_tile_position_embd.gate" { + t.SetRepacker(m.repack(t.Name())) + } else if strings.HasSuffix(t.Name(), "attn_q.weight") || strings.HasSuffix(t.Name(), "attn_k.weight") { + t.SetRepacker(m.repack(t.Name())) + } else if strings.HasSuffix(t.Name(), "attn_gate") || strings.HasSuffix(t.Name(), "ffn_gate") { + t.SetRepacker(m.repack(t.Name())) + } + + out = append(out, &ggml.Tensor{ + Name: t.Name(), + Kind: t.Kind(), + Shape: t.Shape(), + WriterTo: t, + }) } } @@ -137,16 +137,35 @@ func (m *mllamaModel) repack(name string) Repacker { var t tensor.Tensor = tensor.New(tensor.WithShape(dims...), tensor.WithBacking(data)) - t, err = tensor.Tanh(t) - if err != nil { - return nil, err - } + if strings.HasSuffix(name, "attn_q.weight") || strings.HasSuffix(name, "attn_k.weight") { + heads := m.VisionModel.AttentionHeads + if err := t.Reshape(append([]int{int(heads), 2, dims[0] / int(heads) / 2}, dims[1:]...)...); err != nil { + return nil, err + } - if name == "v.position_embd.gate" { - t, err = tensor.Sub(float32(1), t) + if err := t.T(0, 2, 1, 3); err != nil { + return nil, err + } + + if err := t.Reshape(dims...); err != nil { + return nil, err + } + + if err := t.Transpose(); err != nil { + return nil, err + } + } else { + t, err = tensor.Tanh(t) if err != nil { return nil, err } + + if name == "v.position_embd.gate" { + t, err = tensor.Sub(float32(1), t) + if err != nil { + return nil, err + } + } } t = tensor.Materialize(t) diff --git a/model/models/mllama/model_vision.go b/model/models/mllama/model_vision.go index 77ea53731..2d4249472 100644 --- a/model/models/mllama/model_vision.go +++ b/model/models/mllama/model_vision.go @@ -16,8 +16,6 @@ type VisionSelfAttention struct { Key *nn.Linear `gguf:"attn_k"` Value *nn.Linear `gguf:"attn_v"` Output *nn.Linear `gguf:"attn_output"` - - Gate ml.Tensor `gguf:"attn_gate"` } func (sa *VisionSelfAttention) Forward(ctx ml.Context, hiddenState ml.Tensor, opts *VisionModelOptions) ml.Tensor { @@ -25,27 +23,16 @@ func (sa *VisionSelfAttention) Forward(ctx ml.Context, hiddenState ml.Tensor, op query := sa.Query.Forward(ctx, hiddenState) query = query.Reshape(ctx, headDim, opts.numHeads, query.Dim(1), batchSize) - query = query.Permute(ctx, 0, 2, 1, 3).Contiguous(ctx) key := sa.Key.Forward(ctx, hiddenState) key = key.Reshape(ctx, headDim, opts.numHeads, key.Dim(1), batchSize) - key = key.Permute(ctx, 0, 2, 1, 3).Contiguous(ctx) value := sa.Value.Forward(ctx, hiddenState) value = value.Reshape(ctx, headDim, opts.numHeads, value.Dim(1), batchSize) - value = value.Permute(ctx, 1, 2, 0, 3).Contiguous(ctx) - scores := key.Mulmat(ctx, query) - scores = scores.Scale(ctx, 1.0/math.Sqrt(float64(headDim))) - scores = scores.Softmax(ctx) - - attention := value.Mulmat(ctx, scores) - attention = attention.Reshape(ctx, headDim, attention.Dim(1), opts.numHeads, batchSize) - attention = attention.Permute(ctx, 0, 2, 1, 3).Contiguous(ctx) + attention := nn.Attention(ctx, query, key, value, 1./math.Sqrt(float64(headDim)), nil) attention = attention.Reshape(ctx, opts.hiddenSize, attention.Dim(2), batchSize) - - hiddenState = sa.Output.Forward(ctx, attention) - return hiddenState + return sa.Output.Forward(ctx, attention) } type VisionMLP struct { @@ -76,21 +63,18 @@ func (e *VisionEncoderLayer) Forward(ctx ml.Context, hiddenState ml.Tensor, opts // self attention hiddenState = e.AttentionNorm.Forward(ctx, hiddenState, opts.eps) hiddenState = e.SelfAttention.Forward(ctx, hiddenState, opts) - if e.AttentionGate != nil { hiddenState = hiddenState.Mul(ctx, e.AttentionGate) } hiddenState = hiddenState.Add(ctx, residual) residual = hiddenState - // feed forward hiddenState = e.MLPNorm.Forward(ctx, hiddenState, opts.eps) hiddenState = e.MLP.Forward(ctx, hiddenState, opts) - hiddenState = hiddenState.Add(ctx, residual) if e.MLPGate != nil { hiddenState = hiddenState.Mul(ctx, e.MLPGate) } - + hiddenState = hiddenState.Add(ctx, residual) return hiddenState } From d950ff12c09c07a1cda7242373071fb9e7af9ddc Mon Sep 17 00:00:00 2001 From: Daniel Hiltgen Date: Thu, 22 May 2025 14:31:36 -0700 Subject: [PATCH 04/54] sched: fix runner leak during reloading unload (#10819) When the same model is being reloaded rapidly with client connections being canceled before the model finishes loading, the queued unload event could cause a leak of runners by deleting a different runner from the loaded list. --- server/sched.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/server/sched.go b/server/sched.go index 3fc54e55a..612e47020 100644 --- a/server/sched.go +++ b/server/sched.go @@ -387,6 +387,17 @@ func (s *Scheduler) processCompleted(ctx context.Context) { s.loadedMu.Unlock() runner.refMu.Unlock() slog.Debug("duplicate expired event, ignoring", "runner", runner) + } else if runner.pid != runnerToUnload.pid { + // If the pids do not match, we likely had multiple load + // failures for the same model in quick succession due to + // request context canceled and are draining the queue of + // events. Ensure the orphaned runner is properly shut down, but + // do not delete the mismatched loaded runner, or wait for VRAM + // convergence. + slog.Debug("orphaned runner shutting down", "orphan", runner, "loaded", runnerToUnload) + runner.unload() + s.loadedMu.Unlock() + runner.refMu.Unlock() } else { slog.Debug("starting background wait for VRAM recovery", "runner", runner) finished := runner.waitForVRAMRecovery() From 6db8a3771c29d070ef165cca0d7e8dbda3fc341e Mon Sep 17 00:00:00 2001 From: Jesse Gross Date: Fri, 16 May 2025 14:05:08 -0700 Subject: [PATCH 05/54] ggml: Report graph memory for failed allocations GGML has a function to report the allocated size of a backend buffer. However, this returns 0 if we tried to allocate a buffer and it failed. For memory management purposes, it's important to know how much we were trying to allocate. This extends the API to report attempted sizes for all buffers and whether it succeeeded. --- ...16-graph-memory-reporting-on-failure.patch | 156 ++++++++++++++++++ ml/backend/ggml/ggml/include/ggml-alloc.h | 6 + ml/backend/ggml/ggml/include/ggml-backend.h | 6 + ml/backend/ggml/ggml/src/ggml-alloc.c | 38 ++++- ml/backend/ggml/ggml/src/ggml-backend.cpp | 10 ++ 5 files changed, 212 insertions(+), 4 deletions(-) create mode 100644 llama/patches/0016-graph-memory-reporting-on-failure.patch diff --git a/llama/patches/0016-graph-memory-reporting-on-failure.patch b/llama/patches/0016-graph-memory-reporting-on-failure.patch new file mode 100644 index 000000000..921882249 --- /dev/null +++ b/llama/patches/0016-graph-memory-reporting-on-failure.patch @@ -0,0 +1,156 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Jesse Gross +Date: Fri, 18 Apr 2025 15:58:19 -0700 +Subject: [PATCH] graph memory reporting on failure + +--- + ggml/include/ggml-alloc.h | 6 ++++++ + ggml/include/ggml-backend.h | 6 ++++++ + ggml/src/ggml-alloc.c | 38 +++++++++++++++++++++++++++++++++---- + ggml/src/ggml-backend.cpp | 10 ++++++++++ + 4 files changed, 56 insertions(+), 4 deletions(-) + +diff --git a/ggml/include/ggml-alloc.h b/ggml/include/ggml-alloc.h +index 2cb150fd..781b1e10 100644 +--- a/ggml/include/ggml-alloc.h ++++ b/ggml/include/ggml-alloc.h +@@ -66,6 +66,12 @@ GGML_API bool ggml_gallocr_alloc_graph(ggml_gallocr_t galloc, struct ggml_cgraph + + GGML_API size_t ggml_gallocr_get_buffer_size(ggml_gallocr_t galloc, int buffer_id); + ++struct ggml_allocr_buffer_status { ++ size_t size; ++ bool allocated; ++}; ++GGML_API struct ggml_allocr_buffer_status ggml_gallocr_get_attempted_buffer_size(ggml_gallocr_t galloc, int buffer_id); ++ + // Utils + // Create a buffer and allocate all the tensors in a ggml_context + GGML_API struct ggml_backend_buffer * ggml_backend_alloc_ctx_tensors_from_buft(struct ggml_context * ctx, ggml_backend_buffer_type_t buft); +diff --git a/ggml/include/ggml-backend.h b/ggml/include/ggml-backend.h +index 778927f6..74e46716 100644 +--- a/ggml/include/ggml-backend.h ++++ b/ggml/include/ggml-backend.h +@@ -304,6 +304,12 @@ extern "C" { + + GGML_API size_t ggml_backend_sched_get_buffer_size(ggml_backend_sched_t sched, ggml_backend_t backend); + ++ struct ggml_backend_buffer_status { ++ size_t size; ++ bool allocated; ++ }; ++ GGML_API struct ggml_backend_buffer_status ggml_backend_sched_get_attempted_buffer_size(ggml_backend_sched_t sched, ggml_backend_t backend); ++ + GGML_API void ggml_backend_sched_set_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node, ggml_backend_t backend); + GGML_API ggml_backend_t ggml_backend_sched_get_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node); + +diff --git a/ggml/src/ggml-alloc.c b/ggml/src/ggml-alloc.c +index 5fd379f6..04812990 100644 +--- a/ggml/src/ggml-alloc.c ++++ b/ggml/src/ggml-alloc.c +@@ -364,6 +364,7 @@ struct node_alloc { + struct ggml_gallocr { + ggml_backend_buffer_type_t * bufts; // [n_buffers] + ggml_backend_buffer_t * buffers; // [n_buffers] ++ size_t *buffer_sizes; // [n_buffers] + struct ggml_dyn_tallocr ** buf_tallocs; // [n_buffers] + int n_buffers; + +@@ -387,6 +388,9 @@ ggml_gallocr_t ggml_gallocr_new_n(ggml_backend_buffer_type_t * bufts, int n_bufs + galloc->buffers = calloc(n_bufs, sizeof(ggml_backend_buffer_t)); + GGML_ASSERT(galloc->buffers != NULL); + ++ galloc->buffer_sizes = calloc(n_bufs, sizeof(size_t)); ++ GGML_ASSERT(galloc->buffer_sizes != NULL); ++ + galloc->buf_tallocs = calloc(n_bufs, sizeof(struct ggml_dyn_tallocr *)); + GGML_ASSERT(galloc->buf_tallocs != NULL); + +@@ -453,6 +457,7 @@ void ggml_gallocr_free(ggml_gallocr_t galloc) { + ggml_hash_set_free(&galloc->hash_set); + free(galloc->hash_values); + free(galloc->bufts); ++ free(galloc->buffer_sizes); + free(galloc->buffers); + free(galloc->buf_tallocs); + free(galloc->node_allocs); +@@ -748,6 +753,8 @@ bool ggml_gallocr_reserve_n(ggml_gallocr_t galloc, struct ggml_cgraph * graph, c + } + } + ++ bool success = true; ++ + // reallocate buffers if needed + for (int i = 0; i < galloc->n_buffers; i++) { + // if the buffer type is used multiple times, we reuse the same buffer +@@ -769,15 +776,20 @@ bool ggml_gallocr_reserve_n(ggml_gallocr_t galloc, struct ggml_cgraph * graph, c + + ggml_backend_buffer_free(galloc->buffers[i]); + galloc->buffers[i] = ggml_backend_buft_alloc_buffer(galloc->bufts[i], new_size); +- if (galloc->buffers[i] == NULL) { ++ if (galloc->buffers[i]) { ++ galloc->buffer_sizes[i] = ggml_backend_buffer_get_size(galloc->buffers[i]); ++ ggml_backend_buffer_set_usage(galloc->buffers[i], GGML_BACKEND_BUFFER_USAGE_COMPUTE); ++ } else { + GGML_LOG_ERROR("%s: failed to allocate %s buffer of size %zu\n", __func__, ggml_backend_buft_name(galloc->bufts[i]), new_size); +- return false; ++ galloc->buffer_sizes[i] = new_size; ++ success = false; + } +- ggml_backend_buffer_set_usage(galloc->buffers[i], GGML_BACKEND_BUFFER_USAGE_COMPUTE); ++ } else { ++ galloc->buffer_sizes[i] = ggml_backend_buffer_get_size(galloc->buffers[i]); + } + } + +- return true; ++ return success; + } + + bool ggml_gallocr_reserve(ggml_gallocr_t galloc, struct ggml_cgraph *graph) { +@@ -934,6 +946,24 @@ size_t ggml_gallocr_get_buffer_size(ggml_gallocr_t galloc, int buffer_id) { + return ggml_backend_buffer_get_size(galloc->buffers[buffer_id]); + } + ++struct ggml_allocr_buffer_status ggml_gallocr_get_attempted_buffer_size(ggml_gallocr_t galloc, int buffer_id) { ++ GGML_ASSERT(buffer_id >= 0 && buffer_id < galloc->n_buffers); ++ ++ for (int i = 0; i < buffer_id; i++) { ++ if (galloc->buf_tallocs[i] == galloc->buf_tallocs[buffer_id]) { ++ // This buffer is the same as a previous one due to the same buffer type being used multiple times ++ // (See above.) However, we need a different check because multiple buffers might be NULL in our ++ // case and we still want to know the attempted size. ++ ++ struct ggml_allocr_buffer_status status = {0, true}; ++ return status; ++ } ++ } ++ ++ struct ggml_allocr_buffer_status status = {galloc->buffer_sizes[buffer_id], galloc->buffers[buffer_id] != NULL}; ++ return status; ++} ++ + // utils + + static void free_buffers(ggml_backend_buffer_t ** buffers, const size_t * n_buffers) { +diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp +index 0ce73a99..be335e8c 100644 +--- a/ggml/src/ggml-backend.cpp ++++ b/ggml/src/ggml-backend.cpp +@@ -1629,6 +1629,16 @@ size_t ggml_backend_sched_get_buffer_size(ggml_backend_sched_t sched, ggml_backe + return ggml_gallocr_get_buffer_size(sched->galloc, backend_index); + } + ++struct ggml_backend_buffer_status ggml_backend_sched_get_attempted_buffer_size(ggml_backend_sched_t sched, ggml_backend_t backend) { ++ int backend_index = ggml_backend_sched_backend_id(sched, backend); ++ GGML_ASSERT(backend_index >= 0 && backend_index < sched->n_backends); ++ ++ struct ggml_allocr_buffer_status allocr_status = ggml_gallocr_get_attempted_buffer_size(sched->galloc, backend_index); ++ struct ggml_backend_buffer_status status = {allocr_status.size, allocr_status.allocated}; ++ ++ return status; ++} ++ + void ggml_backend_sched_set_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node, ggml_backend_t backend) { + int backend_index = ggml_backend_sched_backend_id(sched, backend); + GGML_ASSERT(backend_index >= 0 && backend_index < sched->n_backends); diff --git a/ml/backend/ggml/ggml/include/ggml-alloc.h b/ml/backend/ggml/ggml/include/ggml-alloc.h index 2cb150fd2..781b1e100 100644 --- a/ml/backend/ggml/ggml/include/ggml-alloc.h +++ b/ml/backend/ggml/ggml/include/ggml-alloc.h @@ -66,6 +66,12 @@ GGML_API bool ggml_gallocr_alloc_graph(ggml_gallocr_t galloc, struct ggml_cgraph GGML_API size_t ggml_gallocr_get_buffer_size(ggml_gallocr_t galloc, int buffer_id); +struct ggml_allocr_buffer_status { + size_t size; + bool allocated; +}; +GGML_API struct ggml_allocr_buffer_status ggml_gallocr_get_attempted_buffer_size(ggml_gallocr_t galloc, int buffer_id); + // Utils // Create a buffer and allocate all the tensors in a ggml_context GGML_API struct ggml_backend_buffer * ggml_backend_alloc_ctx_tensors_from_buft(struct ggml_context * ctx, ggml_backend_buffer_type_t buft); diff --git a/ml/backend/ggml/ggml/include/ggml-backend.h b/ml/backend/ggml/ggml/include/ggml-backend.h index 778927f68..74e467163 100644 --- a/ml/backend/ggml/ggml/include/ggml-backend.h +++ b/ml/backend/ggml/ggml/include/ggml-backend.h @@ -304,6 +304,12 @@ extern "C" { GGML_API size_t ggml_backend_sched_get_buffer_size(ggml_backend_sched_t sched, ggml_backend_t backend); + struct ggml_backend_buffer_status { + size_t size; + bool allocated; + }; + GGML_API struct ggml_backend_buffer_status ggml_backend_sched_get_attempted_buffer_size(ggml_backend_sched_t sched, ggml_backend_t backend); + GGML_API void ggml_backend_sched_set_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node, ggml_backend_t backend); GGML_API ggml_backend_t ggml_backend_sched_get_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node); diff --git a/ml/backend/ggml/ggml/src/ggml-alloc.c b/ml/backend/ggml/ggml/src/ggml-alloc.c index 5fd379f6a..048129908 100644 --- a/ml/backend/ggml/ggml/src/ggml-alloc.c +++ b/ml/backend/ggml/ggml/src/ggml-alloc.c @@ -364,6 +364,7 @@ struct node_alloc { struct ggml_gallocr { ggml_backend_buffer_type_t * bufts; // [n_buffers] ggml_backend_buffer_t * buffers; // [n_buffers] + size_t *buffer_sizes; // [n_buffers] struct ggml_dyn_tallocr ** buf_tallocs; // [n_buffers] int n_buffers; @@ -387,6 +388,9 @@ ggml_gallocr_t ggml_gallocr_new_n(ggml_backend_buffer_type_t * bufts, int n_bufs galloc->buffers = calloc(n_bufs, sizeof(ggml_backend_buffer_t)); GGML_ASSERT(galloc->buffers != NULL); + galloc->buffer_sizes = calloc(n_bufs, sizeof(size_t)); + GGML_ASSERT(galloc->buffer_sizes != NULL); + galloc->buf_tallocs = calloc(n_bufs, sizeof(struct ggml_dyn_tallocr *)); GGML_ASSERT(galloc->buf_tallocs != NULL); @@ -453,6 +457,7 @@ void ggml_gallocr_free(ggml_gallocr_t galloc) { ggml_hash_set_free(&galloc->hash_set); free(galloc->hash_values); free(galloc->bufts); + free(galloc->buffer_sizes); free(galloc->buffers); free(galloc->buf_tallocs); free(galloc->node_allocs); @@ -748,6 +753,8 @@ bool ggml_gallocr_reserve_n(ggml_gallocr_t galloc, struct ggml_cgraph * graph, c } } + bool success = true; + // reallocate buffers if needed for (int i = 0; i < galloc->n_buffers; i++) { // if the buffer type is used multiple times, we reuse the same buffer @@ -769,15 +776,20 @@ bool ggml_gallocr_reserve_n(ggml_gallocr_t galloc, struct ggml_cgraph * graph, c ggml_backend_buffer_free(galloc->buffers[i]); galloc->buffers[i] = ggml_backend_buft_alloc_buffer(galloc->bufts[i], new_size); - if (galloc->buffers[i] == NULL) { + if (galloc->buffers[i]) { + galloc->buffer_sizes[i] = ggml_backend_buffer_get_size(galloc->buffers[i]); + ggml_backend_buffer_set_usage(galloc->buffers[i], GGML_BACKEND_BUFFER_USAGE_COMPUTE); + } else { GGML_LOG_ERROR("%s: failed to allocate %s buffer of size %zu\n", __func__, ggml_backend_buft_name(galloc->bufts[i]), new_size); - return false; + galloc->buffer_sizes[i] = new_size; + success = false; } - ggml_backend_buffer_set_usage(galloc->buffers[i], GGML_BACKEND_BUFFER_USAGE_COMPUTE); + } else { + galloc->buffer_sizes[i] = ggml_backend_buffer_get_size(galloc->buffers[i]); } } - return true; + return success; } bool ggml_gallocr_reserve(ggml_gallocr_t galloc, struct ggml_cgraph *graph) { @@ -934,6 +946,24 @@ size_t ggml_gallocr_get_buffer_size(ggml_gallocr_t galloc, int buffer_id) { return ggml_backend_buffer_get_size(galloc->buffers[buffer_id]); } +struct ggml_allocr_buffer_status ggml_gallocr_get_attempted_buffer_size(ggml_gallocr_t galloc, int buffer_id) { + GGML_ASSERT(buffer_id >= 0 && buffer_id < galloc->n_buffers); + + for (int i = 0; i < buffer_id; i++) { + if (galloc->buf_tallocs[i] == galloc->buf_tallocs[buffer_id]) { + // This buffer is the same as a previous one due to the same buffer type being used multiple times + // (See above.) However, we need a different check because multiple buffers might be NULL in our + // case and we still want to know the attempted size. + + struct ggml_allocr_buffer_status status = {0, true}; + return status; + } + } + + struct ggml_allocr_buffer_status status = {galloc->buffer_sizes[buffer_id], galloc->buffers[buffer_id] != NULL}; + return status; +} + // utils static void free_buffers(ggml_backend_buffer_t ** buffers, const size_t * n_buffers) { diff --git a/ml/backend/ggml/ggml/src/ggml-backend.cpp b/ml/backend/ggml/ggml/src/ggml-backend.cpp index 0ce73a997..be335e8ca 100644 --- a/ml/backend/ggml/ggml/src/ggml-backend.cpp +++ b/ml/backend/ggml/ggml/src/ggml-backend.cpp @@ -1629,6 +1629,16 @@ size_t ggml_backend_sched_get_buffer_size(ggml_backend_sched_t sched, ggml_backe return ggml_gallocr_get_buffer_size(sched->galloc, backend_index); } +struct ggml_backend_buffer_status ggml_backend_sched_get_attempted_buffer_size(ggml_backend_sched_t sched, ggml_backend_t backend) { + int backend_index = ggml_backend_sched_backend_id(sched, backend); + GGML_ASSERT(backend_index >= 0 && backend_index < sched->n_backends); + + struct ggml_allocr_buffer_status allocr_status = ggml_gallocr_get_attempted_buffer_size(sched->galloc, backend_index); + struct ggml_backend_buffer_status status = {allocr_status.size, allocr_status.allocated}; + + return status; +} + void ggml_backend_sched_set_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node, ggml_backend_t backend) { int backend_index = ggml_backend_sched_backend_id(sched, backend); GGML_ASSERT(backend_index >= 0 && backend_index < sched->n_backends); From 73d6a82cce18f84ff5c67148783224cf25b30b32 Mon Sep 17 00:00:00 2001 From: Jesse Gross Date: Thu, 17 Apr 2025 11:00:25 -0700 Subject: [PATCH 06/54] ollamarunner: Memory usage reporting This provides granular information about the backend memory allocations required by the runner: - Per backend - Per layer - Weights, cache and graph - Allocation status This can be used for debugging and validating memory estimates. --- kvcache/causal_test.go | 2 +- ml/backend.go | 84 ++++++++++++++- ml/backend/ggml/ggml.go | 163 ++++++++++++++++++++---------- runner/ollamarunner/multimodal.go | 5 +- runner/ollamarunner/runner.go | 48 +++++---- 5 files changed, 224 insertions(+), 78 deletions(-) diff --git a/kvcache/causal_test.go b/kvcache/causal_test.go index 796987088..820d496d1 100644 --- a/kvcache/causal_test.go +++ b/kvcache/causal_test.go @@ -508,7 +508,7 @@ func (c *testContext) Forward(...ml.Tensor) ml.Context { return c } func (c *testContext) Compute(...ml.Tensor) {} -func (c *testContext) Reserve() error { return nil } +func (c *testContext) Reserve() {} func (c *testContext) MaxGraphNodes() int { return 10 diff --git a/ml/backend.go b/ml/backend.go index 3c417ef9d..7c9b9e313 100644 --- a/ml/backend.go +++ b/ml/backend.go @@ -15,6 +15,10 @@ import ( type Backend interface { Load(ctx context.Context, progress func(float32)) error + + // BackendMemory returns the memory allocations that were made for this model + BackendMemory() BackendMemory + Config() fs.Config Get(name string) Tensor NewContext() Context @@ -68,6 +72,84 @@ type BackendParams struct { FlashAttention bool } +// ErrNoMem is returned when panicing due to insufficient memory. It includes +// the attempted memory allocation. +type ErrNoMem struct { + BackendMemory +} + +func (e ErrNoMem) Error() string { + return fmt.Sprintf("insufficient memory - required allocations: %+v", e.BackendMemory) +} + +type AllocationStatus int + +const ( + // Unallocated memory - have not yet attempted to allocate + Unallocated AllocationStatus = iota + + // Failed memory - tried to allocate the memory and did not succeed + Failed + + // Allocated memory = tried and succeeded to allocate memory + Allocated +) + +// Memory is the size of an allocation and whether it was successful. +type Memory struct { + Size uint64 + Status AllocationStatus +} + +func (m Memory) String() string { + s := fmt.Sprint(m.Size) + + switch m.Status { + case Unallocated: + s += "U" + case Failed: + s += "F" + case Allocated: + s += "A" + } + + return s +} + +// DeviceMemory provides a breakdown of the memory needed +// per device, such as a CPU or GPU. +type DeviceMemory struct { + // Name is the name of the device as labeled by the backend. It + // may not be persistent across instances of the runner. + Name string + + // Weights is the per-layer memory needed for the model weights. + Weights []Memory + + // Cache is the per-layer memory needed for the KV cache. + Cache []Memory + + // Graph is the size of the compute graph. It is not per-layer. + Graph Memory +} + +// BackendMemory provides the amount of memory required to load the model +// per device based on the BackendParams. In some cases, not all required +// allocations will be known at this point. However, the size of the most recent +// allocation is guaranteed to be provided so that if it failed, the caller can +// accommodate that to make forward progress. +type BackendMemory struct { + // InputsWeights are always located on the CPU and cannot be moved + InputWeights Memory + + // CPU model components are located in system memory. This does not + // include unified memory allocated through the GPU. + CPU DeviceMemory + + // GPU model components are located on one or more GPUs. + GPUs []DeviceMemory +} + var backends = make(map[string]func(string, BackendParams) (Backend, error)) func RegisterBackend(name string, f func(string, BackendParams) (Backend, error)) { @@ -102,7 +184,7 @@ type Context interface { // graph, simply preallocates memory. Typically called with a // worst case graph to ensure all resources are available for // for future inference. - Reserve() error + Reserve() MaxGraphNodes() int Close() diff --git a/ml/backend/ggml/ggml.go b/ml/backend/ggml/ggml.go index 44e3b61bc..496ba8a60 100644 --- a/ml/backend/ggml/ggml.go +++ b/ml/backend/ggml/ggml.go @@ -10,7 +10,6 @@ import "C" import ( "context" - "errors" "fmt" "io" "log/slog" @@ -66,6 +65,12 @@ type Backend struct { // layers is the backend used for repeating layers layers map[int]*C.struct_ggml_backend_buffer_type + // requiredMemory is the cumulative memory allocations needed by the backend + requiredMemory *ml.BackendMemory + + // btDeviceMemory maps from a buffer type to the memory allocations associated with that device + btDeviceMemory map[*C.struct_ggml_backend_buffer_type]*ml.DeviceMemory + flashAttention bool // maxGraphNodes is the maximum allowed number of graph nodes in this scheduler @@ -94,6 +99,9 @@ func New(modelPath string, params ml.BackendParams) (ml.Backend, error) { "num_key_values", len(meta.KV()), ) + var requiredMemory ml.BackendMemory + btDeviceMemory := make(map[*C.struct_ggml_backend_buffer_type]*ml.DeviceMemory) + type deviceBufferType struct { d *C.struct_ggml_backend_device bts []*C.struct_ggml_backend_buffer_type @@ -114,6 +122,8 @@ func New(modelPath string, params ml.BackendParams) (ml.Backend, error) { } } + blocks := int(meta.KV().BlockCount()) + // create list of buffer types for the cpu cpuDeviceBufferType := deviceBufferType{d: C.ggml_backend_dev_by_type(C.GGML_BACKEND_DEVICE_TYPE_CPU)} for _, d := range append(accels, append(gpus, cpus...)...) { @@ -121,17 +131,27 @@ func New(modelPath string, params ml.BackendParams) (ml.Backend, error) { case C.GGML_BACKEND_DEVICE_TYPE_CPU, C.GGML_BACKEND_DEVICE_TYPE_ACCEL: cpuDeviceBufferType.bts = append(cpuDeviceBufferType.bts, C.ggml_backend_dev_buffer_type(d)) + btDeviceMemory[C.ggml_backend_dev_buffer_type(d)] = &requiredMemory.CPU } } + requiredMemory.CPU.Name = C.GoString(C.ggml_backend_dev_name(cpuDeviceBufferType.d)) + requiredMemory.CPU.Weights = make([]ml.Memory, blocks+1) + requiredMemory.CPU.Cache = make([]ml.Memory, blocks+1) + // create list of buffer types for each gpu var gpuDeviceBufferTypes []deviceBufferType - for _, d := range gpus { + requiredMemory.GPUs = make([]ml.DeviceMemory, len(gpus)) + for i, d := range gpus { bt := C.ggml_backend_dev_buffer_type(d) gpuDeviceBufferTypes = append(gpuDeviceBufferTypes, deviceBufferType{ d: d, bts: append([]*C.struct_ggml_backend_buffer_type{bt}, cpuDeviceBufferType.bts...), }) + btDeviceMemory[bt] = &requiredMemory.GPUs[i] + requiredMemory.GPUs[i].Name = C.GoString(C.ggml_backend_dev_name(d)) + requiredMemory.GPUs[i].Weights = make([]ml.Memory, blocks+1) + requiredMemory.GPUs[i].Cache = make([]ml.Memory, blocks+1) } useDefaultSplit := true @@ -170,8 +190,6 @@ func New(modelPath string, params ml.BackendParams) (ml.Backend, error) { // inputs always use cpu input := cpuDeviceBufferType - blocks := int(meta.KV().BlockCount()) - // define a range of gpu layers. anything outside of this range is assigned to the cpu gpuRangeStart := max(0, blocks-params.NumGPULayers) gpuRangeStop := min(gpuRangeStart+params.NumGPULayers, blocks+1) @@ -212,7 +230,7 @@ func New(modelPath string, params ml.BackendParams) (ml.Backend, error) { // contexts are shared by tensors of the same buffer type ctxs := make(map[*C.struct_ggml_backend_buffer_type]*C.struct_ggml_context) - createTensor := func(t tensor, bts []*C.struct_ggml_backend_buffer_type) *C.struct_ggml_tensor { + createTensor := func(t tensor, bts []*C.struct_ggml_backend_buffer_type, layer int) *C.struct_ggml_tensor { for _, bt := range bts { if _, ok := ctxs[bt]; !ok { ctxs[bt] = C.ggml_init(C.struct_ggml_init_params{ @@ -238,6 +256,16 @@ func New(modelPath string, params ml.BackendParams) (ml.Backend, error) { C.ggml_set_name(tt, cname) slog.Log(context.TODO(), logutil.LevelTrace, "created tensor", "name", name, "shape", t.source.Shape, "dtype", t.source.Kind, "buffer_type", C.GoString(C.ggml_backend_buft_name(bt))) + + size := pad(C.ggml_backend_buft_get_alloc_size(bt, tt), C.ggml_backend_buft_get_alignment(bt)) + if layer == -1 { + // Assume that InputWeights can be allocated - they're always in system memory and can't be moved in any case + requiredMemory.InputWeights.Status = ml.Allocated + requiredMemory.InputWeights.Size += uint64(size) + } else { + btDeviceMemory[bt].Weights[layer].Size += uint64(size) + } + //nolint:staticcheck // TODO: check if buffer type supports this tensor return tt } @@ -259,22 +287,22 @@ func New(modelPath string, params ml.BackendParams) (ml.Backend, error) { for _, t := range meta.Tensors().Items() { switch { case contains(t.Name, "position_embd", "token_embd", "token_norm_embd", "token_types"): - createTensor(tensor{source: t}, input.bts) + createTensor(tensor{source: t}, input.bts, -1) if _, ok := meta.Tensors().GroupLayers()["output"]; !ok && t.Name == "token_embd.weight" { - createTensor(tensor{source: t, target: "output.weight"}, output.bts) + createTensor(tensor{source: t, target: "output.weight"}, output.bts, blocks) } case contains(t.Name, "cls", "output", "output_norm"): - createTensor(tensor{source: t}, output.bts) + createTensor(tensor{source: t}, output.bts, blocks) case strings.HasPrefix(t.Name, "v.") || strings.HasPrefix(t.Name, "mm."): // TODO: assign vision tensors to the gpu if possible - createTensor(tensor{source: t}, output.bts) + createTensor(tensor{source: t}, output.bts, blocks) case contains(t.Name, "rope_freqs", "rope_factors_long", "rope_factors_short"): // these tensors should be repeated per layer for i, layer := range layers { createTensor(tensor{ source: t, target: "blk." + strconv.Itoa(i) + "." + t.Name, - }, layer.bts) + }, layer.bts, i) } default: layerIndex := -1 @@ -285,10 +313,10 @@ func New(modelPath string, params ml.BackendParams) (ml.Backend, error) { } if layerIndex >= 0 { - createTensor(tensor{source: t}, layers[layerIndex].bts) + createTensor(tensor{source: t}, layers[layerIndex].bts, layerIndex) } else { // load all other tensors on the cpu - createTensor(tensor{source: t}, input.bts) + createTensor(tensor{source: t}, input.bts, -1) } } } @@ -301,8 +329,18 @@ func New(modelPath string, params ml.BackendParams) (ml.Backend, error) { } b := C.ggml_backend_alloc_ctx_tensors_from_buft(c, bt) + for i := range btDeviceMemory[bt].Weights { + if btDeviceMemory[bt].Weights[i].Size != 0 { + if b != nil { + btDeviceMemory[bt].Weights[i].Status = ml.Allocated + } else { + btDeviceMemory[bt].Weights[i].Status = ml.Failed + } + } + } + if b == nil { - return nil, fmt.Errorf("unable to allocate memory from device %v for model weights", C.GoString(C.ggml_backend_buft_name(bt))) + panic(ml.ErrNoMem{BackendMemory: requiredMemory}) } C.ggml_backend_buffer_set_usage(b, C.GGML_BACKEND_BUFFER_USAGE_WEIGHTS) @@ -367,7 +405,9 @@ func New(modelPath string, params ml.BackendParams) (ml.Backend, error) { } return m }(), - maxGraphNodes: maxGraphNodes, + requiredMemory: &requiredMemory, + btDeviceMemory: btDeviceMemory, + maxGraphNodes: maxGraphNodes, }, nil } @@ -446,6 +486,10 @@ func (b *Backend) Load(ctx context.Context, progress func(float32)) error { return nil } +func (b *Backend) BackendMemory() ml.BackendMemory { + return *b.requiredMemory +} + func (b *Backend) Config() fs.Config { return b.meta.KV() } @@ -477,6 +521,7 @@ func (b *Backend) NewContextSize(n int) ml.Context { no_alloc: true, }), allocatedBuffers: &allocatedBuffers, + layer: -1, } } @@ -503,6 +548,9 @@ type Context struct { // maxGraphNodes is the maximum allowed number of graph nodes in this context maxGraphNodes int + + // layer is the graph layer that this context is allocating for - assumed to be cache + layer int } func (c *Context) Input() ml.Context { @@ -513,6 +561,7 @@ func (c *Context) Input() ml.Context { buft: c.b.input, allocatedBuffers: c.allocatedBuffers, maxGraphNodes: c.maxGraphNodes, + layer: -1, } } @@ -527,6 +576,7 @@ func (c *Context) Layer(i int) ml.Context { buft: buft, allocatedBuffers: c.allocatedBuffers, maxGraphNodes: c.maxGraphNodes, + layer: i, } } @@ -564,22 +614,34 @@ func (c *Context) Compute(tensors ...ml.Tensor) { } } -func (c *Context) Reserve() error { - if !C.ggml_backend_sched_reserve(c.b.sched, c.graph) { - C.ggml_backend_sched_reset(c.b.sched) - return errors.New("failed to reserve graph") - } +func (c *Context) Reserve() { + reserved := C.ggml_backend_sched_reserve(c.b.sched, c.graph) slog.Debug("compute graph", "nodes", C.ggml_graph_n_nodes(c.graph), "splits", C.ggml_backend_sched_get_n_splits(c.b.sched)) - for i := range c.b.schedBackends { - size := C.ggml_backend_sched_get_buffer_size(c.b.sched, c.b.schedBackends[i]) - slog.Info("compute graph", "backend", C.GoString(C.ggml_backend_name(c.b.schedBackends[i])), "buffer_type", C.GoString(C.ggml_backend_buft_name(c.b.schedBufts[i])), - "size", format.HumanBytes2(uint64(size))) + + // Reserve may get called multiple times for different graphs - we just want the last run, which will contain the max allocations + for _, bt := range c.b.schedBufts { + c.b.btDeviceMemory[bt].Graph = ml.Memory{} } - C.ggml_backend_sched_reset(c.b.sched) + for i := range c.b.schedBackends { + bufferStatus := C.ggml_backend_sched_get_attempted_buffer_size(c.b.sched, c.b.schedBackends[i]) - return nil + graph := &c.b.btDeviceMemory[c.b.schedBufts[i]].Graph + graph.Size += uint64(bufferStatus.size) + if bufferStatus.allocated && graph.Status != ml.Failed { + graph.Status = ml.Allocated + } else { + graph.Status = ml.Failed + } + + slog.Info("compute graph", "backend", C.GoString(C.ggml_backend_name(c.b.schedBackends[i])), "buffer_type", C.GoString(C.ggml_backend_buft_name(c.b.schedBufts[i])), + "size", format.HumanBytes2(uint64(bufferStatus.size))) + } + + if !reserved { + panic(ml.ErrNoMem{BackendMemory: *c.b.requiredMemory}) + } } func (c *Context) MaxGraphNodes() int { @@ -599,7 +661,7 @@ func pad(length, pad C.size_t) C.size_t { return ((length + pad - 1) / pad) * pad } -func (c *Context) newTensor(dtype ml.DType, shape []int) (ml.Tensor, error) { +func (c *Context) newTensor(dtype ml.DType, shape []int) ml.Tensor { if c.buft == nil { panic("set Input or Layer before creating tensors") } @@ -622,7 +684,7 @@ func (c *Context) newTensor(dtype ml.DType, shape []int) (ml.Tensor, error) { if len(shape) < 1 || shape[0] == 0 { var shape C.int64_t = 0 - return &Tensor{b: c.b, t: C.ggml_new_tensor(c.ctx, cdtype, 1, &shape)}, nil + return &Tensor{b: c.b, t: C.ggml_new_tensor(c.ctx, cdtype, 1, &shape)} } else if len(shape) > 4 { panic("unsupported number of dimensions") } @@ -635,31 +697,34 @@ func (c *Context) newTensor(dtype ml.DType, shape []int) (ml.Tensor, error) { t := C.ggml_new_tensor(c.ctx, cdtype, C.int(len(shape)), shapeToGGML(shape)) size := pad(C.ggml_backend_buft_get_alloc_size(c.buft, t), C.ggml_backend_buft_get_alignment(c.buft)) - b := C.ggml_backend_buft_alloc_buffer(c.buft, size) - if b == nil { - return nil, fmt.Errorf("unable to allocate %v from device %v for new tensor", format.HumanBytes2(uint64(size)), C.GoString(C.ggml_backend_buft_name(c.buft))) - } - *c.allocatedBuffers = append(*c.allocatedBuffers, b) + b := C.ggml_backend_buft_alloc_buffer(c.buft, size) + if c.layer >= 0 { + cache := &c.b.btDeviceMemory[c.buft].Cache[c.layer] + + cache.Size += uint64(size) + if b != nil { + cache.Status = ml.Allocated + } else { + cache.Status = ml.Failed + } + } + + if b == nil { + panic(ml.ErrNoMem{BackendMemory: *c.b.requiredMemory}) + } + + *c.allocatedBuffers = append(*c.allocatedBuffers, b) C.ggml_backend_tensor_alloc(b, t, C.ggml_backend_buffer_get_base(b)) - return &Tensor{b: c.b, t: t}, nil + return &Tensor{b: c.b, t: t} } func (c *Context) Empty(dtype ml.DType, shape ...int) ml.Tensor { - t, err := c.newTensor(dtype, shape) - if err != nil { - panic(err) - } - - return t + return c.newTensor(dtype, shape) } func (c *Context) Zeros(dtype ml.DType, shape ...int) ml.Tensor { - t, err := c.newTensor(dtype, shape) - if err != nil { - panic(err) - } - + t := c.newTensor(dtype, shape) C.ggml_set_zero(t.(*Tensor).t) return t } @@ -687,10 +752,7 @@ func (c *Context) FromFloatSlice(s []float32, shape ...int) (ml.Tensor, error) { return nil, err } - t, err := c.newTensor(ml.DTypeF32, shape) - if err != nil { - return nil, err - } + t := c.newTensor(ml.DTypeF32, shape) if len(s) > 0 { C.ggml_backend_tensor_set(t.(*Tensor).t, unsafe.Pointer(&s[0]), 0, C.ggml_nbytes(t.(*Tensor).t)) @@ -704,10 +766,7 @@ func (c *Context) FromIntSlice(s []int32, shape ...int) (ml.Tensor, error) { return nil, err } - t, err := c.newTensor(ml.DTypeI32, shape) - if err != nil { - return nil, err - } + t := c.newTensor(ml.DTypeI32, shape) if len(s) > 0 { C.ggml_backend_tensor_set(t.(*Tensor).t, unsafe.Pointer(&s[0]), 0, C.ggml_nbytes(t.(*Tensor).t)) diff --git a/runner/ollamarunner/multimodal.go b/runner/ollamarunner/multimodal.go index d78612fee..dbe6bba10 100644 --- a/runner/ollamarunner/multimodal.go +++ b/runner/ollamarunner/multimodal.go @@ -95,10 +95,7 @@ func (m multimodalStore) getTensor(backend ml.Backend, ctx ml.Context, in ml.Ten } } } else { - err := computeCtx.Reserve() - if err != nil { - return nil, err - } + computeCtx.Reserve() } } diff --git a/runner/ollamarunner/runner.go b/runner/ollamarunner/runner.go index a488a104b..99bee1061 100644 --- a/runner/ollamarunner/runner.go +++ b/runner/ollamarunner/runner.go @@ -826,16 +826,12 @@ func (s *Server) reserveWorstCaseGraph() error { return err } - err = ctx.Forward(t).Reserve() - if err != nil { - return err - } + ctx.Forward(t).Reserve() return nil } -func (s *Server) loadModel( - ctx context.Context, +func (s *Server) initModel( mpath string, params ml.BackendParams, lpath multiLPath, @@ -843,21 +839,21 @@ func (s *Server) loadModel( kvCacheType string, kvSize int, multiUserCache bool, -) { +) error { var err error s.model, err = model.New(mpath, params) if err != nil { - panic(err) + return err } // TODO(jessegross): LoRA loading if lpath.String() != "" { - panic("loras are not yet implemented") + return errors.New("loras are not yet implemented") } s.cache, err = NewInputCache(s.model, kvCacheType, int32(kvSize), parallel, s.batchSize, multiUserCache) if err != nil { - panic(err) + return err } if !s.cache.enabled && parallel > 1 { @@ -869,11 +865,25 @@ func (s *Server) loadModel( s.seqs = make([]*Sequence, s.parallel) s.seqsSem = semaphore.NewWeighted(int64(s.parallel)) - err = s.reserveWorstCaseGraph() + return s.reserveWorstCaseGraph() +} + +func (s *Server) load( + ctx context.Context, + mpath string, + params ml.BackendParams, + lpath multiLPath, + parallel int, + kvCacheType string, + kvSize int, + multiUserCache bool) { + err := s.initModel(mpath, params, lpath, parallel, kvCacheType, kvSize, multiUserCache) if err != nil { panic(err) } + slog.Debug("memory", "allocated", s.model.Backend().BackendMemory()) + err = s.model.Backend().Load(ctx, func(progress float32) { s.progress = progress @@ -921,9 +931,14 @@ func Execute(args []string) error { status: llm.ServerStatusLoadingModel, } + server.cond = sync.NewCond(&server.mu) + server.ready.Add(1) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + // TODO(jessegross): Parameters that need to be implemented: // no-mmap - // mlock var tensorSplitFloats []float32 if *tensorSplit != "" { @@ -943,14 +958,7 @@ func Execute(args []string) error { FlashAttention: *flashAttention, } - server.ready.Add(1) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - go server.loadModel(ctx, *mpath, params, lpaths, *parallel, *kvCacheType, *kvSize, *multiUserCache) - - server.cond = sync.NewCond(&server.mu) - + go server.load(ctx, *mpath, params, lpaths, *parallel, *kvCacheType, *kvSize, *multiUserCache) go server.run(ctx) addr := "127.0.0.1:" + strconv.Itoa(*port) From 1f371ea92f7ebe4edd208b6732753473b2c4d0cd Mon Sep 17 00:00:00 2001 From: Jesse Gross Date: Mon, 19 May 2025 10:43:56 -0700 Subject: [PATCH 07/54] ml: Panic rather than return error on tensor allocation failure FromFloatSlice and FromIntSlice return an error if the shape doesn't match the passed data or if memory can't be allocated. Since these are inputs, the memory being allocated is system memory rather than VRAM. In many cases, the caller can't really handle the error and panics. Empty and Zeros directly panic if they can't allocate memory. This makes things consistent by panicing for the first two cases, removing a fair amount of error handling code. This is also consistent with how Go typically handles these situations. --- kvcache/causal.go | 26 ++++++---------------- kvcache/causal_test.go | 18 ++++++++-------- ml/backend.go | 4 ++-- ml/backend/ggml/ggml.go | 31 +++++++++------------------ model/model.go | 6 +----- model/models/gemma2/model.go | 11 ++-------- model/models/gemma3/model.go | 16 +++----------- model/models/llama/model.go | 10 ++------- model/models/llama4/model.go | 22 ++++--------------- model/models/llama4/model_text.go | 6 +----- model/models/llama4/model_vision.go | 5 +---- model/models/mistral3/model.go | 16 +++----------- model/models/mistral3/model_vision.go | 16 +++----------- model/models/mllama/model.go | 22 ++++--------------- model/models/qwen2/model.go | 10 ++------- model/models/qwen25vl/model.go | 16 +++----------- model/models/qwen25vl/model_vision.go | 22 +++++-------------- model/models/qwen3/model.go | 10 ++------- runner/ollamarunner/multimodal.go | 2 +- runner/ollamarunner/runner.go | 8 +++---- 20 files changed, 68 insertions(+), 209 deletions(-) diff --git a/kvcache/causal.go b/kvcache/causal.go index 9bc1d5da2..f6bacaaf8 100644 --- a/kvcache/causal.go +++ b/kvcache/causal.go @@ -211,10 +211,9 @@ func (c *Causal) StartForward(ctx ml.Context, batch input.Batch, reserve bool) e c.curCellRange.max = len(c.cells) - 1 } - var err error - c.curMask, err = c.buildMask(ctx) + c.curMask = c.buildMask(ctx) - return err + return nil } func newRange() cellRange { @@ -297,7 +296,7 @@ func roundUp(length, pad int) int { // Builds a mask of history x batch indicating whether for each token in the batch the // token in the history should apply. This is based on both the sequence and causality (the // position of the history is not ahead of the token in the batch). -func (c *Causal) buildMask(ctx ml.Context) (ml.Tensor, error) { +func (c *Causal) buildMask(ctx ml.Context) ml.Tensor { // Align and pad the two dimensions as required by the backend batchSize := roundUp(c.curBatchSize, c.config.MaskBatchPadding) @@ -325,10 +324,7 @@ func (c *Causal) buildMask(ctx ml.Context) (ml.Tensor, error) { mask[i] = float32(math.Inf(-1)) } - maskTensor, err := ctx.Input().FromFloatSlice(mask, length, batchSize) - if err != nil { - return nil, err - } + maskTensor := ctx.Input().FromFloatSlice(mask, length, batchSize) if c.config.MaskDType != ml.DTypeF32 { out := ctx.Input().Empty(c.config.MaskDType, maskTensor.Shape()...) @@ -336,7 +332,7 @@ func (c *Causal) buildMask(ctx ml.Context) (ml.Tensor, error) { maskTensor = out } - return maskTensor, nil + return maskTensor } func (c *Causal) moveCells(ctx ml.Context, src, dst, length int) { @@ -491,12 +487,7 @@ func (c *Causal) SetCausal(ctx ml.Context, opts CausalOptions) { if !slices.Equal(c.opts.Except, opts.Except) { c.opts = opts if ctx != nil { - var err error - c.curMask, err = c.buildMask(ctx) - if err != nil { - // This error should never occur because we have previously built a mask with the same shape - panic(fmt.Errorf("SetCausal: %w", err)) - } + c.curMask = c.buildMask(ctx) } } } @@ -652,10 +643,7 @@ func (c *Causal) shift(seq int, beginIndex, offset int32) error { } } - kShift, err := ctx.Input().FromIntSlice(offsets, len(offsets)) - if err != nil { - return err - } + kShift := ctx.Input().FromIntSlice(offsets, len(offsets)) for i, key := range c.keys { if key == nil { diff --git a/kvcache/causal_test.go b/kvcache/causal_test.go index 820d496d1..5b1dbe868 100644 --- a/kvcache/causal_test.go +++ b/kvcache/causal_test.go @@ -344,7 +344,7 @@ func testCache(t *testing.T, backend ml.Backend, cache Cache, tests []testCase) } cache.SetLayer(0) - tensor, _ := context.FromFloatSlice(test.in, test.inShape...) + tensor := context.FromFloatSlice(test.in, test.inShape...) cache.Put(context, tensor, tensor) out, _, mask := cache.Get(context) @@ -386,7 +386,7 @@ func TestCanResume(t *testing.T) { } cache.SetLayer(0) - tensor, _ := context.FromFloatSlice([]float32{1, 2, 3, 4}, 1, 1, 4) + tensor := context.FromFloatSlice([]float32{1, 2, 3, 4}, 1, 1, 4) cache.Put(context, tensor, tensor) // with window size 4, nothing has slid out of the window yet @@ -413,7 +413,7 @@ func TestCanResume(t *testing.T) { } cache.SetLayer(0) - tensor, _ = context.FromFloatSlice([]float32{5, 6}, 1, 1, 2) + tensor = context.FromFloatSlice([]float32{5, 6}, 1, 1, 2) cache.Put(context, tensor, tensor) // only the latest position has overlapping windows @@ -470,24 +470,24 @@ func (c *testContext) Zeros(dtype ml.DType, shape ...int) ml.Tensor { return c.Empty(dtype, shape...) } -func (c *testContext) FromFloatSlice(s []float32, shape ...int) (ml.Tensor, error) { +func (c *testContext) FromFloatSlice(s []float32, shape ...int) ml.Tensor { t := c.Empty(ml.DTypeF32, shape...).(*testTensor) copy(t.data, s) - return t, nil + return t } -func (c *testContext) FromIntSlice(s []int32, shape ...int) (ml.Tensor, error) { +func (c *testContext) FromIntSlice(s []int32, shape ...int) ml.Tensor { f := make([]float32, len(s)) for i := range f { f[i] = float32(s[i]) } - out, _ := c.FromFloatSlice(f, shape...) + out := c.FromFloatSlice(f, shape...) out.(*testTensor).dtype = ml.DTypeI32 - return out, nil + return out } func (c *testContext) Arange(start, stop, step float32, dtype ml.DType) ml.Tensor { @@ -496,7 +496,7 @@ func (c *testContext) Arange(start, stop, step float32, dtype ml.DType) ml.Tenso s = append(s, i) } - out, _ := c.FromFloatSlice(s, len(s)) + out := c.FromFloatSlice(s, len(s)) out.(*testTensor).dtype = dtype return out } diff --git a/ml/backend.go b/ml/backend.go index 7c9b9e313..6beb7d2bc 100644 --- a/ml/backend.go +++ b/ml/backend.go @@ -171,8 +171,8 @@ func NewBackend(modelPath string, params BackendParams) (Backend, error) { type Context interface { Empty(dtype DType, shape ...int) Tensor Zeros(dtype DType, shape ...int) Tensor - FromFloatSlice(s []float32, shape ...int) (Tensor, error) - FromIntSlice(s []int32, shape ...int) (Tensor, error) + FromFloatSlice(s []float32, shape ...int) Tensor + FromIntSlice(s []int32, shape ...int) Tensor // Arange creates a 1D tensor with values within an interval (start, stop] increased by step. Arange(start, stop, step float32, dtype DType) Tensor diff --git a/ml/backend/ggml/ggml.go b/ml/backend/ggml/ggml.go index 496ba8a60..76172ae1a 100644 --- a/ml/backend/ggml/ggml.go +++ b/ml/backend/ggml/ggml.go @@ -729,11 +729,11 @@ func (c *Context) Zeros(dtype ml.DType, shape ...int) ml.Tensor { return t } -func checkShape[S ~[]E, E any](s S, shape ...int) error { +func checkShape[S ~[]E, E any](s S, shape ...int) { n := len(s) if n == 0 { - return nil + return } for _, v := range shape { @@ -741,16 +741,12 @@ func checkShape[S ~[]E, E any](s S, shape ...int) error { } if n != 1 { - return fmt.Errorf("invalid shape: %v", shape) + panic(fmt.Errorf("invalid shape: %v", shape)) } - - return nil } -func (c *Context) FromFloatSlice(s []float32, shape ...int) (ml.Tensor, error) { - if err := checkShape(s, shape...); err != nil { - return nil, err - } +func (c *Context) FromFloatSlice(s []float32, shape ...int) ml.Tensor { + checkShape(s, shape...) t := c.newTensor(ml.DTypeF32, shape) @@ -758,13 +754,11 @@ func (c *Context) FromFloatSlice(s []float32, shape ...int) (ml.Tensor, error) { C.ggml_backend_tensor_set(t.(*Tensor).t, unsafe.Pointer(&s[0]), 0, C.ggml_nbytes(t.(*Tensor).t)) } - return t, nil + return t } -func (c *Context) FromIntSlice(s []int32, shape ...int) (ml.Tensor, error) { - if err := checkShape(s, shape...); err != nil { - return nil, err - } +func (c *Context) FromIntSlice(s []int32, shape ...int) ml.Tensor { + checkShape(s, shape...) t := c.newTensor(ml.DTypeI32, shape) @@ -772,7 +766,7 @@ func (c *Context) FromIntSlice(s []int32, shape ...int) (ml.Tensor, error) { C.ggml_backend_tensor_set(t.(*Tensor).t, unsafe.Pointer(&s[0]), 0, C.ggml_nbytes(t.(*Tensor).t)) } - return t, nil + return t } func (c Context) Arange(start, stop, step float32, dtype ml.DType) ml.Tensor { @@ -790,12 +784,7 @@ func (c Context) Arange(start, stop, step float32, dtype ml.DType) ml.Tensor { arange = append(arange, int32(i)) } - t, err := c.Input().FromIntSlice(arange, len(arange)) - if err != nil { - panic(err) - } - - return t + return c.Input().FromIntSlice(arange, len(arange)) default: panic("unsupported dtype for arange") } diff --git a/model/model.go b/model/model.go index 39b68db15..25097e010 100644 --- a/model/model.go +++ b/model/model.go @@ -287,11 +287,7 @@ func Forward(ctx ml.Context, m Model, inputs []int32, batch input.Batch) (ml.Ten return nil, errors.New("batch size cannot be less than 1") } - var err error - batch.Inputs, err = ctx.Input().FromIntSlice(inputs, len(inputs)) - if err != nil { - return nil, err - } + batch.Inputs = ctx.Input().FromIntSlice(inputs, len(inputs)) cache := m.Config().Cache if cache != nil { diff --git a/model/models/gemma2/model.go b/model/models/gemma2/model.go index 3c5a7ea5b..e621d03ae 100644 --- a/model/models/gemma2/model.go +++ b/model/models/gemma2/model.go @@ -175,15 +175,8 @@ func (l *Layer) Forward(ctx ml.Context, hiddenState, positionIDs, outputs ml.Ten } func (m *Model) Forward(ctx ml.Context, batch input.Batch) (ml.Tensor, error) { - positions, err := ctx.Input().FromIntSlice(batch.Positions, len(batch.Positions)) - if err != nil { - return nil, err - } - - outputs, err := ctx.Input().FromIntSlice(batch.Outputs, len(batch.Outputs)) - if err != nil { - return nil, err - } + positions := ctx.Input().FromIntSlice(batch.Positions, len(batch.Positions)) + outputs := ctx.Input().FromIntSlice(batch.Outputs, len(batch.Outputs)) hiddenState := m.TokenEmbedding.Forward(ctx, batch.Inputs) hiddenState = hiddenState.Scale(ctx, math.Sqrt(float64(m.Options.hiddenSize))) diff --git a/model/models/gemma3/model.go b/model/models/gemma3/model.go index 89d1788ef..53bf82758 100644 --- a/model/models/gemma3/model.go +++ b/model/models/gemma3/model.go @@ -101,14 +101,11 @@ func (m *Model) EncodeMultimodal(ctx ml.Context, multimodalData []byte) ([]input return nil, err } - pixelValues, err := ctx.Input().FromFloatSlice(f32s, + pixelValues := ctx.Input().FromFloatSlice(f32s, m.ImageProcessor.imageSize, m.ImageProcessor.imageSize, m.ImageProcessor.numChannels, ) - if err != nil { - return nil, err - } visionOutputs := m.VisionModel.Forward(ctx, pixelValues) visionOutputs = m.MultiModalProjector.Forward(ctx, visionOutputs, m.imageSize, m.patchSize, m.VisionModel.eps) @@ -144,15 +141,8 @@ func (m *Model) PostTokenize(inputs []input.Input) ([]input.Input, error) { } func (m *Model) Forward(ctx ml.Context, batch input.Batch) (ml.Tensor, error) { - positions, err := ctx.Input().FromIntSlice(batch.Positions, len(batch.Positions)) - if err != nil { - return nil, err - } - - outputs, err := ctx.Input().FromIntSlice(batch.Outputs, len(batch.Outputs)) - if err != nil { - return nil, err - } + positions := ctx.Input().FromIntSlice(batch.Positions, len(batch.Positions)) + outputs := ctx.Input().FromIntSlice(batch.Outputs, len(batch.Outputs)) return m.TextModel.Forward(ctx, batch.Inputs, positions, outputs, batch, m.Cache), nil } diff --git a/model/models/llama/model.go b/model/models/llama/model.go index 507f1ebc2..3cf782d00 100644 --- a/model/models/llama/model.go +++ b/model/models/llama/model.go @@ -142,10 +142,7 @@ func (l *Layer) Forward(ctx ml.Context, hiddenState, positions, outputs ml.Tenso } func (m *Model) Forward(ctx ml.Context, batch input.Batch) (ml.Tensor, error) { - positions, err := ctx.Input().FromIntSlice(batch.Positions, len(batch.Positions)) - if err != nil { - return nil, err - } + positions := ctx.Input().FromIntSlice(batch.Positions, len(batch.Positions)) hiddenState := m.TokenEmbedding.Forward(ctx, batch.Inputs) @@ -154,10 +151,7 @@ func (m *Model) Forward(ctx ml.Context, batch input.Batch) (ml.Tensor, error) { var outputs ml.Tensor if i == len(m.Layers)-1 { - outputs, err = ctx.Input().FromIntSlice(batch.Outputs, len(batch.Outputs)) - if err != nil { - return nil, err - } + outputs = ctx.Input().FromIntSlice(batch.Outputs, len(batch.Outputs)) } hiddenState = layer.Forward(ctx, hiddenState, positions, outputs, m.Cache, m.Options) diff --git a/model/models/llama4/model.go b/model/models/llama4/model.go index af5173a16..8084760b0 100644 --- a/model/models/llama4/model.go +++ b/model/models/llama4/model.go @@ -77,10 +77,7 @@ func (m *Model) EncodeMultimodal(ctx ml.Context, multimodalData []byte) ([]input return nil, err } - tilesLocal, err := ctx.Input().FromFloatSlice(pixelsLocal, size.X, size.Y, m.numChannels) - if err != nil { - return nil, err - } + tilesLocal := ctx.Input().FromFloatSlice(pixelsLocal, size.X, size.Y, m.numChannels) ratioW, ratioH := size.X/m.imageSize, size.Y/m.imageSize @@ -91,11 +88,7 @@ func (m *Model) EncodeMultimodal(ctx ml.Context, multimodalData []byte) ([]input pixelValues := tilesLocal if len(pixelsGlobal) > 0 { - tilesGlobal, err := ctx.Input().FromFloatSlice(pixelsGlobal, m.imageSize, m.imageSize, m.numChannels) - if err != nil { - return nil, err - } - + tilesGlobal := ctx.Input().FromFloatSlice(pixelsGlobal, m.imageSize, m.imageSize, m.numChannels) pixelValues = pixelValues.Concat(ctx, tilesGlobal, 3) } @@ -182,15 +175,8 @@ func (m *Model) PostTokenize(inputs []input.Input) ([]input.Input, error) { } func (m *Model) Forward(ctx ml.Context, batch input.Batch) (ml.Tensor, error) { - positions, err := ctx.Input().FromIntSlice(batch.Positions, len(batch.Positions)) - if err != nil { - return nil, err - } - - outputs, err := ctx.Input().FromIntSlice(batch.Outputs, len(batch.Outputs)) - if err != nil { - return nil, err - } + positions := ctx.Input().FromIntSlice(batch.Positions, len(batch.Positions)) + outputs := ctx.Input().FromIntSlice(batch.Outputs, len(batch.Outputs)) return m.TextModel.Forward(ctx, batch.Inputs, positions, outputs, batch, m.Cache), nil } diff --git a/model/models/llama4/model_text.go b/model/models/llama4/model_text.go index ff9f5e20d..27935f401 100644 --- a/model/models/llama4/model_text.go +++ b/model/models/llama4/model_text.go @@ -223,11 +223,7 @@ func (m *TextModel) Forward(ctx ml.Context, inputs, positions, outputs ml.Tensor scales[i] = float32(math.Log(math.Floor(((float64(p)+1.0)/float64(m.attentionFloorScale))+1.0))*m.attentionScale + 1.0) } - var err error - attentionScales, err = ctx.Input().FromFloatSlice(scales, 1, 1, len(scales)) - if err != nil { - panic(err) - } + attentionScales = ctx.Input().FromFloatSlice(scales, 1, 1, len(scales)) } for i, layer := range m.Layers { diff --git a/model/models/llama4/model_vision.go b/model/models/llama4/model_vision.go index e6b1afef6..dc6f82b84 100644 --- a/model/models/llama4/model_vision.go +++ b/model/models/llama4/model_vision.go @@ -245,10 +245,7 @@ func (m *VisionModel) rotaryEmbedding(ctx ml.Context) (ml.Tensor, ml.Tensor) { } } - ropeFreqs, err := ctx.Input().FromFloatSlice(freqs, freqDim/2, numPatches, 2) - if err != nil { - panic(err) - } + ropeFreqs := ctx.Input().FromFloatSlice(freqs, freqDim/2, numPatches, 2) ropeFreqs = ropeFreqs.Permute(ctx, 0, 2, 1, 3).Contiguous(ctx) ropeFreqs = ropeFreqs.Reshape(ctx, freqDim, 1, numPatches) diff --git a/model/models/mistral3/model.go b/model/models/mistral3/model.go index dd01a587b..9d662fc11 100644 --- a/model/models/mistral3/model.go +++ b/model/models/mistral3/model.go @@ -114,10 +114,7 @@ func (m *Model) EncodeMultimodal(ctx ml.Context, multimodalData []byte) ([]input return nil, err } - pixelValues, err := ctx.Input().FromFloatSlice(f32s, size.X, size.Y, m.ImageProcessor.numChannels) - if err != nil { - return nil, err - } + pixelValues := ctx.Input().FromFloatSlice(f32s, size.X, size.Y, m.ImageProcessor.numChannels) visionOutputs := m.VisionModel.Forward(ctx, pixelValues) features, size := m.MultiModalProjector.Forward(ctx, visionOutputs, size) @@ -161,15 +158,8 @@ func (m *Model) PostTokenize(inputs []input.Input) ([]input.Input, error) { } func (m *Model) Forward(ctx ml.Context, batch input.Batch) (ml.Tensor, error) { - positions, err := ctx.Input().FromIntSlice(batch.Positions, len(batch.Positions)) - if err != nil { - return nil, err - } - - outputs, err := ctx.Input().FromIntSlice(batch.Outputs, len(batch.Outputs)) - if err != nil { - return nil, err - } + positions := ctx.Input().FromIntSlice(batch.Positions, len(batch.Positions)) + outputs := ctx.Input().FromIntSlice(batch.Outputs, len(batch.Outputs)) return m.TextModel.Forward(ctx, batch.Inputs, positions, outputs, batch, m.Cache), nil } diff --git a/model/models/mistral3/model_vision.go b/model/models/mistral3/model_vision.go index 245410046..65bdcff2a 100644 --- a/model/models/mistral3/model_vision.go +++ b/model/models/mistral3/model_vision.go @@ -110,15 +110,8 @@ func (m *VisionModel) positionalEmbedding(ctx ml.Context, positionIDs ml.Tensor) } } - h, err := ctx.Input().FromFloatSlice(frequenciesHeight, maxPatchesPerSide, frequencies/2) - if err != nil { - panic(err) - } - - w, err := ctx.Input().FromFloatSlice(frequenciesWidth, maxPatchesPerSide, frequencies/2) - if err != nil { - panic(err) - } + h := ctx.Input().FromFloatSlice(frequenciesHeight, maxPatchesPerSide, frequencies/2) + w := ctx.Input().FromFloatSlice(frequenciesWidth, maxPatchesPerSide, frequencies/2) h = h.Permute(ctx, 1, 0, 2, 3).Contiguous(ctx) w = w.Permute(ctx, 1, 0, 2, 3).Contiguous(ctx) @@ -151,10 +144,7 @@ func (m *VisionModel) Forward(ctx ml.Context, pixelValues ml.Tensor) ml.Tensor { } } - positionIDs, err := ctx.Input().FromIntSlice(positions, len(positions)) - if err != nil { - panic(err) - } + positionIDs := ctx.Input().FromIntSlice(positions, len(positions)) positionEmbedding := m.positionalEmbedding(ctx, positionIDs) cos, sin := positionEmbedding.Cos(ctx), positionEmbedding.Sin(ctx) diff --git a/model/models/mllama/model.go b/model/models/mllama/model.go index 547e2cb32..45cb3e02c 100644 --- a/model/models/mllama/model.go +++ b/model/models/mllama/model.go @@ -80,15 +80,8 @@ func (m *Model) EncodeMultimodal(ctx ml.Context, multimodalData []byte) ([]input f32s = f32s[:m.imageSize*m.imageSize*m.numChannels*m.maxNumTiles] } - pixelValues, err := ctx.Input().FromFloatSlice(f32s, m.imageSize, m.imageSize, m.numChannels, m.maxNumTiles) - if err != nil { - return nil, err - } - - aspectRatio, err := ctx.Input().FromIntSlice([]int32{int32(ratio.rank)}, 1) - if err != nil { - return nil, err - } + pixelValues := ctx.Input().FromFloatSlice(f32s, m.imageSize, m.imageSize, m.numChannels, m.maxNumTiles) + aspectRatio := ctx.Input().FromIntSlice([]int32{int32(ratio.rank)}, 1) positionIDs := ctx.Arange(0, 1601, 1, ml.DTypeI32) crossAttentionStates := m.VisionModel.Forward(ctx, pixelValues, positionIDs, aspectRatio) @@ -113,15 +106,8 @@ func (m *Model) Forward(ctx ml.Context, batch input.Batch) (ml.Tensor, error) { crossAttentionStates = batch.Multimodal[len(batch.Multimodal)-1].Multimodal[0].Tensor } - positions, err := ctx.Input().FromIntSlice(batch.Positions, len(batch.Positions)) - if err != nil { - return nil, err - } - - outputs, err := ctx.Input().FromIntSlice(batch.Outputs, len(batch.Outputs)) - if err != nil { - return nil, err - } + positions := ctx.Input().FromIntSlice(batch.Positions, len(batch.Positions)) + outputs := ctx.Input().FromIntSlice(batch.Outputs, len(batch.Outputs)) // TODO: attention mask, cross attention mask return m.TextModel.Forward(ctx, batch.Inputs, positions, outputs, crossAttentionStates, nil, m.Cache.(*kvcache.WrapperCache)), nil diff --git a/model/models/qwen2/model.go b/model/models/qwen2/model.go index 3c3d81aa5..42338d0d6 100644 --- a/model/models/qwen2/model.go +++ b/model/models/qwen2/model.go @@ -100,10 +100,7 @@ type Model struct { // Forward implements model.Model. func (m Model) Forward(ctx ml.Context, batch input.Batch) (ml.Tensor, error) { - positions, err := ctx.Input().FromIntSlice(batch.Positions, len(batch.Positions)) - if err != nil { - return nil, err - } + positions := ctx.Input().FromIntSlice(batch.Positions, len(batch.Positions)) hiddenStates := m.TokenEmbedding.Forward(ctx, batch.Inputs) @@ -112,10 +109,7 @@ func (m Model) Forward(ctx ml.Context, batch input.Batch) (ml.Tensor, error) { var outputs ml.Tensor if i == len(m.Layers)-1 { - outputs, err = ctx.Input().FromIntSlice(batch.Outputs, len(batch.Outputs)) - if err != nil { - return nil, err - } + outputs = ctx.Input().FromIntSlice(batch.Outputs, len(batch.Outputs)) } hiddenStates = layer.Forward(ctx, hiddenStates, positions, outputs, m.Cache, &m.Options) diff --git a/model/models/qwen25vl/model.go b/model/models/qwen25vl/model.go index 32cca5607..ee38cad92 100644 --- a/model/models/qwen25vl/model.go +++ b/model/models/qwen25vl/model.go @@ -69,10 +69,7 @@ func (m *Model) PixelValues(ctx ml.Context, multimodalData []byte) (ml.Tensor, * m.ImageProcessor.patchSize * m.ImageProcessor.patchSize numPatches := grid.Temporal * grid.Height * grid.Width - pixelValues, err := ctx.Input().FromFloatSlice(f32s, patchDim, numPatches) - if err != nil { - return nil, nil, fmt.Errorf("failed to create tensor from image: %w", err) - } + pixelValues := ctx.Input().FromFloatSlice(f32s, patchDim, numPatches) return pixelValues, grid, nil } @@ -142,15 +139,8 @@ func (m *Model) PostTokenize(inputs []input.Input) ([]input.Input, error) { } func (m *Model) Forward(ctx ml.Context, batch input.Batch) (ml.Tensor, error) { - positions, err := ctx.Input().FromIntSlice(batch.Positions, len(batch.Positions)) - if err != nil { - return nil, err - } - - outputs, err := ctx.Input().FromIntSlice(batch.Outputs, len(batch.Outputs)) - if err != nil { - return nil, err - } + positions := ctx.Input().FromIntSlice(batch.Positions, len(batch.Positions)) + outputs := ctx.Input().FromIntSlice(batch.Outputs, len(batch.Outputs)) return m.TextModel.Forward(ctx, batch.Inputs, positions, outputs, batch, m.Cache) } diff --git a/model/models/qwen25vl/model_vision.go b/model/models/qwen25vl/model_vision.go index 01eef392b..4d7afaa14 100644 --- a/model/models/qwen25vl/model_vision.go +++ b/model/models/qwen25vl/model_vision.go @@ -1,7 +1,6 @@ package qwen25vl import ( - "fmt" "math" "slices" @@ -44,10 +43,8 @@ func blockDiagonalMask(ctx ml.Context, seqLength int, bounds []int, numHeads int } } - mask, err := ctx.Input().FromFloatSlice(flat, seqLength, seqLength) - if err != nil { - panic(err) - } + mask := ctx.Input().FromFloatSlice(flat, seqLength, seqLength) + // Reshape to match [seqLength, seqLength, 1] for broadcasting mask = mask.Reshape(ctx, seqLength, seqLength, 1) @@ -303,10 +300,7 @@ func (m *VisionModel) WindowIndex(ctx ml.Context, grid *Grid) (ml.Tensor, []int) } } - t, err := ctx.Input().FromIntSlice(index, len(index)) - if err != nil { - panic(err) - } + t := ctx.Input().FromIntSlice(index, len(index)) return t, bounds } @@ -326,10 +320,7 @@ func (m *VisionModel) PositionalEmbedding(ctx ml.Context, grid *Grid) ml.Tensor freqVals[i*freq+j] = float32(i) / float32(math.Pow(theta, float64(j*2)/float64(dim))) } } - freqs, err := ctx.Input().FromFloatSlice(freqVals, freq, maxGridSize) - if err != nil { - panic(fmt.Errorf("failed to create tensor from frequencies: %w", err)) - } + freqs := ctx.Input().FromFloatSlice(freqVals, freq, maxGridSize) // Create position coordinates (y,x pairs) for the grid // In PyTorch: Equivalent to generating position ids with torch.arange() @@ -339,10 +330,7 @@ func (m *VisionModel) PositionalEmbedding(ctx ml.Context, grid *Grid) ml.Tensor coords = append(coords, int32(y), int32(x)) } } - pos, err := ctx.Input().FromIntSlice(coords, 2, grid.Width, grid.Height) - if err != nil { - panic(fmt.Errorf("failed to create tensor from positions: %w", err)) - } + pos := ctx.Input().FromIntSlice(coords, 2, grid.Width, grid.Height) // Reshape and permute positions to match spatial merging pattern pos = pos.Reshape(ctx, 2, grid.Width, merge, grid.Height/merge) diff --git a/model/models/qwen3/model.go b/model/models/qwen3/model.go index 44c32f9e3..1930da7e2 100644 --- a/model/models/qwen3/model.go +++ b/model/models/qwen3/model.go @@ -156,10 +156,7 @@ type Model struct { // Forward implements model.Model. func (m *Model) Forward(ctx ml.Context, batch input.Batch) (ml.Tensor, error) { - positions, err := ctx.Input().FromIntSlice(batch.Positions, len(batch.Positions)) - if err != nil { - return nil, err - } + positions := ctx.Input().FromIntSlice(batch.Positions, len(batch.Positions)) hiddenStates := m.TokenEmbedding.Forward(ctx, batch.Inputs) @@ -168,10 +165,7 @@ func (m *Model) Forward(ctx ml.Context, batch input.Batch) (ml.Tensor, error) { var outputs ml.Tensor if i == len(m.Layers)-1 { - outputs, err = ctx.Input().FromIntSlice(batch.Outputs, len(batch.Outputs)) - if err != nil { - return nil, err - } + outputs = ctx.Input().FromIntSlice(batch.Outputs, len(batch.Outputs)) } hiddenStates = layer.Forward(ctx, hiddenStates, positions, outputs, m.Cache, m.Options) diff --git a/runner/ollamarunner/multimodal.go b/runner/ollamarunner/multimodal.go index dbe6bba10..fbdc7d72a 100644 --- a/runner/ollamarunner/multimodal.go +++ b/runner/ollamarunner/multimodal.go @@ -102,7 +102,7 @@ func (m multimodalStore) getTensor(backend ml.Backend, ctx ml.Context, in ml.Ten for i, t := range entry.mm { if in == t.Tensor { if !reserve { - return ctx.Input().FromFloatSlice(entry.data[i], t.Tensor.Shape()...) + return ctx.Input().FromFloatSlice(entry.data[i], t.Tensor.Shape()...), nil } else { return ctx.Input().Empty(t.Tensor.DType(), t.Tensor.Shape()...), nil } diff --git a/runner/ollamarunner/runner.go b/runner/ollamarunner/runner.go index 99bee1061..a7a889f1f 100644 --- a/runner/ollamarunner/runner.go +++ b/runner/ollamarunner/runner.go @@ -808,10 +808,7 @@ func (s *Server) reserveWorstCaseGraph() error { batch.Outputs[i] = int32(i) } - batch.Inputs, err = ctx.Input().FromIntSlice(batchInputs, len(batchInputs)) - if err != nil { - return err - } + batch.Inputs = ctx.Input().FromIntSlice(batchInputs, len(batchInputs)) cache := s.model.Config().Cache if cache != nil { @@ -876,7 +873,8 @@ func (s *Server) load( parallel int, kvCacheType string, kvSize int, - multiUserCache bool) { + multiUserCache bool, +) { err := s.initModel(mpath, params, lpath, parallel, kvCacheType, kvSize, multiUserCache) if err != nil { panic(err) From 884d26093c80491a3fe07f606fc04851dc317199 Mon Sep 17 00:00:00 2001 From: Parth Sareen Date: Thu, 22 May 2025 18:53:31 -0700 Subject: [PATCH 08/54] llama: add minimum memory for grammar (#10820) --- llama/llama.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama/llama.go b/llama/llama.go index adee6f63c..0dc64e57e 100644 --- a/llama/llama.go +++ b/llama/llama.go @@ -580,7 +580,7 @@ func SchemaToGrammar(schema []byte) []byte { defer C.free(unsafe.Pointer(cStr)) // Allocate buffer for grammar based on schema length but with upper bound - maxLen := min(1024*1024, len(schema)*4) + maxLen := max(32768, min(1024*1024, len(schema)*4)) buf := make([]byte, maxLen) // Call C function to convert schema to grammar From e8b981fa5d7c1875ec0c290068bcfe3b4662f5c4 Mon Sep 17 00:00:00 2001 From: Parth Sareen Date: Fri, 23 May 2025 14:19:31 -0700 Subject: [PATCH 09/54] tools: refactor tool call parsing and enable streaming (#10415) --- server/model.go | 124 ---- server/model_test.go | 179 ----- server/routes.go | 68 +- .../testdata}/command-r-plus.gotmpl | 0 .../testdata}/command-r-plus.out | 0 .../testdata}/firefunction.gotmpl | 0 .../tools => tools/testdata}/firefunction.out | 0 .../testdata}/llama3-groq-tool-use.gotmpl | 0 .../testdata}/llama3-groq-tool-use.out | 0 tools/testdata/llama3.2.gotmpl | 44 ++ tools/testdata/llama3.2.out | 24 + .../tools => tools/testdata}/messages.json | 0 .../tools => tools/testdata}/mistral.gotmpl | 0 .../tools => tools/testdata}/mistral.out | 0 .../tools => tools/testdata}/nemotron.gotmpl | 0 .../tools => tools/testdata}/nemotron.out | 0 tools/testdata/qwen2.5.gotmpl | 51 ++ tools/testdata/qwen2.5.out | 31 + tools/testdata/qwen3.gotmpl | 50 ++ tools/testdata/qwen3.out | 31 + .../tools => tools/testdata}/tools.json | 0 .../tools => tools/testdata}/xlam.gotmpl | 0 .../tools => tools/testdata}/xlam.out | 0 tools/tools.go | 271 ++++++++ tools/tools_test.go | 644 ++++++++++++++++++ tools/tools_utils.go | 227 ++++++ tools/tools_utils_test.go | 464 +++++++++++++ 27 files changed, 1868 insertions(+), 340 deletions(-) delete mode 100644 server/model_test.go rename {server/testdata/tools => tools/testdata}/command-r-plus.gotmpl (100%) rename {server/testdata/tools => tools/testdata}/command-r-plus.out (100%) rename {server/testdata/tools => tools/testdata}/firefunction.gotmpl (100%) rename {server/testdata/tools => tools/testdata}/firefunction.out (100%) rename {server/testdata/tools => tools/testdata}/llama3-groq-tool-use.gotmpl (100%) rename {server/testdata/tools => tools/testdata}/llama3-groq-tool-use.out (100%) create mode 100644 tools/testdata/llama3.2.gotmpl create mode 100644 tools/testdata/llama3.2.out rename {server/testdata/tools => tools/testdata}/messages.json (100%) rename {server/testdata/tools => tools/testdata}/mistral.gotmpl (100%) rename {server/testdata/tools => tools/testdata}/mistral.out (100%) rename {server/testdata/tools => tools/testdata}/nemotron.gotmpl (100%) rename {server/testdata/tools => tools/testdata}/nemotron.out (100%) create mode 100644 tools/testdata/qwen2.5.gotmpl create mode 100644 tools/testdata/qwen2.5.out create mode 100644 tools/testdata/qwen3.gotmpl create mode 100644 tools/testdata/qwen3.out rename {server/testdata/tools => tools/testdata}/tools.json (100%) rename {server/testdata/tools => tools/testdata}/xlam.gotmpl (100%) rename {server/testdata/tools => tools/testdata}/xlam.out (100%) create mode 100644 tools/tools.go create mode 100644 tools/tools_test.go create mode 100644 tools/tools_utils.go create mode 100644 tools/tools_utils_test.go diff --git a/server/model.go b/server/model.go index 6b5439a47..401547e4e 100644 --- a/server/model.go +++ b/server/model.go @@ -10,9 +10,6 @@ import ( "log/slog" "net/http" "os" - "slices" - "strings" - "text/template/parse" "github.com/ollama/ollama/api" "github.com/ollama/ollama/fs/ggml" @@ -128,124 +125,3 @@ func detectContentType(r io.Reader) (string, error) { return "unknown", nil } - -func parseObjects(s string) []map[string]any { - var objs []map[string]any - for offset := 0; offset < len(s); { - var obj map[string]any - decoder := json.NewDecoder(strings.NewReader(s[offset:])) - if err := decoder.Decode(&obj); errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { - break - } else if syntax := &(json.SyntaxError{}); errors.As(err, &syntax) { - // skip over any syntax errors - offset += int(syntax.Offset) - } else if unmarshalType := &(json.UnmarshalTypeError{}); errors.As(err, &unmarshalType) { - // skip over any unmarshalable types - offset += int(unmarshalType.Offset) - } else if err != nil { - return nil - } else { - offset += int(decoder.InputOffset()) - objs = append(objs, obj) - } - } - - return objs -} - -// parseToolCalls attempts to parse a JSON string into a slice of ToolCalls. -// mxyng: this only really works if the input contains tool calls in some JSON format -func (m *Model) parseToolCalls(s string) ([]api.ToolCall, bool) { - // create a subtree from the node that ranges over .ToolCalls - tmpl := m.Template.Subtree(func(n parse.Node) bool { - if t, ok := n.(*parse.RangeNode); ok { - return slices.Contains(template.Identifiers(t.Pipe), "ToolCalls") - } - - return false - }) - - if tmpl == nil { - return nil, false - } - - var b bytes.Buffer - if err := tmpl.Execute(&b, map[string][]api.ToolCall{ - "ToolCalls": { - { - Function: api.ToolCallFunction{ - Name: "@@name@@", - Arguments: api.ToolCallFunctionArguments{ - "@@argument@@": 1, - }, - }, - }, - }, - }); err != nil { - return nil, false - } - - templateObjects := parseObjects(b.String()) - if len(templateObjects) == 0 { - return nil, false - } - - // find the keys that correspond to the name and arguments fields - var name, arguments string - for k, v := range templateObjects[0] { - switch v.(type) { - case string: - name = k - case map[string]any: - arguments = k - } - } - - if name == "" || arguments == "" { - return nil, false - } - - responseObjects := parseObjects(s) - if len(responseObjects) == 0 { - return nil, false - } - - // collect all nested objects - var collect func(any) []map[string]any - collect = func(obj any) (all []map[string]any) { - switch o := obj.(type) { - case map[string]any: - all = append(all, o) - for _, v := range o { - all = append(all, collect(v)...) - } - case []any: - for _, v := range o { - all = append(all, collect(v)...) - } - } - - return all - } - - var objs []map[string]any - for _, p := range responseObjects { - objs = append(objs, collect(p)...) - } - - var toolCalls []api.ToolCall - for _, kv := range objs { - n, nok := kv[name].(string) - a, aok := kv[arguments].(map[string]any) - if nok && aok { - toolCalls = append(toolCalls, api.ToolCall{ - Function: api.ToolCallFunction{ - Name: n, - Arguments: a, - }, - }) - } - } - - return toolCalls, len(toolCalls) > 0 -} diff --git a/server/model_test.go b/server/model_test.go deleted file mode 100644 index e5c2f2bb2..000000000 --- a/server/model_test.go +++ /dev/null @@ -1,179 +0,0 @@ -package server - -import ( - "bytes" - "encoding/json" - "fmt" - "os" - "path/filepath" - "testing" - - "github.com/google/go-cmp/cmp" - - "github.com/ollama/ollama/api" - "github.com/ollama/ollama/template" -) - -func readFile(t *testing.T, base, name string) *bytes.Buffer { - t.Helper() - - bts, err := os.ReadFile(filepath.Join(base, name)) - if err != nil { - t.Fatal(err) - } - - return bytes.NewBuffer(bts) -} - -func TestExecuteWithTools(t *testing.T) { - p := filepath.Join("testdata", "tools") - cases := []struct { - model string - output string - ok bool - }{ - {"mistral", `[TOOL_CALLS] [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]`, true}, - {"mistral", `[TOOL_CALLS] [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}] - -The temperature in San Francisco, CA is 70°F and in Toronto, Canada is 20°C.`, true}, - {"mistral", `[TOOL_CALLS] [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"To }]`, false}, - {"mistral", `I'm not aware of that information. However, I can suggest searching for the weather using the "get_current_weather" function: - - [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]`, true}, - {"mistral", " The weather in San Francisco, CA is 70°F and in Toronto, Canada is 20°C.", false}, - {"command-r-plus", "Action: ```json" + ` -[ - { - "tool_name": "get_current_weather", - "parameters": { - "format": "fahrenheit", - "location": "San Francisco, CA" - } - }, - { - "tool_name": "get_current_weather", - "parameters": { - "format": "celsius", - "location": "Toronto, Canada" - } - } -] -` + "```", true}, - {"command-r-plus", " The weather in San Francisco, CA is 70°F and in Toronto, Canada is 20°C.", false}, - {"firefunction", ` functools[{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]`, true}, - {"firefunction", " The weather in San Francisco, CA is 70°F and in Toronto, Canada is 20°C.", false}, - {"llama3-groq-tool-use", ` -{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}} -{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}} -`, true}, - {"xlam", `{"tool_calls": [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]}`, true}, - {"nemotron", `{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]} `, true}, - } - - var tools []api.Tool - if err := json.Unmarshal(readFile(t, p, "tools.json").Bytes(), &tools); err != nil { - t.Fatal(err) - } - - var messages []api.Message - if err := json.Unmarshal(readFile(t, p, "messages.json").Bytes(), &messages); err != nil { - t.Fatal(err) - } - - calls := []api.ToolCall{ - { - Function: api.ToolCallFunction{ - Name: "get_current_weather", - Arguments: api.ToolCallFunctionArguments{ - "format": "fahrenheit", - "location": "San Francisco, CA", - }, - }, - }, - { - Function: api.ToolCallFunction{ - Name: "get_current_weather", - Arguments: api.ToolCallFunctionArguments{ - "format": "celsius", - "location": "Toronto, Canada", - }, - }, - }, - } - - for _, tt := range cases { - t.Run(tt.model, func(t *testing.T) { - tmpl, err := template.Parse(readFile(t, p, fmt.Sprintf("%s.gotmpl", tt.model)).String()) - if err != nil { - t.Fatal(err) - } - - t.Run("template", func(t *testing.T) { - var actual bytes.Buffer - if err := tmpl.Execute(&actual, template.Values{Tools: tools, Messages: messages}); err != nil { - t.Fatal(err) - } - - if diff := cmp.Diff(actual.String(), readFile(t, p, fmt.Sprintf("%s.out", tt.model)).String()); diff != "" { - t.Errorf("mismatch (-got +want):\n%s", diff) - } - }) - - t.Run("parse", func(t *testing.T) { - m := &Model{Template: tmpl} - actual, ok := m.parseToolCalls(tt.output) - if ok != tt.ok { - t.Fatalf("expected %t, got %t", tt.ok, ok) - } - - if tt.ok { - if diff := cmp.Diff(actual, calls); diff != "" { - t.Errorf("mismatch (-got +want):\n%s", diff) - } - } - }) - }) - } -} - -func TestParseObjects(t *testing.T) { - tests := []struct { - input string - want []map[string]any - }{ - { - input: `[{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]`, - want: []map[string]any{ - {"name": "get_current_weather", "arguments": map[string]any{"format": "fahrenheit", "location": "San Francisco, CA"}}, - {"name": "get_current_weather", "arguments": map[string]any{"format": "celsius", "location": "Toronto, Canada"}}, - }, - }, - { - input: `{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}} `, - want: []map[string]any{ - {"name": "get_current_weather", "arguments": map[string]any{"format": "fahrenheit", "location": "San Francisco, CA"}}, - }, - }, - { - input: `{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}} {"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, ON"}} `, - want: []map[string]any{ - {"name": "get_current_weather", "arguments": map[string]any{"format": "fahrenheit", "location": "San Francisco, CA"}}, - {"name": "get_current_weather", "arguments": map[string]any{"format": "celsius", "location": "Toronto, ON"}}, - }, - }, - { - input: `{"name": "get_current_weather", "arguments": `, - want: nil, - }, - } - - for _, tc := range tests { - t.Run(tc.input, func(t *testing.T) { - got := parseObjects(tc.input) - - if diff := cmp.Diff(got, tc.want); diff != "" { - t.Errorf("mismatch (-got +want):\n%s", diff) - } - }) - } -} diff --git a/server/routes.go b/server/routes.go index d0b8f487e..42e8cdd1d 100644 --- a/server/routes.go +++ b/server/routes.go @@ -38,6 +38,7 @@ import ( "github.com/ollama/ollama/server/internal/client/ollama" "github.com/ollama/ollama/server/internal/registry" "github.com/ollama/ollama/template" + "github.com/ollama/ollama/tools" "github.com/ollama/ollama/types/errtypes" "github.com/ollama/ollama/types/model" "github.com/ollama/ollama/version" @@ -1482,11 +1483,20 @@ func (s *Server) ChatHandler(c *gin.Context) { return } + var toolParser *tools.Parser + if len(req.Tools) > 0 { + toolParser, err = tools.NewParser(m.Template.Template) + if err != nil { + slog.Error("failed to create tool parser", "error", err) + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + } + ch := make(chan any) go func() { defer close(ch) - var sb strings.Builder - var toolCallIndex int = 0 + if err := r.Completion(c.Request.Context(), llm.CompletionRequest{ Prompt: prompt, Images: images, @@ -1512,37 +1522,21 @@ func (s *Server) ChatHandler(c *gin.Context) { res.LoadDuration = checkpointLoaded.Sub(checkpointStart) } - // TODO: tool call checking and filtering should be moved outside of this callback once streaming - // however this was a simple change for now without reworking streaming logic of this (and other) - // handlers - if req.Stream != nil && !*req.Stream || len(req.Tools) == 0 { - ch <- res - return - } - - // Streaming tool calls: - // If tools are recognized, use a flag to track the sending of a tool downstream - // This ensures that content is cleared from the message on the last chunk sent - sb.WriteString(r.Content) - if toolCalls, ok := m.parseToolCalls(sb.String()); ok { - res.Message.ToolCalls = toolCalls - for i := range toolCalls { - toolCalls[i].Function.Index = toolCallIndex - toolCallIndex++ + if len(req.Tools) > 0 { + toolCalls, content := toolParser.Add(r.Content) + if len(content) > 0 { + res.Message.Content = content + } else if len(toolCalls) > 0 { + res.Message.ToolCalls = toolCalls + res.Message.Content = "" + } else { + if r.Done { + ch <- res + } + return } - res.Message.Content = "" - sb.Reset() - ch <- res - return - } - - if r.Done { - // Send any remaining content if no tool calls were detected - if toolCallIndex == 0 { - res.Message.Content = sb.String() - } - ch <- res } + ch <- res }); err != nil { ch <- gin.H{"error": err.Error()} } @@ -1551,11 +1545,15 @@ func (s *Server) ChatHandler(c *gin.Context) { if req.Stream != nil && !*req.Stream { var resp api.ChatResponse var sb strings.Builder + var toolCalls []api.ToolCall for rr := range ch { switch t := rr.(type) { case api.ChatResponse: sb.WriteString(t.Message.Content) resp = t + if len(req.Tools) > 0 { + toolCalls = append(toolCalls, t.Message.ToolCalls...) + } case gin.H: msg, ok := t["error"].(string) if !ok { @@ -1571,12 +1569,8 @@ func (s *Server) ChatHandler(c *gin.Context) { } resp.Message.Content = sb.String() - - if len(req.Tools) > 0 { - if toolCalls, ok := m.parseToolCalls(sb.String()); ok { - resp.Message.ToolCalls = toolCalls - resp.Message.Content = "" - } + if len(toolCalls) > 0 { + resp.Message.ToolCalls = toolCalls } c.JSON(http.StatusOK, resp) diff --git a/server/testdata/tools/command-r-plus.gotmpl b/tools/testdata/command-r-plus.gotmpl similarity index 100% rename from server/testdata/tools/command-r-plus.gotmpl rename to tools/testdata/command-r-plus.gotmpl diff --git a/server/testdata/tools/command-r-plus.out b/tools/testdata/command-r-plus.out similarity index 100% rename from server/testdata/tools/command-r-plus.out rename to tools/testdata/command-r-plus.out diff --git a/server/testdata/tools/firefunction.gotmpl b/tools/testdata/firefunction.gotmpl similarity index 100% rename from server/testdata/tools/firefunction.gotmpl rename to tools/testdata/firefunction.gotmpl diff --git a/server/testdata/tools/firefunction.out b/tools/testdata/firefunction.out similarity index 100% rename from server/testdata/tools/firefunction.out rename to tools/testdata/firefunction.out diff --git a/server/testdata/tools/llama3-groq-tool-use.gotmpl b/tools/testdata/llama3-groq-tool-use.gotmpl similarity index 100% rename from server/testdata/tools/llama3-groq-tool-use.gotmpl rename to tools/testdata/llama3-groq-tool-use.gotmpl diff --git a/server/testdata/tools/llama3-groq-tool-use.out b/tools/testdata/llama3-groq-tool-use.out similarity index 100% rename from server/testdata/tools/llama3-groq-tool-use.out rename to tools/testdata/llama3-groq-tool-use.out diff --git a/tools/testdata/llama3.2.gotmpl b/tools/testdata/llama3.2.gotmpl new file mode 100644 index 000000000..b132423e5 --- /dev/null +++ b/tools/testdata/llama3.2.gotmpl @@ -0,0 +1,44 @@ +<|start_header_id|>system<|end_header_id|> + +Cutting Knowledge Date: December 2023 + +{{ if .System }}{{ .System }} +{{- end }} +{{- if .Tools }}When you receive a tool call response, use the output to format an answer to the orginal user question. + +You are a helpful assistant with tool calling capabilities. +{{- end }}<|eot_id|> +{{- range $i, $_ := .Messages }} +{{- $last := eq (len (slice $.Messages $i)) 1 }} +{{- if eq .Role "user" }}<|start_header_id|>user<|end_header_id|> +{{- if and $.Tools $last }} + +Given the following functions, please respond with a JSON for a function call with its proper arguments that best answers the given prompt. + +Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}. Do not use variables. + +{{ range $.Tools }} +{{- . }} +{{ end }} +{{ .Content }}<|eot_id|> +{{- else }} + +{{ .Content }}<|eot_id|> +{{- end }}{{ if $last }}<|start_header_id|>assistant<|end_header_id|> + +{{ end }} +{{- else if eq .Role "assistant" }}<|start_header_id|>assistant<|end_header_id|> +{{- if .ToolCalls }} +{{ range .ToolCalls }} +{"name": "{{ .Function.Name }}", "parameters": {{ .Function.Arguments }}}{{ end }} +{{- else }} + +{{ .Content }} +{{- end }}{{ if not $last }}<|eot_id|>{{ end }} +{{- else if eq .Role "tool" }}<|start_header_id|>ipython<|end_header_id|> + +{{ .Content }}<|eot_id|>{{ if $last }}<|start_header_id|>assistant<|end_header_id|> + +{{ end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/tools/testdata/llama3.2.out b/tools/testdata/llama3.2.out new file mode 100644 index 000000000..a27c6eafc --- /dev/null +++ b/tools/testdata/llama3.2.out @@ -0,0 +1,24 @@ +<|start_header_id|>system<|end_header_id|> + +Cutting Knowledge Date: December 2023 + +You are a knowledgeable assistant. You can answer questions and perform tasks.When you receive a tool call response, use the output to format an answer to the orginal user question. + +You are a helpful assistant with tool calling capabilities.<|eot_id|><|start_header_id|>user<|end_header_id|> + +What's the weather like today in Paris?<|eot_id|><|start_header_id|>assistant<|end_header_id|> + +{"name": "get_current_weather", "parameters": {"format":"celsius","location":"Paris, France"}}<|eot_id|><|start_header_id|>ipython<|end_header_id|> + +22<|eot_id|><|start_header_id|>assistant<|end_header_id|> + +The current temperature in Paris, France is 22 degrees Celsius.<|eot_id|><|start_header_id|>user<|end_header_id|> + +Given the following functions, please respond with a JSON for a function call with its proper arguments that best answers the given prompt. + +Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}. Do not use variables. + +{"type":"function","function":{"name":"get_current_weather","description":"Get the current weather","parameters":{"type":"object","required":["location","format"],"properties":{"format":{"type":"string","description":"The temperature unit to use. Infer this from the user's location.","enum":["celsius","fahrenheit"]},"location":{"type":"string","description":"The city and state, e.g. San Francisco, CA"}}}}} + +What's the weather like today in San Francisco and Toronto?<|eot_id|><|start_header_id|>assistant<|end_header_id|> + diff --git a/server/testdata/tools/messages.json b/tools/testdata/messages.json similarity index 100% rename from server/testdata/tools/messages.json rename to tools/testdata/messages.json diff --git a/server/testdata/tools/mistral.gotmpl b/tools/testdata/mistral.gotmpl similarity index 100% rename from server/testdata/tools/mistral.gotmpl rename to tools/testdata/mistral.gotmpl diff --git a/server/testdata/tools/mistral.out b/tools/testdata/mistral.out similarity index 100% rename from server/testdata/tools/mistral.out rename to tools/testdata/mistral.out diff --git a/server/testdata/tools/nemotron.gotmpl b/tools/testdata/nemotron.gotmpl similarity index 100% rename from server/testdata/tools/nemotron.gotmpl rename to tools/testdata/nemotron.gotmpl diff --git a/server/testdata/tools/nemotron.out b/tools/testdata/nemotron.out similarity index 100% rename from server/testdata/tools/nemotron.out rename to tools/testdata/nemotron.out diff --git a/tools/testdata/qwen2.5.gotmpl b/tools/testdata/qwen2.5.gotmpl new file mode 100644 index 000000000..cbd7302c4 --- /dev/null +++ b/tools/testdata/qwen2.5.gotmpl @@ -0,0 +1,51 @@ +{{- if .Suffix }}<|fim_prefix|>{{ .Prompt }}<|fim_suffix|>{{ .Suffix }}<|fim_middle|> +{{- else if .Messages }} +{{- if or .System .Tools }}<|im_start|>system +{{- if .System }} +{{ .System }} +{{- end }} +{{- if .Tools }} + +# Tools + +You may call one or more functions to assist with the user query. + +You are provided with function signatures within XML tags: + +{{- range .Tools }} +{"type": "function", "function": {{ .Function }}} +{{- end }} + + +For each function call, return a json object with function name and arguments within XML tags: + +{"name": , "arguments": } + +{{- end }}<|im_end|> +{{ end }} +{{- range $i, $_ := .Messages }} +{{- $last := eq (len (slice $.Messages $i)) 1 -}} +{{- if eq .Role "user" }}<|im_start|>user +{{ .Content }}<|im_end|> +{{ else if eq .Role "assistant" }}<|im_start|>assistant +{{ if .Content }}{{ .Content }} +{{- else if .ToolCalls }} +{{ range .ToolCalls }}{"name": "{{ .Function.Name }}", "arguments": {{ .Function.Arguments }}} +{{ end }} +{{- end }}{{ if not $last }}<|im_end|> +{{ end }} +{{- else if eq .Role "tool" }}<|im_start|>user + +{{ .Content }} +<|im_end|> +{{ end }} +{{- if and (ne .Role "assistant") $last }}<|im_start|>assistant +{{ end }} +{{- end }} +{{- else }} +{{- if .System }}<|im_start|>system +{{ .System }}<|im_end|> +{{ end }}{{ if .Prompt }}<|im_start|>user +{{ .Prompt }}<|im_end|> +{{ end }}<|im_start|>assistant +{{ end }}{{ .Response }}{{ if .Response }}<|im_end|>{{ end }} \ No newline at end of file diff --git a/tools/testdata/qwen2.5.out b/tools/testdata/qwen2.5.out new file mode 100644 index 000000000..76bfbfa98 --- /dev/null +++ b/tools/testdata/qwen2.5.out @@ -0,0 +1,31 @@ +<|im_start|>system +You are a knowledgeable assistant. You can answer questions and perform tasks. + +# Tools + +You may call one or more functions to assist with the user query. + +You are provided with function signatures within XML tags: + +{"type": "function", "function": {"name":"get_current_weather","description":"Get the current weather","parameters":{"type":"object","required":["location","format"],"properties":{"format":{"type":"string","description":"The temperature unit to use. Infer this from the user's location.","enum":["celsius","fahrenheit"]},"location":{"type":"string","description":"The city and state, e.g. San Francisco, CA"}}}}} + + +For each function call, return a json object with function name and arguments within XML tags: + +{"name": , "arguments": } +<|im_end|> +<|im_start|>user +What's the weather like today in Paris?<|im_end|> +<|im_start|>assistant + +{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Paris, France"}} +<|im_end|> +<|im_start|>user + +22 +<|im_end|> +<|im_start|>assistant +The current temperature in Paris, France is 22 degrees Celsius.<|im_end|> +<|im_start|>user +What's the weather like today in San Francisco and Toronto?<|im_end|> +<|im_start|>assistant diff --git a/tools/testdata/qwen3.gotmpl b/tools/testdata/qwen3.gotmpl new file mode 100644 index 000000000..26f6656fa --- /dev/null +++ b/tools/testdata/qwen3.gotmpl @@ -0,0 +1,50 @@ +{{- if .Messages }} +{{- if or .System .Tools }}<|im_start|>system +{{- if .System }} +{{ .System }} +{{- end }} +{{- if .Tools }} + +# Tools + +You may call one or more functions to assist with the user query. + +You are provided with function signatures within XML tags: + +{{- range .Tools }} +{"type": "function", "function": {{ .Function }}} +{{- end }} + + +For each function call, return a json object with function name and arguments within XML tags: + +{"name": , "arguments": } + +{{- end }}<|im_end|> +{{ end }} +{{- range $i, $_ := .Messages }} +{{- $last := eq (len (slice $.Messages $i)) 1 -}} +{{- if eq .Role "user" }}<|im_start|>user +{{ .Content }}<|im_end|> +{{ else if eq .Role "assistant" }}<|im_start|>assistant +{{ if .Content }}{{ .Content }} +{{- else if .ToolCalls }} +{{ range .ToolCalls }}{"name": "{{ .Function.Name }}", "arguments": {{ .Function.Arguments }}} +{{ end }} +{{- end }}{{ if not $last }}<|im_end|> +{{ end }} +{{- else if eq .Role "tool" }}<|im_start|>user + +{{ .Content }} +<|im_end|> +{{ end }} +{{- if and (ne .Role "assistant") $last }}<|im_start|>assistant +{{ end }} +{{- end }} +{{- else }} +{{- if .System }}<|im_start|>system +{{ .System }}<|im_end|> +{{ end }}{{ if .Prompt }}<|im_start|>user +{{ .Prompt }}<|im_end|> +{{ end }}<|im_start|>assistant +{{ end }}{{ .Response }}{{ if .Response }}<|im_end|>{{ end }} \ No newline at end of file diff --git a/tools/testdata/qwen3.out b/tools/testdata/qwen3.out new file mode 100644 index 000000000..76bfbfa98 --- /dev/null +++ b/tools/testdata/qwen3.out @@ -0,0 +1,31 @@ +<|im_start|>system +You are a knowledgeable assistant. You can answer questions and perform tasks. + +# Tools + +You may call one or more functions to assist with the user query. + +You are provided with function signatures within XML tags: + +{"type": "function", "function": {"name":"get_current_weather","description":"Get the current weather","parameters":{"type":"object","required":["location","format"],"properties":{"format":{"type":"string","description":"The temperature unit to use. Infer this from the user's location.","enum":["celsius","fahrenheit"]},"location":{"type":"string","description":"The city and state, e.g. San Francisco, CA"}}}}} + + +For each function call, return a json object with function name and arguments within XML tags: + +{"name": , "arguments": } +<|im_end|> +<|im_start|>user +What's the weather like today in Paris?<|im_end|> +<|im_start|>assistant + +{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Paris, France"}} +<|im_end|> +<|im_start|>user + +22 +<|im_end|> +<|im_start|>assistant +The current temperature in Paris, France is 22 degrees Celsius.<|im_end|> +<|im_start|>user +What's the weather like today in San Francisco and Toronto?<|im_end|> +<|im_start|>assistant diff --git a/server/testdata/tools/tools.json b/tools/testdata/tools.json similarity index 100% rename from server/testdata/tools/tools.json rename to tools/testdata/tools.json diff --git a/server/testdata/tools/xlam.gotmpl b/tools/testdata/xlam.gotmpl similarity index 100% rename from server/testdata/tools/xlam.gotmpl rename to tools/testdata/xlam.gotmpl diff --git a/server/testdata/tools/xlam.out b/tools/testdata/xlam.out similarity index 100% rename from server/testdata/tools/xlam.out rename to tools/testdata/xlam.out diff --git a/tools/tools.go b/tools/tools.go new file mode 100644 index 000000000..509ca90ac --- /dev/null +++ b/tools/tools.go @@ -0,0 +1,271 @@ +package tools + +import ( + "encoding/json" + "errors" + "log/slog" + "strings" + gotmpl "text/template" + + "github.com/ollama/ollama/api" + "github.com/ollama/ollama/template" +) + +var ( + errInvalidToolCall = errors.New("invalid tool call format") + errAccumulateMore = errors.New("need to accumulate more content") +) + +type Parser struct { + parseLeadingJSON bool + prefix string + prefixFound bool + tmpl gotmpl.Template + sb strings.Builder + index int + name string + arguments string + done bool +} + +// parseJSONToolCalls attempts to parse a JSON string into a slice of ToolCalls. +// +// Parameters: +// - s: The string to parse +// - name: The field name from template that identifies the tool call name +// - arguments: The field name from template that identifies the tool call arguments +// +// Returns: +// - []api.ToolCall: The parsed tool calls if successful +// - error: ErrAccumulateMore if braces unbalanced, ErrInvalidToolCall if invalid, or nil if successful +func parseJSONToolCalls(s string, name, arguments string, prefix string) ([]api.ToolCall, error) { + // Check for balanced braces before attempting to parse + braceCount := 0 + squareCount := 0 + startIndex := -1 + var rawToolCalls []string + s = strings.TrimSpace(s) + + // Only track these if we don't have a prefix as it will be cut off from the prefix. Also track in the parseLeadingJSON case. + trackSquareBrackets := prefix == "" || !strings.HasSuffix(prefix, "[") || strings.HasPrefix(s, "[") + for i, c := range s { + switch c { + case '{': + braceCount++ + if startIndex == -1 { + startIndex = i + } + case '}': + braceCount-- + if braceCount == 0 { + rawToolCalls = append(rawToolCalls, s[startIndex:i+1]) + startIndex = -1 + } + case '[': + if trackSquareBrackets { + squareCount++ + } + case ']': + if trackSquareBrackets { + squareCount-- + } + } + + // Negative means we have an extra closing brace/bracket + if braceCount < 0 || squareCount < 0 { + return nil, errInvalidToolCall + } + } + + // If braces/brackets aren't balanced, need more input + if braceCount > 0 || squareCount > 0 { + return nil, errAccumulateMore + } + + t := strings.TrimSpace(s) + if len(t) == 0 { + return nil, errAccumulateMore + } + // If the input is a single square bracket, it's not a valid tool call + if t[0] == '[' && len(t) == 1 { + return nil, errAccumulateMore + } + + // Attempt full unmarshal of the JSON + var toolCalls []api.ToolCall + for _, rawToolCall := range rawToolCalls { + var resp map[string]any + if err := json.Unmarshal([]byte(rawToolCall), &resp); err != nil { + continue + } + + // Collect nested objects that could contain tool calls + objs := collect(resp) + if len(objs) == 0 { + continue + } + + // Extract tool calls from objects + for _, kv := range objs { + n, nok := kv[name].(string) + a, aok := kv[arguments].(map[string]any) + if nok && aok { + toolCalls = append(toolCalls, api.ToolCall{ + Function: api.ToolCallFunction{ + Name: n, + Arguments: a, + }, + }) + } else { + slog.Debug("No valid tool call found in object.", "object", kv) + } + } + } + + // Valid JSON, no tool calls found + if len(toolCalls) == 0 { + slog.Debug("No valid tool calls found in any raw tool calls.", "rawToolCalls", rawToolCalls) + return nil, errInvalidToolCall + } + + return toolCalls, nil +} + +// checkPrefix processes a string to find and handle a prefix pattern. +// +// Returns: +// - The processed string with prefix removed if found +// - error: ErrAccumulateMore if prefix is incomplete, or nil if successful +func (p *Parser) checkPrefix(s string) (string, error) { + original := s + if strings.ContainsRune(s, '\n') { + s = strings.ReplaceAll(s, "\n", " ") + } + + if s == "" || p.prefix == "" { + return s, nil + } + + // Check for prefix at start of string + if cut, hasPrefix := strings.CutPrefix(s, p.prefix); hasPrefix { + // Found prefix at start - accumulate for potential tool + p.prefixFound = true + return cut, nil + } + + // Check if prefix overlaps end of string + if idx := suffixOverlap(s, p.prefix); idx != -1 { + // Return everything except overlapping portion + p.sb.Reset() + p.sb.WriteString(s[idx:]) + return original[:idx], errAccumulateMore + } + + // Check if prefix appears in middle of string + if idx := strings.Index(s, p.prefix); idx != -1 { + // Save remainder starting at prefix for next pass + p.sb.Reset() + p.sb.WriteString(strings.TrimSpace(s[idx:])) + // Return everything before prefix + return original[:idx], errAccumulateMore + } + + // No partial prefix found + return s, nil +} + +// Add processes a string input to parse tool calls and content. +// It handles prefix detection and JSON parsing to extract tool calls. +// +// Returns: +// - tools: Any parsed tool calls +// - content: Non-tool call content +func (p *Parser) Add(s string) (tools []api.ToolCall, content string) { + if strings.TrimSpace(s) == "" { + return nil, s + } + if p.done { + if p.index == 0 { + // Return original string if no tool calls found at start + return nil, s + } + // Return empty if no tool calls found after start + return nil, "" + } + p.sb.WriteString(s) + s = p.sb.String() + + // Check for prefix pattern in input + s, err := p.checkPrefix(s) + if err != nil { + // Need more input to complete prefix + return nil, s + } + + // Exit if prefix exists in template, greedy parsing is off, and prefix not found + if !p.parseLeadingJSON && !p.prefixFound { + p.sb.Reset() + return nil, s + } + + toolCalls, err := parseJSONToolCalls(s, p.name, p.arguments, p.prefix) + if err != nil { + if errors.Is(err, errAccumulateMore) { + return nil, "" + } + p.sb.Reset() + // Do not try parsing leading JSON if JSON not found + p.parseLeadingJSON = false + if p.prefix == "" { + p.done = true + } + if p.index != 0 && p.prefix == "" { + return nil, "" + } + if p.prefixFound { + // Drop tokens since prefix was found + return nil, "" + } + return nil, s + } + + for _, tc := range toolCalls { + tc.Function.Index = p.index + p.index++ + } + + p.sb.Reset() + return toolCalls, "" +} + +// NewParser creates a new tool call parser from a template. It extracts the tool call format, +// prefix, and field names from the template to use for parsing tool calls from model output. +// +// Returns an error if the template does not contain valid tool call formatting. +func NewParser(templateToProcess *gotmpl.Template) (*Parser, error) { + parsed, err := template.Parse(templateToProcess.Root.String()) + if err != nil { + return nil, err + } + + tt, err := toolTemplate(parsed) + if err != nil { + return nil, err + } + + tp := toolPrefix(templateToProcess) + + name, arguments, err := extractToolArgs(tt) + if err != nil { + return nil, err + } + + return &Parser{ + tmpl: *tt, + sb: strings.Builder{}, + prefix: tp, + parseLeadingJSON: true, + name: name, + arguments: arguments, + }, nil +} diff --git a/tools/tools_test.go b/tools/tools_test.go new file mode 100644 index 000000000..1ae3bff89 --- /dev/null +++ b/tools/tools_test.go @@ -0,0 +1,644 @@ +package tools + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + + "github.com/ollama/ollama/api" + "github.com/ollama/ollama/template" +) + +func readFile(t *testing.T, base, name string) *bytes.Buffer { + t.Helper() + + bts, err := os.ReadFile(filepath.Join(base, name)) + if err != nil { + t.Fatal(err) + } + + return bytes.NewBuffer(bts) +} + +func TestParseJSONToolCalls(t *testing.T) { + tests := []struct { + name string + input string + nameField string + argsField string + wantToolCalls []api.ToolCall + wantErr error + prefix string + }{ + { + name: "valid single tool call", + input: `{"name": "test_tool", "arguments": {"arg1": "value1"}}`, + nameField: "name", + argsField: "arguments", + wantToolCalls: []api.ToolCall{ + { + Function: api.ToolCallFunction{ + Name: "test_tool", + Arguments: map[string]any{ + "arg1": "value1", + }, + }, + }, + }, + wantErr: nil, + prefix: "", + }, + { + name: "incomplete JSON", + input: `{"name": "test_tool", "arguments": {"arg1": `, + nameField: "name", + argsField: "arguments", + wantToolCalls: nil, + wantErr: errAccumulateMore, + prefix: "", + }, + { + name: "invalid JSON", + input: `not json at all`, + nameField: "name", + argsField: "arguments", + wantToolCalls: nil, + wantErr: errInvalidToolCall, + prefix: "", + }, + { + name: "missing required fields", + input: `{"other": "field"}`, + nameField: "name", + argsField: "arguments", + wantToolCalls: nil, + wantErr: errInvalidToolCall, + prefix: "", + }, + { + name: "multiple tool calls in array", + input: `[ + {"name": "tool1", "arguments": {"arg1": 1}}, + {"name": "tool2", "arguments": {"arg2": "value"}} + ]`, + nameField: "name", + argsField: "arguments", + wantToolCalls: []api.ToolCall{ + { + Function: api.ToolCallFunction{ + Name: "tool1", + Arguments: map[string]any{ + "arg1": float64(1), + }, + }, + }, + { + Function: api.ToolCallFunction{ + Name: "tool2", + Arguments: map[string]any{ + "arg2": "value", + }, + }, + }, + }, + wantErr: nil, + prefix: "", + }, + { + name: "multiple tool calls without array", + input: ` + {"name": "tool1", "arguments": {"arg1": 1}}, + {"name": "tool2", "arguments": {"arg2": "value"}} + `, + nameField: "name", + argsField: "arguments", + wantToolCalls: []api.ToolCall{ + { + Function: api.ToolCallFunction{ + Name: "tool1", + Arguments: map[string]any{ + "arg1": float64(1), + }, + }, + }, + { + Function: api.ToolCallFunction{ + Name: "tool2", + Arguments: map[string]any{ + "arg2": "value", + }, + }, + }, + }, + wantErr: nil, + prefix: "", + }, + { + name: "multiple tool calls with text after", + input: ` + {"name": "tool1", "arguments": {"arg1": 1}} text + {"name": "tool2", "arguments": {"arg2": "value"}} text + `, + nameField: "name", + argsField: "arguments", + wantToolCalls: []api.ToolCall{ + { + Function: api.ToolCallFunction{ + Name: "tool1", + Arguments: map[string]any{ + "arg1": float64(1), + }, + }, + }, + { + Function: api.ToolCallFunction{ + Name: "tool2", + Arguments: map[string]any{ + "arg2": "value", + }, + }, + }, + }, + wantErr: nil, + prefix: "", + }, + { + name: "second tool call in array", + input: ` + , {"name": "tool2", "arguments": {"arg2": "value"}} + `, + nameField: "name", + argsField: "arguments", + wantToolCalls: []api.ToolCall{ + { + Function: api.ToolCallFunction{ + Name: "tool2", + Arguments: map[string]any{ + "arg2": "value", + }, + }, + }, + }, + wantErr: nil, + prefix: "", + }, + // a bad JSON would not return any tool calls or content as it would always accumulate more + { + name: "unbalanced square brackets", + input: `[{"name": "tool1", "arguments": {"arg1": [1, 2}]`, + nameField: "name", + argsField: "arguments", + wantToolCalls: nil, + wantErr: errAccumulateMore, + prefix: "", + }, + { + name: "incomplete square brackets", + input: `[{"name": "tool1", "arguments": {"arg1": [1, 2, 3`, + nameField: "name", + argsField: "arguments", + wantToolCalls: nil, + wantErr: errAccumulateMore, + prefix: "", + }, + { + name: "nested arrays in arguments", + input: `{"name": "tool1", "arguments": {"arg1": [1, 2, ["nested", "array"]]}}`, + nameField: "name", + argsField: "arguments", + wantToolCalls: []api.ToolCall{ + { + Function: api.ToolCallFunction{ + Name: "tool1", + Arguments: map[string]any{ + "arg1": []any{float64(1), float64(2), []any{"nested", "array"}}, + }, + }, + }, + }, + wantErr: nil, + prefix: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotCalls, err := parseJSONToolCalls(tt.input, tt.nameField, tt.argsField, tt.prefix) + + if err != tt.wantErr { + t.Errorf("parseJSONToolCalls() error = %v, want %v", err, tt.wantErr) + } + + if len(gotCalls) != 0 && tt.wantErr != nil { + t.Errorf("parseJSONToolCalls() valid = %v, want %v", len(gotCalls) == 0, tt.wantErr == nil) + } + + if diff := cmp.Diff(gotCalls, tt.wantToolCalls); diff != "" { + t.Errorf("parseJSONToolCalls() tool calls mismatch (-got +want):\n%s", diff) + } + }) + } +} + +func TestParseToolCalls(t *testing.T) { + p := filepath.Join("testdata") + t1 := api.ToolCall{ + Function: api.ToolCallFunction{ + Name: "get_current_weather", + Arguments: api.ToolCallFunctionArguments{ + "format": "fahrenheit", + "location": "San Francisco, CA", + }, + }, + } + t2 := api.ToolCall{ + Function: api.ToolCallFunction{ + Name: "get_current_weather", + Arguments: api.ToolCallFunctionArguments{ + "format": "celsius", + "location": "Toronto, Canada", + }, + }, + } + + cases := []struct { + name string + model string + output string + expectedToolCall []api.ToolCall + expectedTokens string + }{ + { + name: "mistral malformed json with tool calls prefix", + model: "mistral", + output: `[TOOL_CALLS] [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_curren}]`, + expectedToolCall: []api.ToolCall{t1}, + expectedTokens: "", + }, + { + name: "mistral multiple tool calls without prefix", + model: "mistral", + output: `[{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}} ]`, + expectedToolCall: []api.ToolCall{t1, t2}, + expectedTokens: "", + }, + { + name: "mistral tool calls with text between no prefix", + model: "mistral", + output: `[{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}] + model outputs more tokens here and then [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]`, + expectedToolCall: []api.ToolCall{t1, t2}, + expectedTokens: `model outputs more tokens here and then [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]`, + }, + { + name: "mistral valid json with tool calls prefix", + model: "mistral", + output: `[TOOL_CALLS] [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]`, + expectedToolCall: []api.ToolCall{t1, t2}, + expectedTokens: "", + }, + { + name: "mistral multiple tool calls with text between and prefix", + model: "mistral", + output: `[TOOL_CALLS] [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}] + model outputs more tokens here and then [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]`, + expectedToolCall: []api.ToolCall{t1, t2, t1, t2}, + expectedTokens: "", + }, + { + name: "mistral incomplete json with tool calls prefix", + model: "mistral", + output: `[TOOL_CALLS] [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, `, + expectedToolCall: []api.ToolCall{}, + expectedTokens: "", + }, + { + name: "mistral invalid tool call with explanatory text no prefix", + model: "mistral", + output: `I'm not aware of that information. However, I can suggest searching for the weather using the "get_current_weather" function: + + [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]`, + expectedToolCall: []api.ToolCall{}, + expectedTokens: `I'm not aware of that information. However, I can suggest searching for the weather using the "get_current_weather" function: [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]`, + }, + { + name: "mistral tool calls without prefix", + model: "mistral", + output: `[{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]`, + expectedToolCall: []api.ToolCall{t1, t2}, + expectedTokens: "", + }, + { + name: "command r plus tool calls with json block format", + model: "command-r-plus", + output: "Action: ```json" + ` + [ + { + "tool_name": "get_current_weather", + "parameters": { + "format": "fahrenheit", + "location": "San Francisco, CA" + } + }, + { + "tool_name": "get_current_weather", + "parameters": { + "format": "celsius", + "location": "Toronto, Canada" + } + } + ] + ` + "```", + expectedToolCall: []api.ToolCall{t1, t2}, + expectedTokens: "", + }, + { + name: "firefunction tool calls with functools prefix", + model: "firefunction", + output: ` functools[{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]`, + expectedToolCall: []api.ToolCall{t1, t2}, + expectedTokens: "", + }, + { + name: "llama3 groq single tool call with xml tags", + model: "llama3-groq-tool-use", + output: ` + {"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}} + `, + expectedToolCall: []api.ToolCall{t1}, + expectedTokens: "", + }, + { + name: "xlam tool calls with wrapper object", + model: "xlam", + output: `{"tool_calls": [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]}`, + expectedToolCall: []api.ToolCall{t1, t2}, + expectedTokens: "", + }, + { + name: "qwen2.5 single tool call with prefix", + model: "qwen2.5", + output: `{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}}`, + expectedToolCall: []api.ToolCall{t1}, + expectedTokens: "", + }, + { + name: "qwen2.5 multiple tool calls with and without prefix", + model: "qwen2.5", + output: `{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}} {"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}} {"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}`, + expectedToolCall: []api.ToolCall{t1, t1, t2}, + expectedTokens: "", + }, + { + name: "qwen2.5 plain text response no tool calls", + model: "qwen2.5", + output: "The weather in San Francisco, CA is 70°F and in Toronto, Canada is 20°C.", + expectedToolCall: []api.ToolCall{}, + expectedTokens: "The weather in San Francisco, CA is 70°F and in Toronto, Canada is 20°C.", + }, + { + name: "qwen2.5 tool calls with trailing text", + model: "qwen2.5", + output: `[{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}}, {"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}] some tokens after call`, + expectedToolCall: []api.ToolCall{t1, t2}, + expectedTokens: "some tokens after call", + }, + { + name: "qwen2.5 tool calls with initial text", + model: "qwen2.5", + output: `some tokens before call [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}}, {"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]`, + expectedToolCall: []api.ToolCall{}, + expectedTokens: `some tokens before call [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}}, {"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]`, + }, + { + name: "qwen2.5 tool calls with prefix and trailing text", + model: "qwen2.5", + output: ` [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}}, {"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}] some tokens after call`, + expectedToolCall: []api.ToolCall{t1, t2}, + expectedTokens: "", + }, + { + name: "qwen2.5 tool calls with prefix and initial text", + model: "qwen2.5", + output: `some tokens before call [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}}, {"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}] `, + expectedToolCall: []api.ToolCall{t1, t2}, + expectedTokens: "some tokens before call", + }, + { + name: "qwen2.5 tool calls without and with prefix", + model: "qwen2.5", + output: `{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}} {"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}`, + expectedToolCall: []api.ToolCall{t1, t2}, + expectedTokens: "", + }, + { + name: "qwen2.5 tool calls without and with prefix and text between", + model: "qwen2.5", + output: `{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}} some tokens between {"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}} some tokens after call`, + expectedToolCall: []api.ToolCall{t1, t2}, + expectedTokens: "some tokens between", + }, + { + name: "qwen2.5 tool calls without prefix and invalid tool call with other tokens", + model: "qwen2.5", + output: `hi [{"options": "foo"}]`, + expectedToolCall: []api.ToolCall{}, + expectedTokens: `hi [{"options": "foo"}]`, + }, + { + name: "qwen2.5 tool calls with prefix and invalid tool call", + model: "qwen2.5", + output: ` [{"options": "foo"}] `, + expectedToolCall: []api.ToolCall{}, + expectedTokens: ``, + }, + { + name: "qwen3 tool call with think prefix and tool prefix (sent as a single token)", + model: "qwen3", + output: `Okay, let me think what tool we should use...{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}}`, + expectedToolCall: []api.ToolCall{t1}, + expectedTokens: "Okay, let me think what tool we should use...", + }, + { + name: "qwen3 tool call with think prefix, tool prefix, and whitespace (sent as separate tokens)", + model: "qwen3", + output: `Okay, let me think what tool we should use... { "name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}} `, + expectedToolCall: []api.ToolCall{t1}, + expectedTokens: "Okay, let me think what tool we should use...", + }, + { + name: "qwen3 empty think prefix without tool prefix and invalid tool call", + model: "qwen3", + output: ` {"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}} `, + expectedToolCall: []api.ToolCall{}, + expectedTokens: ` {"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}} `, + }, + { + name: "qwen3 empty think prefix with tool prefix and valid tool call", + model: "qwen3", + output: `{ "name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}} `, + expectedToolCall: []api.ToolCall{t1}, + expectedTokens: ``, + }, + { + name: "qwen3 invalid tool call with fake tool prefix (single rune suffix match)", + model: "qwen3", + output: `< fakeout {"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}} `, + expectedToolCall: []api.ToolCall{}, + expectedTokens: `< fakeout {"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}} `, + }, + { + name: "qwen3 invalid tool call with partial tool prefix (multiple rune suffix match)", + model: "qwen3", + output: ``, + expectedToolCall: []api.ToolCall{}, + expectedTokens: ``, + }, + { + name: "qwen3 invalid tool call with malformed tool prefix", + model: "qwen3", + output: ``, + expectedToolCall: []api.ToolCall{}, + expectedTokens: ``, + }, + { + name: "model with prefix in template, no prefix in output", + model: "qwen2.5", + output: `[{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}} {"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]`, + expectedToolCall: []api.ToolCall{t1, t2}, + expectedTokens: "", + }, + { + name: "model with prefix in template, prefix in output", + model: "qwen2.5", + output: `[{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}} {"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]`, + expectedToolCall: []api.ToolCall{t1, t2}, + expectedTokens: "", + }, + { + name: "model without prefix in template, no prefix in output", + model: "llama3.2", + output: `[{"name": "get_current_weather", "parameters": {"format":"fahrenheit","location":"San Francisco, CA"}} {"name": "get_current_weather", "parameters": {"format":"celsius","location":"Toronto, Canada"}}]`, + expectedToolCall: []api.ToolCall{t1, t2}, + expectedTokens: "", + }, + { + name: "model without prefix in template, no prefix in output, single tool call", + model: "llama3.2", + output: `{"name": "get_current_weather", "parameters": {"format":"fahrenheit","location":"San Francisco, CA"}}`, + expectedToolCall: []api.ToolCall{t1}, + expectedTokens: "", + }, + { + name: "model without prefix in template, prefix in output", + model: "llama3.2", + output: ` [{"name": "get_current_weather", "parameters": {"format":"fahrenheit","location":"San Francisco, CA"}} {"name": "get_current_weather", "parameters": {"format":"celsius","location":"Toronto, Canada"}}]`, + expectedToolCall: []api.ToolCall{}, + expectedTokens: ` [{"name": "get_current_weather", "parameters": {"format":"fahrenheit","location":"San Francisco, CA"}} {"name": "get_current_weather", "parameters": {"format":"celsius","location":"Toronto, Canada"}}]`, + }, + { + name: "model with prefix in template, no prefix in output, tokens before", + model: "qwen2.5", + output: `some tokens before [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}} {"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]`, + expectedToolCall: []api.ToolCall{}, + expectedTokens: `some tokens before [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}} {"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]`, + }, + { + name: "model with prefix in template, prefix in output, tokens after", + model: "qwen2.5", + output: `[{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}} {"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}] some tokens after`, + expectedToolCall: []api.ToolCall{t1, t2}, + expectedTokens: "", + }, + { + name: "model without prefix in template, no prefix in output, tokens after", + model: "llama3.2", + output: `[{"name": "get_current_weather", "parameters": {"format":"fahrenheit","location":"San Francisco, CA"}} {"name": "get_current_weather", "parameters": {"format":"celsius","location":"Toronto, Canada"}}] some tokens after`, + expectedToolCall: []api.ToolCall{t1, t2}, + expectedTokens: "", + }, + { + name: "model without prefix in template, no prefix in output, tokens before", + model: "llama3.2", + output: `some tokens before [{"name": "get_current_weather", "parameters": {"format":"fahrenheit","location":"San Francisco, CA"}} {"name": "get_current_weather", "parameters": {"format":"celsius","location":"Toronto, Canada"}}]`, + expectedToolCall: []api.ToolCall{}, + expectedTokens: `some tokens before [{"name": "get_current_weather", "parameters": {"format":"fahrenheit","location":"San Francisco, CA"}} {"name": "get_current_weather", "parameters": {"format":"celsius","location":"Toronto, Canada"}}]`, + }, + { + name: "model without prefix in template, prefix in output, tokens after", + model: "llama3.2", + output: ` [{"name": "get_current_weather", "parameters": {"format":"fahrenheit","location":"San Francisco, CA"}} {"name": "get_current_weather", "parameters": {"format":"celsius","location":"Toronto, Canada"}}] some tokens after`, + expectedToolCall: []api.ToolCall{}, + expectedTokens: ` [{"name": "get_current_weather", "parameters": {"format":"fahrenheit","location":"San Francisco, CA"}} {"name": "get_current_weather", "parameters": {"format":"celsius","location":"Toronto, Canada"}}] some tokens after`, + }, + } + + var tools []api.Tool + if err := json.Unmarshal(readFile(t, p, "tools.json").Bytes(), &tools); err != nil { + t.Fatal(err) + } + + var messages []api.Message + if err := json.Unmarshal(readFile(t, p, "messages.json").Bytes(), &messages); err != nil { + t.Fatal(err) + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + tmpl, err := template.Parse(readFile(t, p, fmt.Sprintf("%s.gotmpl", tt.model)).String()) + if err != nil { + t.Fatal(err) + } + + t.Run("template", func(t *testing.T) { + actual := &bytes.Buffer{} // Create new buffer for each test + if err := tmpl.Execute(actual, template.Values{Tools: tools, Messages: messages}); err != nil { + t.Fatal(err) + } + + if diff := cmp.Diff(actual.String(), readFile(t, p, fmt.Sprintf("%s.out", tt.model)).String()); diff != "" { + t.Errorf("mismatch (-got +want):\n%s", diff) + } + }) + + t.Run("parse", func(t *testing.T) { + tp, err := NewParser(tmpl.Template) + if err != nil { + t.Fatal(err) + } + got := []api.ToolCall{} + var gotTokens strings.Builder + + tokens := strings.Fields(tt.output) + for _, tok := range tokens { + s := " " + tok + + toolCalls, content := tp.Add(s) + if len(content) > 0 { + gotTokens.WriteString(content) + } else if len(toolCalls) > 0 { + got = append(got, toolCalls...) + } + } + + // Compare tool calls if we expect any + if diff := cmp.Diff(got, tt.expectedToolCall); diff != "" { + t.Errorf("tool calls mismatch (-got +want):\n%s", diff) + } + + // Compare tokens if we expect any + stripped := strings.TrimSpace(gotTokens.String()) + if diff := cmp.Diff(stripped, tt.expectedTokens); diff != "" { + t.Log("actualTokens", stripped, "expectedTokens", tt.expectedTokens) + t.Errorf("tokens mismatch (-got +want):\n%s", diff) + } + }) + }) + } +} diff --git a/tools/tools_utils.go b/tools/tools_utils.go new file mode 100644 index 000000000..48531b789 --- /dev/null +++ b/tools/tools_utils.go @@ -0,0 +1,227 @@ +package tools + +import ( + "bytes" + "encoding/json" + "errors" + "log/slog" + "slices" + "strings" + gotmpl "text/template" + "text/template/parse" + + "github.com/ollama/ollama/api" + "github.com/ollama/ollama/template" +) + +// extractToolCallsFormat traverses a template AST to find text that follows a ".ToolCalls" condition. +// It walks the template nodes looking for if-statements containing ".ToolCalls" and extracts any +// immediate text nodes that follow. This is used to identify tool call prefixes and formatting. +// +// Returns: +// - string: The extracted text following the first ".ToolCalls" condition found +// - bool: Whether a ".ToolCalls" condition was found in the template +func extractToolCallsFormat(tmpl *gotmpl.Template) (string, bool) { + if tmpl == nil || tmpl.Tree == nil { + slog.Debug("template or tree is nil") + return "", false + } + + var result string + var found bool + + var walk func(nodes []parse.Node) + walk = func(nodes []parse.Node) { + for _, node := range nodes { + if found { + return + } + + switch n := node.(type) { + case *parse.IfNode: + if isToolCallsNode(n) { + // Collect immediate TextNode(s) at start of IfNode's list + var sb strings.Builder + for _, innerNode := range n.List.Nodes { + if tn, ok := innerNode.(*parse.TextNode); ok { + sb.Write(tn.Text) + } else { + // Stop at first non-text node + break + } + } + result = sb.String() + found = true + return + } + // Recurse into child nodes + walk(n.List.Nodes) + if n.ElseList != nil { + walk(n.ElseList.Nodes) + } + case *parse.ListNode: + walk(n.Nodes) + case *parse.RangeNode: + walk(n.List.Nodes) + if n.ElseList != nil { + walk(n.ElseList.Nodes) + } + case *parse.WithNode: + walk(n.List.Nodes) + if n.ElseList != nil { + walk(n.ElseList.Nodes) + } + default: + // Continue to next node + continue + } + } + } + + walk(tmpl.Tree.Root.Nodes) + return result, found +} + +// isToolCallsNode detects if a node's condition includes ".ToolCalls" +func isToolCallsNode(n *parse.IfNode) bool { + for _, cmd := range n.Pipe.Cmds { + for _, arg := range cmd.Args { + if field, ok := arg.(*parse.FieldNode); ok { + if slices.Contains(field.Ident, "ToolCalls") { + return true + } + } + } + } + return false +} + +func toolPrefix(tmpl *gotmpl.Template) string { + tokenText, ok := extractToolCallsFormat(tmpl) + if !ok { + return "" + } + tokenText = strings.TrimSpace(tokenText) + tokenText = strings.ReplaceAll(tokenText, "\r", "") + tokenText = strings.ReplaceAll(tokenText, "\n", " ") + + return tokenText +} + +// toolTemplate creates a subtree from the node that ranges over .ToolCalls +// +// Returns: +// - *gotmpl.Template: The subtree containing the .ToolCalls range +// - error: Error if parsing failed +func toolTemplate(t *template.Template) (*gotmpl.Template, error) { + tmpl := t.Subtree(func(n parse.Node) bool { + if t, ok := n.(*parse.RangeNode); ok { + return slices.Contains(template.Identifiers(t.Pipe), "ToolCalls") + } + + return false + }) + + if tmpl == nil { + return nil, errors.New("failed to find tool template") + } + + return tmpl, nil +} + +// suffixOverlap returns the index in s where the longest suffix overlap with prefix begins +// +// Returns: +// - int: The starting index in s where the suffix overlap begins +func suffixOverlap(s, prefix string) int { + max := min(len(prefix), len(s)) + for i := max; i > 0; i-- { + if strings.HasSuffix(s, prefix[:i]) { + return len(s) - i + } + } + return -1 +} + +// extractToolArgs executes a template with a known tool call format to extract the name and arguments +// +// Returns: +// - string: The name of the tool call +// - string: The arguments of the tool call +// - error: Error if parsing failed +func extractToolArgs(tmpl *gotmpl.Template) (name, arguments string, err error) { + var b bytes.Buffer + if err := tmpl.Execute(&b, map[string][]api.ToolCall{ + "ToolCalls": { + { + Function: api.ToolCallFunction{ + Name: "@@name@@", + Arguments: api.ToolCallFunctionArguments{ + "@@argument@@": 1, + }, + }, + }, + }, + }); err != nil { + return "", "", err + } + + var obj any + err = json.Unmarshal(b.Bytes(), &obj) + if err != nil { + return "", "", err + } + + var objs []map[string]any + switch v := obj.(type) { + case map[string]any: + objs = []map[string]any{v} + case []map[string]any: + objs = v + case []any: + objs = collect(v) + } + if len(objs) == 0 { + return "", "", errors.New("no template objects found") + } + + // find the keys that correspond to the name and arguments fields + for k, v := range objs[0] { + switch v.(type) { + case string: + name = k + case map[string]any: + arguments = k + } + } + + if name == "" || arguments == "" { + slog.Debug("missing required fields in tool call template", "name", name, "arguments", arguments) + return "", "", errors.New("missing required fields in tool call template") + } + + return name, arguments, nil +} + +// collect recursively traverses an object to collect all nested maps +// +// Returns: +// - []map[string]any: A slice of all nested maps found in the object +func collect(obj any) []map[string]any { + var all []map[string]any + switch o := obj.(type) { + case map[string]any: + all = append(all, o) + for _, v := range o { + all = append(all, collect(v)...) + } + case []any: + for _, v := range o { + all = append(all, collect(v)...) + } + default: + return nil + } + + return all +} diff --git a/tools/tools_utils_test.go b/tools/tools_utils_test.go new file mode 100644 index 000000000..769183b73 --- /dev/null +++ b/tools/tools_utils_test.go @@ -0,0 +1,464 @@ +package tools + +import ( + "testing" + gotmpl "text/template" + + "github.com/ollama/ollama/template" +) + +func TestExtractToolCallsFormat(t *testing.T) { + cases := []struct { + name string + template string + want string + found bool + }{ + { + name: "nil template", + template: "", + want: "", + found: false, + }, + { + name: "basic tool call with text", + template: "{{if .ToolCalls}}Hello world{{end}}", + want: "Hello world", + found: true, + }, + { + name: "tool call with json format", + template: "{{if .ToolCalls}}```json\n{{end}}", + want: "```json\n", + found: true, + }, + { + name: "tool call in range", + template: "{{range .ToolCalls}}tool: {{.}}{{end}}", + want: "", + found: false, + }, + { + name: "tool call with multiple text nodes", + template: "{{if .ToolCalls}}First text{{if .Something}}inner{{end}}Second text{{end}}", + want: "First text", + found: true, + }, + { + name: "nested if without tool calls", + template: "{{if .Something}}{{if .OtherThing}}text{{end}}{{end}}", + want: "", + found: false, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + tmpl, err := gotmpl.New("test").Parse(tc.template) + if err != nil && tc.template != "" { + t.Fatalf("failed to parse template: %v", err) + } + + got, found := extractToolCallsFormat(tmpl) + if got != tc.want { + t.Errorf("got text %q, want %q", got, tc.want) + } + if found != tc.found { + t.Errorf("got found %v, want %v", found, tc.found) + } + }) + } +} + +func TestToolPrefix(t *testing.T) { + cases := []struct { + name string + template string + want string + }{ + { + name: "basic tool call with action prefix", + template: "{{if .ToolCalls}}Action: ```json{{end}}", + want: "Action: ```json", + }, + { + name: "incomplete functools bracket", + template: "{{if .ToolCalls}}functools[{{end}}", + want: "functools[", + }, + { + name: "tool call with angle brackets", + template: "{{if .ToolCalls}}Hello, world! {{end}}", + want: "Hello, world! ", + }, + { + name: "multiple tool call formats", + template: "{{if .ToolCalls}}[tool_call] {{end}}", + want: "[tool_call] ", + }, + { + name: "single angle bracket tool call", + template: "{{if .ToolCalls}}{{end}}", + want: "", + }, + { + name: "incomplete angle bracket after tool call", + template: "{{if .ToolCalls}}[tool_call] <{{end}}", + want: "[tool_call] <", + }, + { + name: "angle bracket prefix with tool call", + template: "{{if .ToolCalls}}> {{end}}", + want: "> ", + }, + { + name: "uppercase tool call with incomplete bracket", + template: "{{if .ToolCalls}}[TOOL_CALL] [{{end}}", + want: "[TOOL_CALL] [", + }, + { + name: "uppercase tool call with adjacent bracket", + template: "{{if .ToolCalls}}[TOOL_CALL][{{end}}", + want: "[TOOL_CALL][", + }, + { + name: "tool call with pipe delimiters", + template: "{{if .ToolCalls}}<|tool_call|>{{end}}", + want: "<|tool_call|>", + }, + { + name: "tool with no prefix", + template: "{{if .ToolCalls}}{{end}}", + want: "", + }, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + tmpl, err := gotmpl.New("test").Parse(tt.template) + if err != nil { + t.Fatalf("failed to parse template: %v", err) + } + got := toolPrefix(tmpl) + if got != tt.want { + t.Errorf("ToolToken(%q) = %q; want %q", tt.template, got, tt.want) + } + }) + } +} + +func TestToolTemplate(t *testing.T) { + cases := []struct { + name string + template string + want bool + }{ + { + name: "basic tool call range", + template: "{{range .ToolCalls}}test{{end}}", + want: true, + }, + { + name: "no tool calls", + template: "{{range .Other}}test{{end}}", + want: false, + }, + { + name: "nested tool calls", + template: "{{range .Outer}}{{range .ToolCalls}}test{{end}}{{end}}", + want: true, + }, + { + name: "empty template", + template: "", + want: false, + }, + { + name: "tool calls in if statement", + template: "{{if .ToolCalls}}test{{end}}", + want: false, + }, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + tmpl, err := gotmpl.New("test").Parse(tt.template) + if err != nil { + t.Fatalf("failed to parse template: %v", err) + } + + parsed, err := template.Parse(tmpl.Root.String()) + if err != nil { + t.Fatalf("failed to parse template: %v", err) + } + + _, err = toolTemplate(parsed) + if err != nil && tt.want { + t.Errorf("toolTemplate() = %v; want %v", err, tt.want) + } + }) + } +} + +func TestSuffixOverlap(t *testing.T) { + cases := []struct { + name string + s string + d string + want int + }{ + { + name: "no overlap", + s: "hello world", + d: "", + want: -1, + }, + { + name: "full overlap", + s: "", + d: "", + want: 0, + }, + { + name: "partial overlap", + s: "text ", + d: "", + want: 5, + }, + { + name: "delimiter longer than string", + s: "", + d: "", + want: -1, + }, + { + name: "empty string", + s: "", + d: "", + want: -1, + }, + { + name: "empty delimiter", + s: "", + d: "", + want: -1, + }, + { + name: "single char overlap", + s: "test<", + d: "", + want: 4, + }, + { + name: "partial tool call", + s: "hello ", + want: 6, + }, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + got := suffixOverlap(tt.s, tt.d) + if got != tt.want { + t.Errorf("suffixOverlap(%q, %q) = %d; want %d", tt.s, tt.d, got, tt.want) + } + }) + } +} + +func TestExtractToolArgs(t *testing.T) { + cases := []struct { + name string + template string + want string + ok bool + }{ + { + name: "basic tool call with text after", + template: `{{if .ToolCalls}}tool response{{end}}`, + want: "tool response", + ok: true, + }, + { + name: "tool call with mixed content after", + template: `{{if .ToolCalls}}{{.Something}}{{end}}`, + want: "", + ok: true, + }, + { + name: "tool call with no text after", + template: `{{if .ToolCalls}}{{.Something}}{{end}}`, + want: "", + ok: true, + }, + { + name: "nested tool call", + template: `{{if .Something}}{{if .ToolCalls}}[TOOL_CALL]{{end}}{{end}}`, + want: "[TOOL_CALL]", + ok: true, + }, + { + name: "no tool calls", + template: `{{if .Something}}no tools here{{end}}`, + want: "", + ok: false, + }, + { + name: "empty template", + template: ``, + want: "", + ok: false, + }, + { + name: "multiple tool calls sections", + template: `{{if .ToolCalls}}first{{end}}{{if .ToolCalls}}second{{end}}`, + want: "first", + ok: true, + }, + { + name: "range over tool calls", + template: `{{if .ToolCalls}}{{range .ToolCalls}}tool{{end}}{{end}}`, + want: "", + ok: true, + }, + { + name: "tool calls with pipe delimiters", + template: `{{if .ToolCalls}}<|tool|>{{end}}`, + want: "<|tool|>", + ok: true, + }, + { + name: "tool calls with nested template", + template: `{{if .ToolCalls}}{{template "tool" .}}{{end}}`, + want: "", + ok: true, + }, + { + name: "tool calls with whitespace variations", + template: `{{if .ToolCalls}} tool {{end}}`, + want: " tool ", + ok: true, + }, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + tmpl, err := gotmpl.New("test").Parse(tt.template) + if err != nil { + t.Fatalf("failed to parse template: %v", err) + } + + got, ok := extractToolCallsFormat(tmpl) + if got != tt.want { + t.Errorf("TextAfterToolCalls() got = %q, want %q", got, tt.want) + } + if ok != tt.ok { + t.Errorf("TextAfterToolCalls() ok = %v, want %v", ok, tt.ok) + } + }) + } +} + +func TestCollect(t *testing.T) { + cases := []struct { + name string + obj any + want []map[string]any + }{ + { + name: "simple map", + obj: map[string]any{ + "key": "value", + }, + want: []map[string]any{ + {"key": "value"}, + }, + }, + { + name: "nested map", + obj: map[string]any{ + "outer": map[string]any{ + "inner": "value", + }, + }, + want: []map[string]any{ + {"outer": map[string]any{"inner": "value"}}, + {"inner": "value"}, + }, + }, + { + name: "array of maps", + obj: []any{ + map[string]any{"key1": "val1"}, + map[string]any{"key2": "val2"}, + }, + want: []map[string]any{ + {"key1": "val1"}, + {"key2": "val2"}, + }, + }, + { + name: "deeply nested", + obj: map[string]any{ + "l1": map[string]any{ + "l2": map[string]any{ + "l3": "value", + }, + }, + }, + want: []map[string]any{ + {"l1": map[string]any{"l2": map[string]any{"l3": "value"}}}, + {"l2": map[string]any{"l3": "value"}}, + {"l3": "value"}, + }, + }, + { + name: "non-map value", + obj: "string", + want: nil, + }, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + got := collect(tt.obj) + if len(got) != len(tt.want) { + t.Errorf("collect() got %d maps, want %d", len(got), len(tt.want)) + return + } + + // Compare each map in the result + for i := range tt.want { + if !mapsEqual(got[i], tt.want[i]) { + t.Errorf("collect() map[%d] = %v, want %v", i, got[i], tt.want[i]) + } + } + }) + } +} + +// mapsEqual compares two maps for deep equality +func mapsEqual(m1, m2 map[string]any) bool { + if len(m1) != len(m2) { + return false + } + for k, v1 := range m1 { + v2, ok := m2[k] + if !ok { + return false + } + switch val1 := v1.(type) { + case map[string]any: + val2, ok := v2.(map[string]any) + if !ok || !mapsEqual(val1, val2) { + return false + } + default: + if v1 != v2 { + return false + } + } + } + return true +} From f18e0cb5508450bd14db5ec8015709d2c4ab820f Mon Sep 17 00:00:00 2001 From: Jesse Gross Date: Fri, 23 May 2025 15:37:32 -0700 Subject: [PATCH 10/54] ml: Improve slog formatting for BackendMemory --- ml/backend.go | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/ml/backend.go b/ml/backend.go index 6beb7d2bc..65f169486 100644 --- a/ml/backend.go +++ b/ml/backend.go @@ -5,6 +5,7 @@ import ( "context" "encoding/binary" "fmt" + "log/slog" "math" "slices" "strconv" @@ -133,6 +134,27 @@ type DeviceMemory struct { Graph Memory } +func memoryPresent(mem []Memory) bool { + return slices.ContainsFunc(mem, func(m Memory) bool { return m.Size != 0 }) +} + +func (m DeviceMemory) LogValue() slog.Value { + var attrs []slog.Attr + if memoryPresent(m.Weights) { + attrs = append(attrs, slog.Any("Weights", m.Weights)) + } + + if memoryPresent(m.Cache) { + attrs = append(attrs, slog.Any("Cache", m.Cache)) + } + + if m.Graph.Size != 0 { + attrs = append(attrs, slog.Any("Graph", m.Graph)) + } + + return slog.GroupValue(attrs...) +} + // BackendMemory provides the amount of memory required to load the model // per device based on the BackendParams. In some cases, not all required // allocations will be known at this point. However, the size of the most recent @@ -150,6 +172,20 @@ type BackendMemory struct { GPUs []DeviceMemory } +func (m BackendMemory) LogValue() slog.Value { + var attrs []slog.Attr + if m.InputWeights.Size != 0 { + attrs = append(attrs, slog.Any("InputWeights", m.InputWeights)) + } + + attrs = append(attrs, slog.Any(m.CPU.Name, m.CPU)) + for _, g := range m.GPUs { + attrs = append(attrs, slog.Any(g.Name, g)) + } + + return slog.GroupValue(attrs...) +} + var backends = make(map[string]func(string, BackendParams) (Backend, error)) func RegisterBackend(name string, f func(string, BackendParams) (Backend, error)) { From eda472df1bd420517ca05c59ba0096e8b518fb69 Mon Sep 17 00:00:00 2001 From: frob Date: Sat, 24 May 2025 22:17:04 +0200 Subject: [PATCH 11/54] server: add hint to the error message when model path access fails (#10843) --- server/modelpath.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/modelpath.go b/server/modelpath.go index d1bcae93b..af82b8b3b 100644 --- a/server/modelpath.go +++ b/server/modelpath.go @@ -116,7 +116,7 @@ func (mp ModelPath) BaseURL() *url.URL { func GetManifestPath() (string, error) { path := filepath.Join(envconfig.Models(), "manifests") if err := os.MkdirAll(path, 0o755); err != nil { - return "", err + return "", fmt.Errorf("%w: ensure path elements are traversable", err) } return path, nil @@ -139,7 +139,7 @@ func GetBlobsPath(digest string) (string, error) { } if err := os.MkdirAll(dirPath, 0o755); err != nil { - return "", err + return "", fmt.Errorf("%w: ensure path elements are traversable", err) } return path, nil From 6623898198fece95e6c5cef7ad55d9d1ecd1da1f Mon Sep 17 00:00:00 2001 From: frob Date: Sat, 24 May 2025 22:17:26 +0200 Subject: [PATCH 12/54] docs: remove unsupported quantizations (#10842) --- docs/import.md | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/docs/import.md b/docs/import.md index 01fea5426..df06ce4b3 100644 --- a/docs/import.md +++ b/docs/import.md @@ -132,22 +132,12 @@ success ### Supported Quantizations -- `q4_0` -- `q4_1` -- `q5_0` -- `q5_1` - `q8_0` #### K-means Quantizations -- `q3_K_S` -- `q3_K_M` -- `q3_K_L` - `q4_K_S` - `q4_K_M` -- `q5_K_S` -- `q5_K_M` -- `q6_K` ## Sharing your model on ollama.com From 2307fc2bcd8e9067ed5575d7570b69fff9cc4f38 Mon Sep 17 00:00:00 2001 From: Daniel Hiltgen Date: Sat, 24 May 2025 13:17:53 -0700 Subject: [PATCH 13/54] tests: drop llama3.2-vision embedding tests (#10837) --- integration/testdata/embed.json | 1 - 1 file changed, 1 deletion(-) diff --git a/integration/testdata/embed.json b/integration/testdata/embed.json index a6ef65aa7..eea33bdfc 100644 --- a/integration/testdata/embed.json +++ b/integration/testdata/embed.json @@ -15,7 +15,6 @@ "falcon:latest": [-0.145434, 0.551458, 0.579686, 0.056737, 0.122334, -0.098554, 0.453500, 0.152636, 0.006995, 0.353793, -0.057853, 0.141001, -0.251169, 0.123486, -0.386756, 0.380073, 0.113592, 0.246952, 0.221665, 0.222362, 0.110978, -0.156950, -0.095453, -0.446011, 0.380302, -0.181973, -0.355711, 0.083788, 0.529327, 0.396823, -0.235883, 0.463392, -0.453481, 0.111083, -0.109795, 0.096437, 0.184561, -0.006152, -0.404893, -0.505164, -0.512443, -0.248063, 0.264552, 0.211518, -0.408603, -0.177284, -0.358847, 0.055804, 0.302788, -0.161492, -0.030894, -0.017345, -32.815334, -0.083117, -0.250543, 0.163018, 0.198646, -0.159876, -0.472978, -0.341143, -0.634069, 0.191124, -0.064307, 0.090823, -0.458379, 0.471753, -0.101117, 0.290473, 0.348141, 0.077560, 0.459839, 0.713714, -0.385911, 0.357724, -0.028543, 0.215573, 0.420039, -0.160133, 0.720215, 0.377099, 0.286783, 0.130918, -78.763008, 0.235381, 0.492549, 0.421542, 0.292254, -0.320543, 0.125360, 0.286009, -0.266907, -0.161205, 0.468458, 0.075248, -0.607871, -0.498977, -0.248155, 0.137575, 0.217211, 0.167880, 0.721968, 0.437299, -0.006375, -0.114837, -0.217905, 0.325793, 0.548343, -0.304491, 0.050522, -0.296999, -0.015102, -0.066116, 0.326557, -0.683608, 0.848642, 0.139344, -0.386741, 0.095375, 0.137059, -0.524803, -0.308266, 0.207935, 0.910517, -0.047032, 0.016144, -0.282897, 0.030230, -0.118328, -0.216214, -0.303300, 0.196013, -0.501000, 0.369998, -0.028593, 0.309059, 0.496592, -0.330565, -0.581806, 0.117767, 0.187372, -0.144573, -0.227210, 0.181474, -0.181403, 0.144057, 0.269330, 0.046298, -0.699166, 0.538736, 0.174926, -0.561055, -0.088680, -0.007410, 0.730400, -0.471273, -0.197240, -0.027064, -0.109930, 0.149596, -0.431747, -0.033780, 0.269468, 0.410048, -0.164335, 0.079866, -0.167682, -0.260156, 0.352913, 0.227972, -0.491290, -0.305480, -0.233817, 0.048377, -0.192452, 76.728951, -0.184723, -0.035661, 0.021605, -0.363392, -0.380687, -0.095017, -0.363713, -0.331915, -0.450960, -0.384010, 98.522270, -0.376337, 0.332839, -0.049045, 0.044364, -0.076163, -0.176768, 0.273568, 0.060393, 0.105446, 0.286757, -0.061164, -0.454434, 0.415012, -0.140746, -0.040113, -0.565394, -0.060276, -0.234516, 0.312736, -0.069791, 0.352157, 0.496571, -0.248295, 0.169454, 0.254744, -0.125044, 0.095099, 0.158710, -0.174195, 0.307172, 0.600827, 0.102560, 0.584701, 0.191773, -0.210491, 0.224133, -0.133474, 0.060811, -0.040503, -0.486995, -0.257410, 0.131609, 0.188029, -0.003900, -0.226279, 0.014537, -0.214499, -0.285907, -0.500149, 0.220990, -0.061304, -0.449743, 0.320928, 0.395599, -0.405548, 0.214390, 0.296444, 0.175481, 0.196306, -0.181649, -0.175221, 0.562277, -0.083596, -0.135095, -0.226197, 0.060174, 0.425703, -0.263677, -0.442736, -0.848780, 0.140745, 0.158711, -0.246688, 0.513777, -0.133425, 0.150256, -0.165640, -0.204918, -0.390736, -0.400310, 0.078730, 0.042572, 0.127664, 0.146947, -0.031153, -0.076668, 0.239822, -0.114565, 0.338568, 0.057678, 0.276141, -0.158014, -0.047334, 0.813145, 0.096568, -0.204652, 0.069818, -0.162866, 0.474288, 0.010297, -0.042806, 0.708678, 0.266313, 0.264811, 0.178802, 0.090678, -0.153934, -0.245041, -0.249606, -0.606384, 1.116905, -0.250234, -0.134183, 0.109237, 0.162539, 0.071664, -0.028506, 0.256520, -0.270116, 0.370725, 0.072267, 0.595097, -0.444489, -0.306348, 0.375055, 0.052390, 0.689186, 0.141866, -0.022982, -0.028208, 0.692657, 0.168249, -0.289483, -0.068967, -0.145280, -0.667039, 0.606857, 0.572388, -0.461176, -0.083282, 0.357757, 0.263421, -0.338740, 0.014035, 0.136918, -0.027329, 0.405532, 0.123614, 0.228298, -0.604249, -0.259012, 0.191833, -0.188005, 0.052887, -0.036500, 0.051033, -0.017828, -0.013952, -0.076532, -0.322442, 0.034422, -0.158676, -0.380441, 0.068665, 37.625778, -1.467730, -0.198724, -0.366973, 0.094293, -0.008216, -0.019456, 0.045300, 0.241962, 0.206565, 0.200205, -0.170838, -0.239032, -0.092126, 0.364009, 0.078817, 0.044788, 0.307262, -0.396150, 0.345320, -0.246881, -0.092816, -0.252418, -0.180483, -0.254689, 0.004093, -0.200098, 0.210746, 0.029551, 0.245714, -0.063405, -0.003852, -0.331452, 0.648124, -0.156812, 0.173493, -0.454535, -0.054677, 0.309534, -0.462142, 0.451003, -0.153110, 0.369131, 0.301209, 0.173910, -0.131474, -0.194955, -0.717491, -0.016214, 0.543446, 0.239048, -0.154631, 0.153298, -0.721207, 0.015260, -0.547927, -0.272240, -1.158423, -0.387883, 0.179015, 0.290848, -0.178274, -0.537356, 0.123705, 0.337805, -0.196415, -0.124260, 0.274265, -0.027145, -0.112975, 0.271123, 0.200173, 0.309503, 0.368623, -0.130785, 0.145833, -0.102706, -0.194807, -0.355944, 0.233473, -0.422869, -0.249461, 0.275593, 0.261122, -0.041622, 0.321849, 0.351184, -0.731175, -0.455722, 0.376331, -0.073822, -0.400349, 0.297684, -0.254106, -0.179405, -0.740918, 0.396933, -0.624790, -0.245724, -0.758950, -0.761523, 0.171997, 0.582886, 0.122688, 0.454277, 0.393659, -0.125691, 0.526288, 0.245751, 0.236661, -0.268370, 0.259706, 0.471990, 0.070545, 0.262786, -0.195632, -0.373156, 0.233073, -0.319479, -0.180365, -0.356844, 0.501312, -0.404738, -0.253011, 0.381910, -0.230624, 0.322234, 0.322426, -0.406919, 0.278409, 0.189779, -0.477614, 0.398916, 0.282304, -0.097129, -0.206756, 0.078960, -0.140931, -0.327043, -0.043203, -0.238777, 0.054524, 0.345428, 0.379250, -0.028527, -0.346552, 0.135533, -0.324663, 0.097073, 0.405679, 0.144856, 0.203372, -0.135059, -0.262663, -0.038000, 0.389909, -0.832866, -0.061717, 0.439580, -0.620867, 0.043219, -0.032121, 0.448644, -0.445477, 0.479233, -0.349839, 0.376309, 0.034940, -0.093427, -0.018762, 0.237700, -0.175609, 0.521672, 0.212652, -0.147365, -0.611912, 0.433886, -0.218107, 0.044754, 0.199969, -0.084820, -0.180790, 0.020706, -0.192604, 0.911501, -0.205838, -0.328527, -0.468358, -0.292061, -0.165838, -0.239964, 0.134404, -0.085968, 0.593265, -0.035909, 0.166329, -0.038361, -0.180849, 0.134523, -0.347261, -0.383150, 0.203595, -0.527209, 0.655002, 0.036892, -0.271854, -0.722635, -0.183820, 0.041028, 0.534431, -0.321399, -0.418089, 0.205068, 0.079445, -0.313515, 0.291001, 0.376464, -0.024004, 0.494628, 0.302450, 0.075265, -0.267347, 0.131573, -0.295871, 0.160672, -0.115069, 0.890312, -0.041931, 0.229802, 0.140957, -0.566682, 0.494251, 0.228695, 0.119960, -0.250476, 0.542480, 0.438343, -0.089255, 0.539449, -0.477021, 0.099634, -0.404685, 0.306461, -0.043212, 0.233839, 0.048056, -0.186367, 0.199987, 0.047861, -0.289694, -0.537992, -0.030393, -0.089514, 0.143944, 0.128985, -0.247290, -0.148287, -0.007974, 0.302635, -0.353074, 0.140356, 0.267783, -0.438974, 0.037830, 0.136492, 0.285023, -0.161996, -0.577940, 0.466486, 0.099931, -0.207497, 0.478511, -0.477054, 0.378640, 0.114531, -0.454839, 0.205362, -0.264606, -0.345706, -0.495824, 0.015153, 0.063968, 0.570447, 0.097192, -0.171304, 0.976958, -0.167555, 0.102738, -0.436587, -0.416574, 0.246487, -0.708184, 0.023123, 0.282953, -0.282671, -0.100751, 0.016980, 0.458428, 0.210551, 0.728111, -0.064927, -0.518779, 0.023455, -0.523534, -0.352629, -0.204977, -0.031492, 0.492380, 0.037194, 0.275549, 0.049998, -0.042273, 0.024827, -0.391516, -0.194958, -0.433102, -0.268930, -0.385709, 0.035979, -0.236354, 0.036609, -0.106188, -0.051624, 0.874279, -0.019348, 0.500591, -0.124548, 0.324765, -0.367921, -0.222806, 0.540817, -0.078370, 0.043867, 0.420149, -0.059615, -0.440197, -0.087696, -0.292653, 0.111633, 0.670683, 0.455422, 0.189336, -0.042270, -0.260230, 0.051045, -0.335735, -0.506230, -0.358802, -0.266131, -0.203469, 0.257531, -0.079949, -0.371272, -0.413529, 0.099179, 0.181491, 0.150417, 0.225561, -0.346218, 0.386289, -0.052629, 0.298301, 0.905319, 0.350629, 0.162920, -0.197367, 0.032053, 0.451291, 0.156002, 0.093688, -0.061183, -0.513632, 0.197035, -0.259313, 0.299369, -0.044861, -0.141547, -0.285648, 0.202531, 0.323211, -0.128848, 0.146046, -0.771048, 0.315781, 0.132139, -0.224002, -0.046500, 0.392186, 0.015927, -0.539539, -0.217051, -0.165655, -0.112836, -0.694149, 0.339086, -0.414199, -0.237005, -0.950271, -0.279200, -0.627571, 0.069664, 0.370548, -0.062389, 0.586872, 0.213491, 0.115808, -0.041785, 0.070284, 0.475451, 0.620675, 0.153083, 0.570752, 0.077146, -0.593140, -0.395619, -0.169177, -0.077108, 0.033796, 0.110675, -0.144388, -0.484813, 0.101560, 0.118790, 0.585920, -0.051071, 0.522129, -0.096738, 0.039210, 0.130928, -0.165276, 0.021268, -0.138029, 0.452325, 0.340640, -0.240428, 0.037821, -0.356228, 0.230344, -0.362754, -0.077494, 0.106675, -0.458798, -0.373419, 0.031627, -0.154582, 0.372776, -0.041724, 0.265408, 0.351702, -0.294240, 0.822546, 0.024259, 0.029190, 0.116972, -0.298376, -0.022028, 0.036926, -0.000887, 0.198864, -0.196297, -0.319581, 0.104491, -0.003430, 0.157759, 0.118287, 0.417707, 0.329480, 0.477700, 0.300714, -0.037461, -0.082834, -0.691333, 0.119406, 0.333651, -0.015797, -0.241354, 0.322229, -0.316640, 0.336315, -0.333795, 0.152463, -0.063831, -0.093148, -0.190586, 0.866467, 0.206135, -0.333121, -0.334417, 0.427795, 0.433384, 0.030511, -0.081860, -0.327655, 0.321278, 0.308877, -0.146296, 0.250392, 0.352685, -0.261779, -0.391084, 0.343821, -0.623625, -0.232097, 0.730628, -0.009076, -0.278967, -0.024484, 0.126125, -0.272171, 0.395506, 0.385768, -0.485491, 0.060047, -0.462739, -0.056647, -0.384519, -0.158643, -0.362551, 0.475326, -0.029004, -0.547719, 0.048068, 0.214930, -0.050708, -0.737851, 0.249612, -0.273211, 0.547692, -0.119554, -0.489587, -0.183487, -0.422387, -0.021538, -0.357277, 0.442413, 0.742838, -0.306633, -0.417351, 0.665328, 0.195444, -0.137114, -0.503417, 0.106885, 0.313572, -0.132145, 0.361419, 0.209603, -0.503253, -0.481884, 0.218992, -0.205752, -0.239932, -0.071110, 0.198684, 0.044104, 0.170742, 0.087633, 0.306695, 0.128361, -0.694840, -0.178137, -0.062519, 0.269586, -0.420635, 0.441342, -0.307177, -0.113335, 0.545861, 0.245667, -0.061284, 0.463628, 0.188273, -0.425786, -0.599940, -0.498917, -0.097796, 0.110285, 0.832950, -0.219720, 0.157499, 0.090185, 0.919885, -0.057676, 0.432318, -0.276865, 0.225659, -1.100891, 0.191885, -0.538571, -0.070240, 0.178796, -0.850664, -0.281632, 0.101154, 0.292607, -0.697670, 0.065382, 0.243841, -0.446409, -0.747985, 0.421588, 0.134048, 0.014038, 0.012480, 0.093830, 0.065262, -0.028793, -0.460956, 0.001223, -0.435688, -0.216765, 0.092236, -0.205991, -0.346419, -0.150538, -0.115917, -0.034806, 0.121649, -0.155723, 0.509418, 0.161882, -0.247912, -1.040411, 0.113688, -0.232234, 0.083722, -0.356446, 0.171696, -0.551805, 0.644128, -0.361454, -0.069536, 0.344402, -0.144846, 0.593874, 0.046123, 0.071060, 0.044171, 0.109494, 0.162099, 0.016161, 0.160383, -0.657956, 0.426369, 0.430661, 0.167817, 0.371546, 0.038148, 0.175910, 0.308253, -0.379426, 0.308540, 0.028549, -0.238940, -0.216455, 0.583075, -0.325968, 0.251503, 0.521912, 0.405485, -0.058058, -0.085051, 0.486884, 0.177756, 0.437350, -0.531478, -0.309321, -0.053716, -0.577053, -0.135426, 0.037255, -0.151116, -0.249868, 0.044155, -0.143193, -0.256241, -0.094862, -0.071529, -0.179807, 0.177920, 0.322324, -0.038461, 0.146140, -0.170251, -0.189690, 0.621307, 0.123934, 0.030048, -0.024102, 0.172103, 0.412975, -0.478826, -0.629470, 0.323721, -0.306728, -0.006467, 0.321414, 0.400523, -0.001823, 0.011803, -0.120583, 0.601702, 0.185840, 0.700491, 0.149545, 0.067835, 0.470991, 0.664574, -0.552964, -0.316984, -0.231059, -0.300164, -0.558253, -0.371432, -0.427449, 0.150378, 0.205915, 0.007517, 0.848489, -0.270204, 0.645367, -0.190823, 0.118916, -0.048641, -0.555063, 0.180961, 0.193901, 0.520572, -0.087776, 0.935664, -0.117911, 0.409115, -0.214746, -0.062976, -0.582564, -0.081324, -0.010914, 0.697559, 0.827275, -0.194161, -0.525253, 0.123935, 0.110752, -0.243882, -0.464774, 0.015078, -0.350227, -0.113486, 0.011749, 0.153878, 0.241713, 0.309567, 0.159847, 0.382380, -0.121834, 0.159451, -0.704771, 0.006205, 0.008906, 0.346081, -0.160726, 0.367822, -0.023612, -0.010030, 0.374448, -0.019864, -0.096305, -0.143692, 0.101554, 0.203875, -0.122353, -0.456235, 0.368918, -0.026021, 0.096329, -0.418561, -0.096026, -0.323244, -0.289808, -0.177779, -0.380370, -0.277077, 0.225156, 0.373779, 0.756219, -0.338806, -1.064188, 0.086767, 0.112865, 0.410580, -0.160529, 0.685520, 0.343053, 0.015070, 0.034452, 0.050372, 0.006650, 0.317398, 0.658390, -0.431091, 0.506827, -0.100293, -0.258241, -0.098443, 0.440216, -0.351024, -0.425316, 0.190989, -0.198299, 0.598305, 0.592949, -0.761293, 0.291733, 0.298414, -0.596820, -0.056470, 0.506640, 0.614545, -0.038101, 0.304769, 0.047714, -0.178597, 0.606069, -0.020600, -0.324800, -0.517904, -0.244066, -0.781475, -0.347456, -0.177254, 0.442952, -0.502207, -0.313296, -0.221216, -0.158749, -0.000308, 0.210433, -0.293535, 0.143593, -0.187607, 0.410221, -0.101963, 0.038983, 0.216233, 0.109033, 0.071577, -0.274980, -0.519410, 0.748628, 0.132990, 0.017301, -0.318737, 0.462538, -0.219474, -0.257446, 0.106117, -1.033327, 0.555366, 0.314015, -0.099034, -0.101012, 0.316906, 0.525460, -0.067212, -0.234236, -0.594703, -0.301416, 0.049783, 0.360627, -0.340874, -0.108111, -0.030016, -0.068959, 0.419880, 0.079512, 0.136864, -0.870394, 0.190759, 0.264447, -0.055466, -0.274267, 0.294311, 0.145651, 0.181500, 0.068148, -0.079297, 0.116602, 0.398743, -0.596057, -0.283995, -0.631194, 0.272831, 0.028005, -0.269716, 0.821832, -0.264261, 0.296154, 0.000392, 0.268086, 0.256604, 0.438840, -0.025024, 0.229269, 0.372718, -0.111645, 0.275149, 0.074714, -0.149943, -0.677116, -0.423984, -0.055012, 0.807868, -0.793449, -0.288004, -0.217445, -0.417038, 0.048146, 0.271043, -0.240449, 0.003111, 0.200560, 0.515866, -0.112648, 0.480673, -0.153305, 0.162125, -0.360713, -0.281660, -0.433440, 0.062965, -0.287887, 0.376615, -0.010636, -0.070215, -0.413233, -0.407751, -0.569790, 0.620695, -0.445826, -0.266545, -0.143776, 0.556380, -0.358060, 0.091419, 0.154067, -0.424598, 0.678844, 0.586404, -0.293496, -0.565051, 0.211552, -0.060263, -0.154836, 0.118186, 0.659808, 0.048121, 0.028627, 0.310143, -0.108596, 0.233567, -0.028822, 0.211269, -0.069226, 0.083919, 0.067399, 0.551407, 0.423780, -0.168638, -0.212640, -0.304896, -0.148888, 0.044441, -0.146128, -0.221889, 0.151668, -0.279946, -0.454766, -0.137937, 0.329202, 0.392256, -0.123300, -0.013104, 0.020592, -0.154003, -0.141471, 0.245494, 0.342663, -0.524716, -0.256356, 0.096822, 0.377818, 0.058905, 0.199189, 0.192921, 0.046306, -0.310740, 0.375515, 0.113114, 0.067435, -0.171354, -0.187886, 0.098279, -0.443635, -0.418073, -0.076781, -0.216082, -0.459061, 0.174019, 0.092634, 0.095597, 0.229115, -0.063759, 0.166816, 0.224667, 0.389135, 0.201666, 0.280591, -0.505003, -0.199995, 0.164115, -0.018507, 0.216536, -0.228796, -0.591178, -0.108026, -0.061647, -0.155654, 0.063727, -0.290692, 0.071313, 0.164409, 0.025392, 0.333114, 0.058157, 0.341020, -0.008189, -0.657713, -0.469880, 0.380310, -0.422738, 0.146785, 0.138773, -0.181386, -0.080411, 0.223552, 0.362902, -0.642274, -0.216622, 0.244858, -0.053043, -0.255551, 0.420415, -0.162121, 0.178924, 0.692984, -0.513166, -0.353268, -0.566853, 0.654280, -0.097715, 0.617112, 0.737402, -0.252336, -0.919382, -0.002810, -0.243296, -0.697557, -0.152351, -0.020530, -0.044934, -0.028899, -0.191319, -0.194357, -0.176970, 0.400549, 0.291772, -0.242575, 0.424298, 0.457313, -0.631985, -0.030793, 0.099553, 0.011074, 0.058982, 0.193177, 0.192687, -0.396982, -0.226565, -0.427594, 0.366640, 0.581517, -0.384248, -0.604846, 0.314109, 0.230639, -0.521342, 0.201063, 0.692748, 0.048733, 0.597786, 0.063185, 0.225998, -0.180704, 0.329972, 0.150493, -0.042691, -0.159609, -0.309584, -0.275296, 0.984232, -0.030626, 0.040339, -0.253096, 0.007542, -0.286120, -0.222316, 0.158408, 0.383668, 0.429613, -0.571129, -0.086979, -0.774914, 0.035450, 0.399168, 0.141715, 0.261576, 0.757710, 0.092665, -0.444902, 0.364449, -0.026400, 0.486443, -0.191074, 0.267344, 0.391305, 0.019776, -0.162476, 0.011955, -0.169685, 0.360746, -0.057660, 4.487291, -0.752153, 0.426690, -0.514092, -0.135662, 0.128860, -0.352554, 0.339426, 0.187895, -0.105337, -0.067870, -0.561997, 0.384668, 0.139748, -0.076339, -0.383534, 0.193489, 0.217438, 0.310760, -0.471129, 0.408550, -0.783918, 0.924570, 0.544810, 0.173359, 0.325069, 0.118672, 0.157661, 0.495307, 0.043571, 0.420747, 0.461203, 0.030940, 0.079866, 0.202492, -0.143818, 0.388617, 0.284566, 0.085482, -0.424856, 0.169660, 5.002161, 0.496126, 0.368753, -0.291746, 0.049863, 0.052142, 0.562981, 0.643301, -0.097461, -0.153821, -0.237668, -0.369237, 0.234187, 0.218032, -0.617891, -0.360784, 0.184486, 0.035056, -0.343886, 0.100110, 0.015195, -0.149124, 0.577815, 0.609786, -0.190856, 0.157805, -0.111564, 0.056562, -0.264586, -0.228257, 0.143420, -0.409625, -0.407802, -0.193288, -0.178765, 0.457437, -0.001436, 0.413320, 0.456115, 0.493243, -0.467128, -0.516562, -0.063919, 0.772535, -0.142793, -0.273115, -0.326627, -0.252157, 0.386950, 0.000773, 0.080755, 0.169150, 0.207458, -0.113956, -0.269044, -0.411961, 0.196148, 0.279117, 0.476607, 0.002481, -0.446647, -0.131562, -0.290507, -0.031059, 0.522627, 0.374723, -0.189284, -0.022486, 0.121929, -0.004285, 0.320933, 0.433629, -0.872581, 0.510533, -0.047597, -0.043823, -0.172759, 0.340392, 0.585445, -0.344459, 0.179457, 0.052765, -0.247181, -0.092283, 0.166314, -0.082850, 0.624757, 0.408819, 0.517236, 0.592212, 0.266402, 0.063619, -0.332631, 0.093216, -0.002696, 0.496564, 0.017645, -0.081635, -0.045906, 0.131908, -0.360834, -0.074638, 0.884801, -0.112605, 0.152780, 0.241896, 0.178782, 0.302635, 0.096668, -0.569904, -0.235891, 0.697626, -0.153779, 0.083589, 0.118558, 0.148341, 0.138840, -0.483483, 0.039876, -0.215070, 0.378216, 0.477800, -0.376276, -0.107364, -0.056474, 0.634841, -0.829850, -0.132656, 0.166909, 0.284389, -0.751289, -0.661793, 0.215448, 0.434656, 0.023345, -0.133867, 0.439496, -0.496212, -0.117264, -0.017485, -0.372388, 1.472339, -0.363553, -0.193867, 0.281123, 0.351186, 0.103060, -0.298733, 0.369226, 0.295971, -0.239668, -1.152384, 0.448451, 0.275586, -0.237399, 0.174515, 0.195369, 0.043802, -0.264791, 0.013253, 0.040941, 0.027880, -0.142671, 0.074643, -0.553351, 0.129982, -0.100195, 0.180619, 0.071297, -0.371206, -0.193323, -0.468507, -0.805926, 0.184589, 0.080032, 0.084080, 0.275789, 0.192904, -0.078385, 0.290711, -0.027608, 0.440038, -0.587812, -0.004581, 0.162749, -0.407841, 0.368317, -0.112203, 0.299965, 0.172529, -0.076716, 0.450040, -0.303669, 0.169699, 0.115787, -0.083244, 0.287229, -0.098873, 0.045379, -0.142650, 0.225791, 0.117558, 0.331856, -0.449384, 0.030248, 0.661376, -0.147196, 0.219943, 0.449944, 0.153362, 0.580988, -0.014734, 0.219984, -0.031598, -0.087134, 0.624181, 0.321511, 0.159627, 0.065644, 0.052018, -0.334922, -0.194216, -0.214867, 0.373563, 0.569381, 0.320083, -0.218419, 0.392195, -0.208969, -0.043425, 0.549654, -0.405465, -0.393805, 0.037566, -0.388948, -0.224721, -0.294911, 0.308722, -0.081107, 0.685822, -0.048916, -0.165033, 0.142796, -0.269439, -0.222129, -0.724417, -0.286839, 0.608340, 0.327775, -58.517979, -0.544269, -0.001866, -0.332386, 0.199435, 0.252479, -0.223065, -0.405478, -0.249875, -0.104632, -0.554372, 0.442492, 0.092624, -1.184105, 0.306272, 0.094480, 0.180830, -0.188470, 0.199765, -0.158313, 0.219714, 0.490681, -0.341416, -0.366322, -0.456658, -0.166933, -0.581015, -0.068403, -0.253523, -0.303116, 0.084059, 0.248953, 0.151603, 0.113430, 0.826785, -0.997589, -0.021810, -0.476477, 0.258189, 0.772523, 0.136442, -0.239684, 0.186631, 0.024477, -0.110479, -0.279112, 0.227488, 0.287025, -0.114060, -0.046808, 0.235135, -0.282328, 0.259069, -0.037700, 0.412282, 0.450889, -0.564557, 0.362352, -0.249231, -0.283077, 0.596018, 0.210083, -0.160349, -0.145695, 0.279737, 0.432935, -0.060723, 0.205922, -0.077641, -0.005025, 0.032035, -0.147291, -0.089267, 0.022549, 0.096784, 0.316596, -0.335415, -0.154303, 9.210744, 0.439763, -0.474908, 0.547985, 0.333362, -0.083074, -0.113624, -0.358873, -0.001362, 0.342030, 0.487156, -0.212562, 0.381375, 0.187135, -0.076945, -0.455437, -0.128274, -0.070635, -0.241333, -0.100881, -0.181602, -0.364256, -0.003553, 0.078503, -0.229145, -0.102308, 0.197712, 0.027701, -0.779926, 0.536838, 0.599103, 0.012468, -0.057213, 0.015195, 0.180001, 0.458210, -0.035604, -0.381750, -0.162652, 0.062134, -0.018506, 0.322445, 0.430900, 0.494851, -0.001246, -0.063209, 0.027377, 0.572648, 0.501041, -0.330562, 0.005837, -0.250195, -0.197133, -0.322219, -0.241606, 0.777974, -0.214089, -0.517721, 0.054999, 0.079865, -0.392691, 0.436211, 0.170465, 0.012087, -0.570184, 0.342888, -0.248057, -0.206725, -0.031366, -0.455187, -0.014615, 0.157380, 0.836096, -0.526227, 0.272770, -0.238688, -0.423359, 0.184629, -0.558110, 0.024373, -0.151099, 0.561669, 0.598668, 0.297249, 0.027630, -0.355015, 0.242124, 0.076283, 0.281585, 0.539186, 0.105147, 0.421329, 0.097530, 0.683345, -0.052615, -0.135516, -0.139486, -0.133028, -0.413589, 0.058511, 0.174991, 0.026137, -0.128775, 0.032074, 0.079121, -0.097758, -0.100383, -0.140296, 0.653418, -0.458114, 0.115476, 0.015241, -0.171333, -0.002086, -0.215181, 0.295451, 0.152256, -0.074834, -0.073559, 0.242919, -0.361564, -0.105575, -0.216026, 0.285056, 0.216080, -0.017842, -0.203024, 0.165294, -0.153601, -0.077933, -0.115630, 0.081592, 0.049974, -0.676193, 0.314703, -0.164646, -0.030029, -0.488286, -0.197653, -0.107439, 0.057860, -0.215854, 1.589793, 0.217210, 0.187354, -0.106198, -0.019009, -0.011475, -0.016074, -0.082618, -0.361909, -0.087632, -0.064170, 0.235486, 0.430535, 0.326502, -0.158178, 0.298049, -0.182466, 0.113623, -0.318923, 0.001735, 0.340390, 0.232367, 0.124780, -0.329235, -0.258752, 0.232064, 0.057419, 0.071112, -0.393735, 0.243256, 0.444519, 0.086050, 0.239353, 0.491001, 0.005872, 0.332629, 0.355181, 0.453798, -0.406120, -0.000941, -0.504079, 0.147405, 0.410417, -0.263093, -0.082063, -0.330414, -0.024870, 0.603955, 0.011611, 0.247248, -0.437377, 0.374511, -0.386487, -0.052682, -0.704873, 0.137623, -0.302090, 0.611321, 0.347271, -0.128144, -0.070892, 0.555710, 0.235903, 0.620517, 0.356831, -0.088181, -0.160007, 0.040110, -0.402164, -0.132727, -0.304703, -0.037726, 0.249084, 0.463306, 0.410124, 0.206482, 0.084820, -0.448904, 0.441546, 0.493072, 0.177585, -0.202363, -0.028921, 0.279302, 0.206562, 0.210913, -0.330171, -0.465549, 0.120795, 0.345071, 0.735387, 0.512654, -0.197911, 0.433159, 0.039721, 0.779497, -0.224703, 0.258248, -0.097690, -0.123420, -0.614652, -0.402285, 0.541440, 0.066245, 0.084928, 0.117902, -0.200090, 0.050084, 0.132697, -0.505208, 0.336530, 0.370033, -0.026980, -0.182485, 0.137949, 0.097208, -0.373811, -0.432984, 0.365751, 0.116326, 0.786149, -0.336528, -0.143944, 0.276964, -0.337612, -0.537772, -0.135477, -0.157744, 0.480776, -0.016617, 0.178157, 0.447851, -0.460623, -0.082251, -0.066533, -0.221613, 0.263356, 0.364406, 0.033944, -0.270242, 0.301467, -0.181274, -0.774562, -0.336110, 0.013128, 0.379040, -0.491963, 0.510146, 0.115073, 0.120280, 0.541961, 0.340013, -0.030580, 0.080096, 0.215101, 0.109936, -0.498924, -0.311070, 0.359271, 0.007063, -0.440929, 0.494422, -0.285638, 0.636720, -0.141487, 0.515842, -0.552986, -0.224828, 0.077687, -0.618073, -0.191511, -0.243853, 0.518603, -0.099494, 0.009308, -0.480245, 0.537420, 0.192950, -0.265831, 0.051783, 0.308917, 0.155403, 0.018711, 0.211687, 0.407065, 0.442201, 0.136252, 0.084571, 0.095548, -0.225721, -0.000452, 0.138476, -0.149119, -0.664407, 0.180302, 0.024027, -0.168956, -0.060498, 0.176705, 0.197362, 0.810919, 0.100887, -0.343077, -0.250311, 0.202211, -0.533182, 0.565174, 0.071214, -0.071025, 0.294692, 0.345120, -0.369266, -0.363377, 0.094061, 0.053798, -0.675334, 0.105947, -0.366219, -0.233770, -0.146928, 0.059898, -0.260274, -0.351740, 0.509495, 0.466080, 0.533641, -0.283434, -0.170006, 0.497754, 0.113559, -0.234416, 0.006083, 0.091491, -0.393112, 0.243943, 0.068198, -0.135965, 0.071724, 0.267782, -0.299795, 0.225524, 1.235563, -0.060205, 0.328835, -0.436151, 0.408247, -0.826190, 0.398270, 0.206538, 0.275721, 0.916620, -0.380989, -0.155483, -0.165130, 0.108895, 0.192115, -0.203651, 0.378474, -0.264306, 0.075935, -0.263781, -0.194646, -0.089090, -0.463814, -0.084640, -0.277837, -0.342353, -0.024847, -0.192613, 0.186768, 0.111856, 0.178706, -0.120027, -0.541726, -0.337925, 0.118754, 0.238603, 0.208689, -0.145369, -0.150549, 0.653374, 0.094129, 0.106789, -1.053561, -0.196887, -0.240114, -0.745993, -0.152362, -1.204002, -0.082581, 0.305619, -0.058988, -0.598331, 0.185987, -0.060489, -0.423577, 0.511601, -0.325674, -0.120736, -0.115833, 0.159326, -0.285350, 0.046486, 0.190043, -0.050766, 0.455020, 0.403656, 0.323676, -0.360419, -0.349012, 0.049382, -0.285226, -0.303766, 0.506560, -0.419006, 0.309048, 0.217033, -0.084596, -0.243125, 0.644726, -0.052139, 0.160264, -0.375426, -0.394390, 0.098200, 0.259292, -0.638499, 0.021518, -0.129735, -0.101215, -0.224675, -0.042415, -0.101307, -0.089895, 0.188957, -0.548835, -0.143101, 0.536356, 0.472715, 0.358453, 0.414846, -0.370800, -0.235898, 0.262339, 0.449393, -0.263371, -0.217770, -0.226305, 0.047581, 0.003470, -0.028333, -0.130409, -0.138864, 0.496755, 0.231264, -0.030316, -0.476095, 0.256390, -0.104659, -0.518679, 0.125111, -0.423105, 0.494313, -0.181924, 0.242448, 0.280324, -0.129197, -0.375469, -0.433732, 0.134280, 0.389087, -0.506155, 0.527342, 0.194793, 0.178930, -0.333685, 0.181848, 0.243475, -0.612548, 0.289363, 0.029938, 0.110186, 0.255495, -0.144533, -0.387284, 0.345772, -0.311679, -0.045539, -0.294871, 0.597431, 0.305880, 0.200646, 0.282531, -0.158703, -0.103099, -0.169067, -0.008382, -0.079756, 0.170810, 0.399198, -0.066380, -0.000559, -0.004255, 0.231632, 0.278643, 0.301572, 0.158929, -0.362739, -0.174208, 0.090002, 0.658097, -0.753288, -0.115159, -0.916517, 0.345335, 0.315035, -0.185459, 0.099901, -0.345710, -0.024454, 0.149417, 0.203911, -0.621984, 0.475272, 0.381955, 0.285558, 0.620195, -0.230048, -0.193346, 0.426024, -0.082065, -0.505288, -0.264816, -0.614786, 0.032445, -0.211640, -0.743929, -0.352548, -0.282483, 0.083452, 0.155323, 0.063746, -0.081079, 0.143076, 0.304706, -0.302146, -0.586734, -0.018925, 0.135363, 1.372922, 0.022931, -0.203815, -0.249750, -0.096802, -0.322165, -0.087625, -0.354634, 0.158380, 0.221725, 0.366852, 0.150404, -0.269403, 0.051307, -0.228197, 0.501474, -0.508713, 0.310570, 0.105104, 0.175922, -0.160948, -0.532149, 0.789218, -0.296985, 0.621667, -0.034217, 0.269299, -0.473348, 0.419473, -0.168176, -0.059621, 0.347642, 0.095557, 0.119908, 0.189069, 0.072328, -0.592562, 0.205006, -0.456758, -0.051765, 0.186611, -0.559713, 0.498142, 0.081539, -0.126901, -0.188289, -0.577864, -0.548407, 0.401423, 0.509642, -0.104356, 0.108317, 0.898374, 0.040251, -0.698863, -0.078856, -0.188995, -0.003449, -0.061420, -0.096289, 0.266162, 0.108274, 0.346424, -0.441824, 0.258435, -0.142721, -0.439990, 0.212413, 0.165938, -0.020241, 0.303992, -0.042712, -0.240819, 0.041794, 0.002941, 0.399268, 0.036751, 0.134195, -0.134292, 0.143634, 0.516429, 0.132746, -0.935939, 0.153861, 0.304214, -0.235507, 0.430125, 0.079114, 0.117013, 0.439177, 0.716553, -0.093885, -0.044955, -0.330311, 0.550033, -0.535871, 0.010232, -0.314547, 0.230491, 0.654590, 0.102710, -0.238373, -0.516637, 0.401736, 0.037948, 0.276305, 0.443498, 0.228219, 0.352056, 0.012582, 0.220353, -0.407780, 0.438028, 0.208541, 0.175988, -0.270985, -0.525523, -0.548477, 0.057955, -0.194781, -0.003298, 0.269760, -0.233289, -0.540260, -0.169692, -0.484772, -0.114232, -0.401115, -0.451770, -0.181977, -0.505564, -0.083401, 0.376819, -0.421163, -0.201286, -0.602337, -0.222767, 0.614156, -0.016797, -0.176580, 0.508542, -0.081163, 0.134763, -0.419954, 0.037240, -0.185230, -0.205294, 0.193024, 0.417523, -0.698548, -0.807030, -0.149474, -0.094982, -0.464999, -0.068225, -0.432381, 0.094518, -0.034902, -0.376230, -0.190843, -0.014541, -0.007451, 0.503071, 0.039062, -0.476695, -1.198664, -0.111433, 0.107706, 0.328139, -0.316463, -0.217056, 0.050820, -0.164824, 0.319711, -0.208170, 0.614053, -0.104899, -0.555545, -0.287073, -0.262042, -0.200674, -0.006250, -0.231419, -0.642396, -0.137048, -0.145983, 0.069366, 0.240843, -0.658315, -0.597073, 0.463778, -0.183305, 0.119860, 0.648310, 0.299937, -0.208176, 0.022095, -0.278521, -0.177953, 0.202277, 0.155420, 0.234455, 0.239173, 0.262826, 0.519646, 0.006134, 0.463624, 0.502869, -0.273425, -0.230075, 0.078392, 0.024440, -0.299886, 0.079623, -0.031235, 0.077222, -0.575926, 0.647744, -0.685791, 0.246712, -0.258738, -0.268204, -0.050904, -0.216824, 0.030123, -0.059568, 0.032685, 0.456928, 0.447742, 0.000349, -0.330170, -0.283173, 0.339716, -0.715647, 0.350537, -0.348117, -0.425378, 0.275241, -0.037290, -0.301914, -0.418955, -0.057846, 0.038219, 0.490801, -0.169052, -0.531855, -0.126881, 0.214531, -0.016972, 0.486737, -0.307092, 0.060385, -0.312028, -0.121100, -0.474154, -0.033297, -0.117146, -0.145217, -0.534372, 0.215293, 0.089433, -0.014680, 0.038871, -0.454982, 0.468077, -0.157630, 0.546171, 0.098118, 0.372806, -0.065871, 0.243025, -0.094658, -0.116481, 0.125599, -0.121057, -0.072374, 0.656453, -0.041724, 0.033926, -0.041462, -0.456399, -0.186608, -0.389824, -0.091279, -0.003108, 0.092814, 0.340213, -0.175039, 0.666803, -0.261088, 0.368768, -0.252754, 0.163965, -0.585329, -0.243940, -0.321711, -0.133012, -0.253066, -0.076904, -0.283974, -0.197103, -0.403979, 0.147247, -0.062263, -0.075245, -0.562477, -0.205014, 0.222767, -0.054252, -0.329677, 0.815129, 0.127072, -0.426652, -0.394265, 0.127829, 0.049436, -0.196066, 0.009618, 0.540864, -0.206275, -0.050193, -0.484369, 0.663735, -0.174202, 0.404585, 0.052897, 0.197160, 0.118619, 0.260436, 0.124425, -0.006648, 0.607803, -0.084435, 0.182488, -0.911126, -0.399702, -0.234616, -0.425689, 0.574178, 0.028556, 0.346813, -0.227521, -0.021377, -0.578073, -0.755381, -0.108397, 0.211026, -0.040924, -0.294499, -0.208813, -0.170734, 0.164126, 0.122645, 0.010301, -0.073459, 0.233719, -0.109440, -0.098003, -0.205121, -0.087854, 0.028307, 0.532311, -0.228900, -0.152958, 0.527488, 0.364191, 0.192383, -0.351570, -0.234025, 0.440298, -0.060356, -0.425945, 0.171791, 0.221136, -0.396888, -0.277037, -0.666172, -0.037598, -0.010303, -0.425237, -0.211664, -0.486224, 0.277630, 0.023025, 0.408915, -0.195016, 0.047634, 0.102783, -0.485706, 0.308589, -0.411144, -0.053444, 0.714636, 0.113829, 0.033845, 0.443375, -0.068617, -0.390009, 0.075894, 0.448207, -0.177000, -0.023104, -0.175386, 0.555945, 0.029201, -0.457334, -0.357339, 0.378139, 0.238245, -0.110313, -0.500821, 0.104544, -0.281480, 0.587670, -0.373495, -0.031328, -0.065652, -0.002243, 0.044564, -0.489538, 0.399526, -0.477148, 0.174514, 0.234356, -0.095146, -0.111628, -0.140329, -0.014722, 0.120857, -0.577776, -0.435228, 0.355946, -0.013572, -0.721267, -0.317641, 0.217404, 0.607119, 0.134049, -0.278283, -0.041902, -0.119571, 0.305286, -0.412544, 0.012801, 0.406974, 0.057604, -0.505895, 0.571090, 0.132650, 0.261956, 0.170358, -63.303562, 0.072853, -0.519040, -0.178422, 0.119113, 0.378141, 0.063010, 0.112086, -0.068155, 0.143270, -0.695979, -0.185095, 0.029685, 0.101319, 0.243841, -0.099949, 0.537030, -0.507941, -0.072316, -0.282923, -0.095952, 0.277304, 0.279224, 0.180773, 0.588230, -0.293736, -0.419753, 0.124619, -0.439369, -0.203452, 0.322170, 0.218235, -0.135500, 0.153501, -0.101683, -0.188600, 0.075205, 0.135442, -0.469343, 0.088359, 0.188366, 0.252388, -0.129069, 0.600527, 0.231637, -0.129259, -0.012753, -0.375286, -0.155583, 0.010971, 0.067799, 0.301400, -0.200182, -111.950508, -0.203151, 0.112066, -0.561678, 0.199826, 0.240353, 0.024038, -0.051374, -0.191646, 0.126553, 0.212573, -0.397948, -0.364226, -0.430931, -0.275204, -0.089596, -0.264655, 0.148102, 0.095208, -0.158669, -0.152007, -0.115978, -0.417394, -0.417549, 0.121950, -0.005634, -0.209508, 0.026816, 0.097941, 0.356630, -0.327798, -0.577439, 0.128759, -0.181856, 0.341304, -0.401386, -0.326418, 0.910050, 0.602965, -0.501181, 0.499152, -0.081410, -0.445701, 0.283482, 0.128661, -0.162188, 0.477510, -0.479452, 0.445276, 0.613333, -0.230295, -0.706247, -0.385147, -0.216893, -0.355799, -0.426881, 0.294281, 0.029298, -0.469063, 0.032645, 0.377577, -0.136977, 0.229560, -0.248002, 0.231691, 0.195523, -0.626372, 0.487773, -0.046927, 0.258056, 0.435159, -0.148099, -0.214715, 0.064264, -0.436486, 0.581707, -0.436593, 0.105906, -0.026306, 0.014113, 0.179391, -0.207917, -0.343595, -0.005207, 0.196769, -0.042901, 0.063794, 0.336295, -0.545930, -0.357988, 0.289421, -0.300983, 0.284619, -0.280840, 0.172100, 0.115409, 0.575850, -0.238892, 0.714082, -0.077031, -0.006099, 0.256360, -0.335220, 0.010254, -0.108844, 0.427106, 0.144753, -0.501155, 0.062318, -0.124000, 0.221674, 0.834580, 0.110858, -0.012859, 0.568384, -0.610597, 0.660640, -0.116576, -0.069433, 0.786274, -0.164275, -0.076165, -0.051904, 0.149984, 0.419948, 0.000599, -0.677398, 0.008406, -0.954607, 0.185710, 0.442692, 0.498803, 0.072520, -0.947522, -0.119591, -0.001380, -0.499802, 0.019757, -0.521202, -0.370886, 0.026982, 0.187350, 0.188310, 0.041130, 0.150330, -0.069001, 0.405078, -0.064037, -0.170146, 0.371683, 0.318880, -0.644247, 0.235556, -0.033404, -0.507976, -0.227250, 0.002788, 0.061252, -0.474965, -0.231972, -0.367271, 0.486901, -0.053223, -0.059750, -0.304050, -0.058611, 0.157256, 0.175846, 0.395777, 0.008041, -0.273340, -0.205810, 0.243421, -0.182496, -0.391188, 0.377847, 0.187971, 0.037746, 0.198323, -0.077953, 0.698547, -0.498333, 0.330154, -0.068499, 0.217126, 0.308641, 0.351311, 0.470557, -0.105092, 0.424691, 0.009478, 0.033756, -0.260245, 0.310565, 0.035176, 0.096856, -0.544571, 0.565485, -0.450896, 0.238636, -0.755101, 0.350827, 0.415188, -0.067474, -0.598875, 0.159576, -0.031585, 0.304653, 0.366733, 0.508857, 0.094383, 0.099464, 0.119461, 0.115205, 0.158623, 0.196619, 0.236457, 0.215316, 0.612975, 0.200485, 0.058877, -0.025619, -0.205582, 0.714231, 0.136720, -0.388014, -0.196395, 0.079210, -0.423071, -0.123676, 0.474725, -0.050195, 0.970900, -0.171853, -0.083888, -0.179501, -0.064880, -0.223606, -0.006045, -0.080197, 0.252009, -0.350478, 0.026701, 0.014843, -0.027661, -0.832059, 0.008524, 0.424898, -0.018123, -0.540465, -0.423612, -0.029289, 0.013382, -0.661031, 0.262038, -0.232594, 0.666213, 0.276438, 0.294243, -0.137279, 0.595431, -0.343497, -0.335292, 0.270682, -0.050774, -0.357238, 0.464678, 0.166497, -0.214094, 0.412270, 0.237754, -0.456134, 0.069817, -0.240430, 0.211049, 0.282824, -0.002317, 0.005693, -0.191463, -0.583706, -0.221818, -0.012686, -0.338345, -0.070691, -0.581066, 0.068721, 0.214868, 0.449330, 0.286828, -0.724386, 0.094391, -0.135015, -0.060382, 0.069094, 0.477051, -0.039083, -0.416415, -0.160632, 0.307268, 0.644104, -0.157020, -0.185252, 0.103966, -0.107401, 0.200582, -0.256117, -0.471047, 0.576146, -0.426443, -0.473182, 0.363728, -0.448481, 0.198918, 0.262472, -0.481285, 0.124921, -0.790910, 0.555112, -0.416907, -0.686696, 0.157033, 0.151132, -0.101561, -0.165456, -0.120177, 0.040872, -0.353821, 0.464011, 0.356632, -0.641347, 0.002591, -0.529365, -0.375631, -0.088245, 0.157345, -0.168277, -0.077418, -0.104852, -0.224176, -0.583345, -0.200973, 0.330438, 0.095610, 0.044952, -0.111227, 2.766827, -0.075490, -0.004877, 0.122718, -0.211410, -0.255072, 0.014199, -0.044222, -0.232582, -0.213509, -0.058883, -0.111578, 0.575816, 0.382956, 0.168038, -0.660369, -0.317659, -0.287882, -0.159493, 0.303753, 0.161251, -0.084348, 0.177310, -0.023731, -0.208123, 0.375143, -0.130619, 0.013847, 0.084544, 0.385289, -0.844378, 0.783994, -0.133632, 0.453322, -0.368930, 0.468662, 0.074394, -0.085951, -0.492908, -0.102703, -0.070889, 0.019265, -0.132152, -0.490296, -0.154806, -0.438457, 0.106065, -0.362225, 0.098467, -0.300806, 0.082323, -0.066897, 0.534250, 0.646450, 0.452268, -0.157001, 0.063397, -0.202829, 0.173803, -0.187386, 0.587771, -0.348677, -0.511564, 0.085221, -0.224883, 0.157183, 0.246904, 0.638365, -0.585779, -0.287903, 0.340373, -0.030720, 0.061279, 0.074192, 0.301575, 0.018036, -0.261474, 0.063272, 0.015081, 0.398179, -0.035079, -0.370379, -0.052801, -0.530252, 0.093011, 0.002836, -0.697104, 0.546006, -0.332340, -0.122048, 0.289079, -0.541533, -0.503000, -0.054692, 0.134673, 0.006598, 0.017476, -0.134000, -0.175561, -0.208997, -0.003923, 0.051280, 0.086685, -0.240404, -0.357857, 0.243271, 0.566339, -0.173989, -0.064073, 0.295804, 0.233289, -0.092659, -0.665500, 0.190993, 0.167569, -0.278074, -0.244135, -0.168051, -0.463405, -0.177617, -0.034471, 0.007265, -0.297528, -0.075948, -0.047524, 0.122029, 0.211117, 0.120729, -0.014987, -0.632226, 0.213933, 0.220751, -0.416721, -0.432320, -0.204294, 0.197827, -0.168607, 0.240733, 0.250329, 0.280815, 0.289349, -0.309672, 0.147626, -0.490905, 0.131334, 0.044870, 0.244089, -0.017633, -0.301772, -0.072316, 0.146994, -0.293942, -0.420279, 0.161965, 0.148996, -0.126067, 0.491702, -0.119142, 0.063983, 0.395035, -0.213297, 0.140833, 0.196409, -0.048128, 0.017225, -0.505947, 0.664136, -0.202357, 0.317128, 0.168667, 0.093162, -0.380574, -0.092151, 0.044294, -0.870981, 0.241743, -0.536381, -0.090947, -0.648787, -0.355641, 0.165425, -0.238059, -0.596338, 0.266993, -0.339680, -0.382530, 0.291458, 0.768516, -0.176065, -0.569123, 0.096203, 0.148821, 0.546749, 0.138208, -0.365413, 0.518124, -0.101714, 0.804016, -0.252335, 0.263548, 0.344599, -0.042573, 0.189019, 0.249793, -0.605840, 0.522157, -0.359445, 0.140590, -0.134246, 0.302941, 0.272368, -0.274690, 0.744960, 0.187399, 0.031517, -0.522490, 0.492635, -0.265579, 0.556953, 0.002599, -0.475620, -0.273551, 0.061263, -0.107502, -0.058321, 0.335491, 0.093252, 0.338381, 0.580982, 0.004722, 0.306945, -0.357699, 0.204085, -0.403294, -0.339715, 0.038231, -0.318873, -0.157163, -0.286547, -0.014830, -0.408272, 0.646497, -0.297623, -0.716684, -0.037028, -0.359614, -0.607903, -0.080210, -0.243790, -0.372580, 0.327745, 0.174814, 0.007795, -0.323086, 0.278917, 0.193464, -0.183376, -0.158650, -0.150630, 0.237812, -0.554636, -0.307655, -43.962406, 0.747826, 0.170216, -0.375895, -0.110257, -0.561347, 0.516887, -0.050687, -0.239293, 0.075187, 0.214272, -0.583057, 12.469608, 0.157744, 0.439794, 0.303756, -0.331535, 0.010205, 0.261059, 0.047994, 0.124434, 0.148169, -0.040808, -0.038213, 0.568745, 0.759167, -0.161330, -0.147977, 0.209662, 0.184281, 0.034607, -0.435261, -0.914145, -0.448240, 0.502655, 0.171096, 0.162151, -0.330909, 0.644280, -0.230030, 0.400959, -0.398285, 0.243113, 0.819535, -0.015852, -0.416345, -0.035277, 0.475769, -0.331067, 0.097865, 0.243495, -0.104292, -0.019232, 0.216380, 0.644561, -0.040559, 0.058224, -0.022566, -0.086129, -0.033864, -0.107755, -0.452065, 0.232334, 0.260739, -0.491365, -0.583293, -0.385895, -0.032558, 0.422507, -0.104322, -0.199052, -0.450761, 0.188655, -0.015979, 0.101417, -0.108862, -0.445502, -0.133705, -0.177996, -0.163099, -0.119531, -0.171117, 0.427125, 0.238072, -0.338347, 0.201365, -0.159817, -0.007986, -0.612904, -0.590535, -0.537798, 0.119414, 0.221465, 0.378790, 0.208833, -0.014505, 0.360619, -0.548349, 0.216484, 0.256683, 0.385207, 0.706974, 0.283812, -0.470752, 0.290828, -0.313211, 0.595083, -0.867086, -0.099950, -0.006093, 0.328319, -0.174707, 0.061710, -0.149805, 1.078194, 0.353151, -0.642559, -0.077821, 0.227179, 0.333391, 0.007280, 0.522016, 0.349228, -0.273700, -0.092780, 0.020995, -0.248111, -0.178455, 0.095094, -0.057657, 0.430333, 0.131980, 0.129095, 0.375191, -0.076389, -0.125741, 0.223384, -0.356347, 0.353186, -0.023569, -0.242601, -0.549070, 0.215102, -0.368639, -0.316551, 0.274893, -0.523182, 0.007249, 0.256059, -0.174845, 0.097209, -0.344082, -0.195790, -0.205222, -0.601514, 0.215583, -0.090739, 0.279482, -0.269076, -0.103268, 0.192448, 0.343325, -0.083246, -0.548744, 0.427873, -0.101012, -0.624601, 0.132522, 0.332115, -0.602158, 0.100890, 0.363860, -0.269838, 0.326628, 0.123457, -0.670479, -0.476265, 0.158514, 0.236981, 0.179209, -0.047529, -0.307782, 0.100836, -0.113148, -0.232984, -0.231841, -0.206003, 0.090617, 0.506436, -0.088726, 0.060476, 0.503427, 0.701742, -0.349184, 0.198925, -0.189434, -0.242362, 0.060725, 0.099748, 0.038268, -0.391647, 0.629047, 0.516061, 0.209719, 0.497822, 0.031479, -0.268111, -0.201153, -0.449376, -0.248165, 0.497311, -0.135929, -0.426607, 0.067370, -25.183310, -0.217668, 0.244560, -0.113699, -0.034418, 0.279075, -0.087385, 0.090707, -0.057644, -0.038125, 0.556788, -0.185287, -0.570150, -0.224455, 0.177524, 0.295346, -0.079438, 0.456904, -1.079334, -0.205251, -0.241942, 0.030973, -0.412797, -0.039332, -0.078231, -0.625990, -0.752923, -0.103901, -0.260115, 0.343021, -0.274866, -0.338814, -0.084887, 0.440932, 0.021582, 0.504624, 0.122857, -0.114787, 0.298529, 0.105958, 0.489619, 0.023746, 0.315594, 0.449650, 0.291635, 0.456601, 0.009884, 0.411796, -0.053264, 0.003709, 0.146712, 0.551897, 0.747416, -0.292591, -0.418243, 0.235221, -0.076943, -0.266084, -0.249324, -0.296882, -0.198107, -0.645373, -0.300805, 0.539361, -0.779099, -0.247917, 0.190751, 0.161664, 0.476388, -0.065680, -0.249199, 0.418452, 0.159107, 0.429605, 0.161158, -0.122370, 0.595451, 0.982097, 0.093595, -0.179688, 0.247968, 0.154720, 0.151772, 0.511285, 0.429992, 2.756784, -0.477258, 0.210764, 0.060735, -0.206560, 0.050323, 0.226779, 0.488080, -0.322912, 0.215526, 0.079592, 0.581980, -0.074688, 0.156174, -0.439357, 0.177422, 0.084449, -0.244841, 0.041477, -0.124318, 0.522598, 0.331280, 0.219801, -0.001475, 0.191698, 0.235577, 0.397007, -0.140321, 0.127137, 0.326186, 0.080475, -0.198508, -0.016927, -0.647423, 0.118334, 0.151125, -0.260203, 0.370355, -0.372183, 0.684103, -0.044755, 0.411336, -0.508753, 0.137960, -0.578130, -0.308091, 0.662036, -0.223479, 0.528630, -0.088039, 0.301694, 0.581005, -0.571935, -0.326560, 0.074047, 0.560205, -0.517768, -0.118131, -0.825478, -0.052467, 0.087111, 0.156990, -0.125347, 0.200030, -0.232904, 80.721504, -0.353159, 0.366250, 0.661608, 0.076759, -0.675517, 0.160898, -0.327791, -0.218044, -0.286360, -0.110733, -1.071516, -0.166569, 0.236425, 0.076636, 0.110097, 0.025876, 0.136520, -0.289324, -0.052212, -0.116298, -0.413791, -0.261230, -0.305566, 0.213748, -0.213800, -0.181269, 0.551312, 0.220690, 0.754416, 0.246657, 0.569155, 0.251539, 0.176920, 0.161606, -0.531014, -0.195418, -0.095278, 0.553006, -0.337154, -0.577858, -0.308738, -0.046533, -0.260912, -0.037302, -0.461034, -0.245663, -0.409261, -0.124479, 0.556365, -0.376639, 0.100875, 0.092979, 0.564593, 0.361897, -0.145732, 0.145205, -0.663406, 0.029300, -0.542234, 0.114352, 0.108448, 0.103333, -0.109936, -0.598299, 0.671874, 0.007481, -0.067422, -0.047742, 0.225532, -0.224111, 0.202110, 0.233249, 0.132545, -0.249386, -0.251535, -0.008667, 0.486993, -0.417082, 0.657399, 0.234556, 0.103305, -0.333804, 0.601285, 0.350520, 0.213891, 0.336645, 0.362093, -0.728231, 0.082136, -0.602861, -0.315520, -0.174943, 0.939620, 0.721859, 0.055698, -0.052252, 0.005834, 0.295090, -0.253800, 0.549283, 0.408780, -0.037937, 0.030686, 0.093003, -0.155114, -0.104932, 0.285636, 0.134702, -0.235902, -0.073232, -0.016380, -0.196810, -0.231825, -0.039652, 0.191879, 0.708907, 0.046643, -0.679296, -0.272560, 0.220830, 0.632073, 0.011146, 0.338630, -0.006068, -0.417746, -0.790530, -0.044891, -0.112715, 0.119640, -0.555511, 0.385198, 26.227066, 0.327086, -0.359925, -0.246532, 0.315119, 0.315461, 0.184825, 0.874743, -0.243538, 0.034515, -0.099371, 0.384990, -0.588993, -0.329512, 0.202681, 0.108792, -0.522285, 0.317060, 0.258312, -0.340065, 0.353208, -0.226905, -0.467262, 0.197749, -0.205172, -0.173064, -0.003351, 0.592039, -0.122060, 0.194374, -0.540868, -0.138176, 0.341844, 0.081864, 0.242733, 0.373292, -0.228865, 0.148295, 0.120463, -0.094172, -0.023198, -0.016959, 0.237407, -54.570545, -0.573048, 0.675045, -0.190351, 0.164020, -0.054506, -0.056234, -0.145207, 0.239472, 0.296177, -0.228059, 0.566965, 0.066044, -0.390903, -0.266558, -0.301220, 0.327292, -0.387228, -0.326956, 0.046026, -0.537017, -0.113912, 0.009175, -0.175402, -0.095018, -0.330905, 0.084470, -0.136133, -0.354516, -0.469037, 0.438247, -0.231660, 0.032670, -0.097507, 0.094830, 0.058257, -0.154819, -0.390733, -0.600798, 0.396636, -0.294010, -0.608886, 0.159131, -0.488123, 0.253940, 0.310930, -0.223843, -0.505954, -0.019320, 0.451123, 0.033688, -0.305418, -0.521554, -0.435363, 0.323155, -0.312042, -0.008847, -0.252040, -0.180144, -0.310767, -0.178879, 0.130031, 0.857571, -0.107182, -0.299440, 0.559908, 0.155175, 0.366574, 0.170108, -0.399408, 0.217482, 0.401690, -0.321029, -0.045608, -0.068220, 0.157180, -0.177041, -0.105864, 0.044081, -0.435274, -0.008077, -0.537582, 0.047345, -0.313762, 0.100752, 0.147813, 0.158115, 0.128763, 0.052037, 0.156659, -0.200804, -0.530616, 0.043076, -0.484326, 0.115019, -0.426879, 0.149550, -0.379401, 0.265588, 0.110010, -0.137560, -0.046419, -0.190895, 0.185832, -0.424423, 0.238436, -0.331180, -0.208932, 0.093624, -0.136191, -0.024762, -0.284244, 0.031795, -0.036643, 0.343705, 0.086551, 0.040446, 0.576904, -0.533034, 0.577420, -0.326808, -0.414999, 0.202424, 0.118209, 0.829163, 0.636830, -0.537408, 0.020064, 0.206316, -0.161511, -0.346788, 0.161839, -0.026227, -0.223244, 0.261731, -0.382454, 0.108277, -0.355422, 0.103880, -45.383808, -0.162695, -0.150417, -0.127754, -0.070344, 0.119024, -0.116959, -0.034194, 0.281006, 0.047009, -0.478871, -0.337656, -0.401619, 0.260010, 0.109437, -0.230819, -0.169927, 0.253672, 0.187743, -0.142163, 0.026885, 0.312044, 0.012883, -0.075687, 0.187942, -0.864674, -0.161560, -0.180769, 0.423245, 0.286899, -0.322536, 0.082467, 0.129985, -0.038326, 0.400418, -0.058516, 0.277353, 0.456197, 0.539164, -0.083526, 0.026757, 0.039200, -0.235375, 0.052175, -0.201285, -0.225762, 0.416601, -0.418963, 0.252038, 0.136844, 0.261401, -0.321984, 0.021641, 0.297252, -0.045968, 0.015987, -0.632211, 0.079325, 0.178558, -0.686958, 0.065480, 0.519709, 0.642042, -0.067845, 0.037921, 0.154914, -0.014848, -0.024645, 0.106363, -0.512707, 0.045547, 0.496962, -0.158557, 0.217693, 0.007438, -0.582735, -0.274856, 0.251478, -0.168064, -0.218378, -0.187458, 0.235936, -0.303580, 0.133740, 0.433543, -0.416777, 0.163158, -0.312299, -0.542848, 0.314374, 0.362104, 0.192649, 0.357390, -0.229162, -0.168068, -0.437465, 0.052538, 0.281516, 0.368737, -0.097629, 0.968332, 0.208670, -0.460906, -0.274308, 0.238028, 0.236472, 0.210751, 0.070369, -0.012518, 0.160674, 0.005216, -0.122002, 0.292114, -0.093093, 0.153105, -0.204351, 0.578618, -0.321862, -0.400429, 0.156500, 0.084270, 0.381356, 0.067640, 0.051333, 0.399535, -0.307002, 0.340882, 0.135507, 0.089646, -0.237054, 0.281415, 0.090153, 0.462730, 0.171334, 0.454288, -0.291779, -0.587445, -0.012028, -0.045764, 0.449051, 0.142538, 0.409746, -0.105807, 0.066156, 0.277036, 0.749279, 0.060583, 0.179168, 0.016475, -0.148920, -0.023366, -0.071207, -0.346201, 0.011620, 0.048880, 0.374774, -0.428500, -0.617913, 0.200059, -0.312047, -0.069751, 0.212112, 0.469310, -0.322054, -0.436160, -0.151759, -0.362400, -0.100673, -0.390196, -0.720022, 0.013152, -0.093669, 0.500391, -0.181266, 0.171757, 0.205245, -0.491997, 0.150053, -0.303285, 0.213764, 0.181915, 0.042309, 0.117224, 0.243289, 0.163959, 0.249842, 0.096863, -0.027731, -0.371531, -0.153938, 0.580561, 0.096046, -0.302468, 0.400171, 0.505695, -0.165858, -0.123545, -0.023400, 0.045407, 0.485218, -0.918825, -0.379058, 0.030079, -0.237506, -0.267864, -0.423086, -0.861124, 0.339907, -0.120479, -0.469097, 0.068623, 0.029885, -0.089538, 0.517054, 0.389653, 0.260604, -0.697998, -0.163827, 0.174017, -0.102305, 0.016205, -0.635075, -0.597663, -0.165878, 0.092621, 0.079050, 0.743767, 0.460755, 0.141955, -0.437859, 0.222894, 0.521355, -0.322920, 0.021801, 0.437041, 0.586200, 0.036765, 0.155797, 0.139985, -0.041782, -0.709892, 0.013074, -0.547880, 0.155922, 0.070960, 0.569953, -0.219364, -0.014088, -0.335218, 0.293984, -0.440478, -0.020124, -0.123338, 0.212336, 0.409897, -0.151092, 0.206929, 0.108662, 0.003318, 0.480850, -0.388049, -0.373116, 0.508014, 0.646062, 0.338924, -0.014462, 0.072739, 0.503070, -0.571416, -0.053392, 0.261631, 0.593776, -0.324953, 0.396804, 0.309049, 0.453608, 0.104420, -0.480800, -0.556374, 0.342257, 0.100447, -0.134773, -0.252847, -0.256652, -0.206923, 0.125899, 0.127910, 0.259252, -0.247190, 0.271972, -0.559636, -0.121707, -0.268752, -0.576536, -0.150227, 0.483635, -0.634572, 0.468136, -0.471012, 0.473711, 0.300323, -0.380895, -0.137505, -0.299252, 0.208652], "falcon2:latest": [-1.275306, 3.837295, 3.104335, -0.132523, 3.717942, 0.372535, -1.772705, 3.119587, 0.330449, -0.700744, -2.615506, 10.353819, -2.361413, -2.383462, 1.663645, 2.710775, -4.230299, 2.722366, -0.357314, -2.037119, 1.449030, -4.184108, -1.876729, -0.485423, 1.748516, -0.127038, -3.764583, 0.180338, -1.962256, -1.628213, -0.470965, 0.818017, 0.607546, -1.474351, 3.853329, -2.645384, 0.169052, 1.954102, 2.811140, 0.609534, 0.137802, -0.385182, -3.722800, -2.822938, 1.028885, 4.215508, -1.851904, -0.174632, 0.901177, -1.909700, 1.275259, -3.331348, -0.757433, 0.559766, -3.048050, -1.044024, 1.142050, -0.407469, -3.040350, 3.580981, 4.013700, 3.739869, 0.390057, -0.821299, 0.094822, 3.214658, 2.524949, -0.300330, 0.556939, -3.069934, -0.515992, -2.658362, -0.433554, 9.095484, 2.195112, -3.849095, 5.091149, 0.481200, -1.134630, -0.239280, 0.887909, 1.042396, 2.646552, 1.802093, 1.645549, -3.790758, 0.988161, -1.200287, 1.894759, -1.166587, -0.253824, -1.039379, -2.276851, -0.460683, 0.297967, -3.424223, -4.195358, -3.542181, -1.769864, -1.338850, 0.911988, -2.302943, 2.484190, 2.044442, -4.596624, 1.267139, -2.049352, 2.546231, 0.792628, -1.380133, -0.790615, 6.012039, -0.964435, -0.969051, -1.812450, -0.890375, -0.798536, 4.244349, 0.251844, -1.319340, -8.636320, -0.711426, -0.413526, -0.978065, -2.059388, 3.617761, 0.892199, -0.481894, 2.143296, 0.515694, -0.860783, 4.284967, 0.612026, 4.006313, -0.335522, 0.782485, -0.058630, -0.286828, 0.756606, -1.973024, -1.502667, 0.269806, 2.558606, -0.341148, 3.807795, 2.500325, 0.260572, 3.994771, 3.990319, 4.135689, -5.712500, -2.158962, 0.670838, 1.928773, 0.011099, 2.180364, 4.558554, -5.859202, -1.996603, 1.865865, -2.038666, -2.903919, 2.060443, 0.326185, -0.932059, -1.541007, 3.649105, 1.813717, 1.723021, -0.905025, -3.768096, -1.637242, 3.691015, -2.636121, 1.495227, -0.221481, -2.330304, 1.152045, 3.441514, -1.925960, 0.132836, 1.376882, 0.099456, -0.139153, -2.827578, 3.249744, 2.968992, -3.029808, -0.102355, -3.398517, -2.040715, -2.388101, -0.241652, -3.439781, -1.627563, -3.316260, 1.659161, -1.469869, 2.303240, 0.234823, -2.916763, -3.792274, -3.562071, 1.947090, -0.283635, 1.218568, 3.385237, -0.217122, -2.976565, 1.250648, -1.260653, 1.940458, -2.756780, 1.184896, 2.428695, 0.372655, 2.108495, 2.850650, 1.011860, -3.017907, -1.272859, -5.748411, -2.802590, -1.651772, -3.874090, -6.691458, 1.628203, 0.902252, 1.731206, 1.233933, 2.390242, 4.491738, 1.581355, 1.289003, -3.980048, 0.603783, -3.222681, -0.143940, -4.617625, -2.092155, 0.132560, 0.233982, -1.955884, 3.025106, -1.231423, 1.970144, 1.114148, -0.862942, -0.308265, 0.604513, 1.433046, -3.165818, 5.211211, -0.752265, -0.261660, -0.435479, 0.435807, -3.130022, -3.382930, -0.302572, -0.304755, -0.875783, 2.048395, -1.610556, 0.004490, 3.130698, 2.056630, -0.268167, 1.429440, -1.549552, 2.836768, 1.683631, 0.112271, -2.846099, -3.467909, -1.734015, -1.098224, -0.865726, 3.147281, -1.294847, 0.867880, -0.441601, 0.848505, 0.525628, 3.181535, 3.977831, -2.883601, 1.395165, 1.380046, -1.757408, 3.363719, 2.468548, 1.804424, -1.532946, -2.130015, 2.167905, 0.806237, 2.135587, -0.616036, 2.803680, -2.744563, 3.122983, 2.290845, -1.941239, -9.350484, -3.405049, -2.718211, 0.868880, -11.208566, -2.547097, 1.621465, 0.609452, -2.762423, 0.948383, 2.396591, -3.263968, 2.348562, -0.143500, -0.433124, 3.383017, 1.096289, 0.702994, 1.397233, -3.194034, 3.071927, -0.490006, 1.249623, 0.095233, 1.227319, 2.841045, 1.574195, 0.175490, 4.525906, 2.250069, 4.384015, 0.057987, 0.493614, 2.313399, 1.154731, 3.197756, -1.679319, 1.088407, -2.359752, -0.140352, 2.276346, 1.908259, -1.822850, -7.785120, -3.179761, -1.742176, 1.736056, 0.866745, 1.359070, 1.303184, -2.294070, 0.053626, 0.213622, -3.149813, -1.677805, 5.298675, -4.036638, -1.087946, -9.100941, 0.924279, 2.780944, 0.060562, 0.627181, -3.341824, -3.136374, 1.235826, 3.671900, -3.921594, -3.728671, -0.480427, 6.906900, 2.382487, 2.390993, -2.643352, -0.856610, -1.719015, -1.395070, -0.955716, -0.926107, -0.762820, -0.575991, 2.222496, 0.387985, -1.718496, 2.484186, -1.179515, -0.119917, -2.718995, -0.767698, -0.807503, 0.944953, -0.757398, -2.630700, -2.638915, -2.941369, -2.706702, 0.776204, 1.073829, -2.168399, 2.919386, 0.639097, 2.180970, -3.533285, -3.003216, 0.523227, -2.920606, -1.932676, 3.508812, -0.450955, -2.594667, -2.833239, 0.684165, 1.521983, -4.122576, 0.048962, 6.750394, -0.751528, 4.363237, 2.375494, -1.676940, -0.282620, -1.074734, 5.402434, -1.364364, 2.213356, 3.225933, 0.355178, 0.533272, -0.000414, -1.026736, -3.608583, 3.379501, 0.643868, -0.129584, -2.414845, 0.739444, 1.379362, 1.128999, 1.212591, 2.126804, -3.392489, 2.190962, -1.066917, 0.072771, -1.777203, 0.017392, -0.069404, 2.575250, -0.143043, -1.058601, 0.293741, 1.353118, -1.655753, -2.091888, 0.584350, -0.720455, -3.424674, -1.981546, -9.107293, 1.949826, -1.522261, -2.681044, -8.341246, -3.131934, -0.714103, 0.207412, 2.508620, 1.767241, 2.730778, -0.825572, -4.030791, 0.238825, -0.371033, 0.656116, 3.989294, 1.633813, 0.325307, -0.367170, 3.002364, 5.156171, 1.634276, -0.971831, -1.604515, -2.526582, -2.878766, 2.807352, 1.909165, -3.935943, -1.386310, 3.170664, -1.649927, 2.220213, 1.246705, -3.735313, 0.272806, 2.752307, 3.109334, -0.102484, -2.823759, -0.632412, 4.712577, 0.411905, 1.390527, -0.006540, -0.578229, -0.902223, 2.687959, 0.999556, 5.464525, -1.069448, -1.659709, 1.730871, 3.159081, -3.505104, -1.294691, 0.470974, 2.520426, -0.375588, -1.659444, -2.902283, -0.643385, 0.652468, 0.084223, -0.908147, -1.238602, -1.975286, -2.141201, 0.736527, -1.523522, 1.027745, 3.354010, -2.870191, -2.455250, -0.858900, 1.902766, -1.251234, -0.778018, -0.829765, 1.343457, 3.192241, 2.146025, -2.868140, 1.453027, -1.014266, -0.626704, -1.431984, 1.272131, 1.996223, 2.216051, 1.321754, -3.712698, -0.555156, 1.665618, -2.150003, 2.469344, -2.743378, -0.621012, 2.153028, -1.323079, -2.781879, -3.508905, 2.613299, -1.131783, 1.250944, -0.726568, -0.350890, -0.510844, -0.491322, -0.969947, 0.056512, 2.073022, -0.684992, 2.324330, 1.388874, -0.592233, -0.007646, -2.144584, 3.341875, -3.342694, 1.472340, 1.924615, 0.260784, 1.455808, 2.527028, -0.212072, -2.648695, -3.802433, 2.224674, 1.380913, -2.425954, 0.170378, -3.111950, -2.677550, -2.053871, 3.144357, 2.069789, 1.802598, 2.778758, 1.854536, 0.374600, 2.614195, 0.576047, 0.823646, 2.809750, 2.347219, -1.944691, -4.136095, 0.170414, -2.563500, -3.029089, -2.152708, 0.650496, -0.629906, -6.837932, -0.400384, -2.685872, 2.135639, -1.354106, 1.782213, 1.624843, 0.522975, 0.623956, -3.265278, 4.157569, 1.269328, 2.334650, 2.017716, -1.755639, 0.314134, 2.929660, 1.085749, 0.075581, 5.186268, -3.722156, -0.716771, -0.292780, 0.854932, -1.265511, -1.754162, -2.801681, 2.599700, 1.192626, 0.355987, -2.978914, 3.488926, -7.477931, 0.302190, 0.159773, -1.786207, 5.423014, -3.205537, 3.759773, -0.824988, 1.123907, 1.247544, -1.165879, 4.103333, 1.861792, 0.432820, 2.554196, -6.796187, 3.449815, 2.632658, -0.926445, -1.542485, 3.184329, 2.700121, 1.650769, -4.734762, -0.469165, 1.700503, 0.710945, 0.347955, 3.436646, -1.542240, -1.133071, 2.124078, 2.539431, -5.082206, -1.449178, 0.902514, 0.330178, 0.733471, 0.341730, 1.352243, 1.385499, -1.848245, -2.310890, 1.524840, -0.910095, 3.168741, 0.410063, 0.614422, 1.316848, 1.423225, -2.094517, -3.665807, -0.068259, 0.396121, 0.212791, -1.601394, 1.471721, 0.772471, 2.288241, -0.320014, -2.428612, -2.265923, -2.968658, -0.796963, 5.248973, -6.998604, 2.486512, -4.433386, 0.621024, 0.518264, 0.806498, -2.155303, 0.587934, 1.328967, 2.631720, -3.549683, 2.099039, 10.083544, 3.482533, -1.516241, -1.431416, -2.913141, -0.618922, 0.779420, 0.236411, -0.174201, -0.076979, 0.220478, -2.811640, -2.158289, 1.648178, 1.729170, 1.846626, 4.329294, 2.369144, 1.389106, 2.679754, -2.294376, 0.030183, -1.699179, -3.449418, -2.010730, 4.066484, -4.480805, 2.193635, -3.104744, 3.195236, -1.664788, -5.720037, -1.266570, 3.600276, -3.170063, 1.536892, 1.569534, 1.513302, 1.860432, -1.550267, -4.262663, -2.133216, 1.650067, -1.284678, -6.684666, 2.014430, -0.744968, 1.426262, -1.076141, -0.831970, 0.365885, -0.528754, 0.758301, 3.348044, 0.187185, -1.889567, 2.917908, 3.023930, 3.023178, -2.817112, -3.179446, 2.497720, 2.188169, 1.301504, 3.268865, -1.754922, -1.822780, -0.972488, 0.873712, 1.323133, -2.205761, -1.612030, 2.303227, 2.813283, -4.188452, -2.182941, 0.085170, -3.564689, -2.566697, 0.568231, 0.818893, -1.662748, 1.361568, -3.690748, -0.252375, 0.785948, 1.647538, -0.423860, -0.906425, -1.824527, -2.521552, 0.867587, -2.148677, 2.452386, -1.397286, 1.692210, -1.310338, 0.637889, 2.583428, -1.523493, 1.296878, -3.746370, 2.423034, 1.349175, 0.721364, 3.474914, -0.023369, 0.163030, -1.579503, 0.504772, 0.925550, 2.454133, 0.964500, 1.823297, 0.661305, 1.748695, 1.492398, 0.669027, -0.978568, -3.156673, 0.232972, -0.302810, 1.174894, 5.136952, 1.104236, -2.518059, 3.446657, 3.263038, 1.778748, 0.553110, -0.299201, 1.262304, -0.469016, 0.049288, 2.680809, 3.362026, 1.534753, 2.141226, 0.736711, 0.353176, 1.463385, -1.981672, 1.908352, 2.260112, -2.573204, 2.284262, 3.457025, 1.571169, 0.957621, 0.818471, 2.919568, -2.219922, -0.831442, 2.851260, -0.567659, -2.098528, -1.197574, -0.571810, -3.110029, 4.160790, 0.262709, 2.311606, -0.834847, 2.457886, -2.212020, 3.465058, -1.780708, -0.743038, 2.232378, -1.010238, -3.758393, 3.163526, 1.855109, 0.639158, 0.407191, -2.256367, 1.018602, 2.291389, 2.191854, 0.564081, 0.571482, 0.289375, 1.938669, 2.964486, -1.157557, 3.346216, -0.604904, -1.879804, -3.309341, 2.884830, 1.600047, -1.182005, 0.213239, 2.088271, -2.216653, 0.805338, 3.107358, 1.409771, 2.426908, 4.041484, 2.601931, -2.011261, 1.826019, -3.202612, -1.295027, 3.832757, -1.571364, 6.595190, 2.578595, 1.157802, -0.552183, 2.183033, 0.745915, -1.118431, 0.077764, 3.768408, -1.987089, 0.194293, 0.610980, -0.361441, 2.990337, -2.133753, 2.919108, -1.559100, 2.185312, -1.411286, 2.620373, 3.767267, 2.179324, -1.849180, 0.756302, 1.529904, 3.719940, -1.179238, -2.207529, -3.936229, -2.072637, -2.803760, -3.367758, 1.098284, -2.009740, 3.083918, -0.394910, 2.485883, 0.113272, 2.655967, 1.349244, -2.255511, 1.025767, -1.384176, 1.732267, 2.225488, 1.098336, -0.509963, -0.263632, 2.146725, -0.957762, 1.923358, 0.133071, -3.363287, -1.703886, 1.032311, -0.755408, 2.498776, -2.764693, 1.509607, -1.617618, 1.436502, -1.559158, -0.746124, 1.562070, 17.831846, 4.211090, -1.706081, 2.346309, -3.273780, -2.834237, -3.982527, -1.515350, -3.250439, 0.453186, 0.603446, 3.211556, 3.159271, 0.628849, -4.001279, -0.854389, 3.856479, -3.991511, -0.177681, -2.286148, 3.187693, -0.223602, -0.075862, 2.957560, -2.254858, -0.967292, -3.214899, 2.144798, 2.144349, -4.565749, 3.582543, -0.960689, 2.410637, -2.425848, 2.851839, 1.097054, -0.605512, -1.531101, 0.845345, 1.495862, -0.475145, -1.574789, -0.029720, 0.645808, 3.104814, -2.337604, 0.435249, 1.841378, 0.299883, 2.139250, 2.561849, -2.265870, 2.402302, 0.791734, 4.197929, 0.943617, 3.297447, -2.381050, -3.254916, -3.195335, -2.113678, 1.651127, 0.783532, 3.214834, 0.408355, 3.205199, 1.115050, 2.330830, -2.771334, -1.927293, -1.882555, 7.298583, 1.038048, 3.148403, -2.869358, -1.115117, 0.074848, 4.928357, 2.089816, 1.986463, 0.470056, -6.525270, 1.073113, -4.135253, 0.774483, 1.268632, -2.197856, -2.068014, 4.605621, -2.308162, -2.780638, -0.721838, -4.479808, -3.508407, -2.428396, 4.441323, 1.775349, 0.214881, -1.635818, 3.045053, -0.610118, 1.187438, -1.759295, 3.817451, -1.664751, 2.842919, -2.662443, -1.073685, -1.129181, 0.843486, -0.775921, -2.891870, 2.612444, -3.581708, 0.276234, -1.549791, -0.161535, 9.922145, -2.840818, -1.211635, 2.635808, 6.841765, 0.938128, 0.082862, -0.197744, 0.354956, -2.333372, -0.819113, 0.838161, 5.306095, 1.898514, -0.049039, 2.551995, -0.244460, 3.227154, 0.455535, 2.356509, 1.989387, 1.096240, 1.646904, -4.098421, 0.545672, 1.364193, 1.341040, 1.556323, 5.145891, 2.229922, -1.214043, 1.089752, -1.778068, -2.560244, 2.266327, 1.981164, 2.707444, 2.781528, 0.979474, 0.972002, -0.075105, 0.934945, -0.866869, 3.448736, -1.561079, 1.061687, 2.550811, -0.469708, 4.136203, 3.849202, -1.115006, -1.362522, -0.477934, -0.217236, 3.059588, -1.913679, -0.223996, -2.238563, -0.246944, -2.901762, 2.800288, 0.591907, 2.385238, 1.304711, 1.502628, 0.314458, -0.560291, 2.228894, 4.770694, 0.590195, 1.130904, -0.154598, -2.602068, 2.284669, -2.444100, 2.262833, 2.987651, -1.718937, 0.172004, -4.404370, -0.713307, 3.951679, 1.743177, 2.184051, 2.084052, -1.200920, 3.804940, 1.190594, 0.423731, 0.387230, -2.698928, 4.313253, -1.562275, 3.057487, 1.332809, 3.277320, -0.025386, -7.029213, 2.271002, -1.368629, 1.512957, -1.627040, -1.321970, 1.505093, 2.593931, 2.138204, -0.703714, -2.467005, 2.581830, 0.924391, -0.011639, 0.399110, -1.663732, -1.625977, -2.453845, 1.651928, 2.514879, -1.845028, -0.415811, -2.509327, 2.051333, 1.113616, 0.354290, -3.648528, 2.132520, 1.173429, -1.130877, 5.016101, -0.490695, 3.052016, -3.462008, -1.558317, 2.028475, 2.340829, -2.054028, -2.028140, -1.076622, -3.786443, 2.211594, 0.838984, -4.107341, 1.741582, 2.438528, -1.545592, 0.031897, -4.673401, -1.738998, -0.626893, 1.342387, -0.145155, -1.596622, 2.264874, -2.561344, -1.419518, -2.779258, 3.840589, -2.814966, -1.121065, 3.103562, -1.434957, 1.904631, -0.152767, 3.093784, -0.006062, 1.727092, 3.195707, 1.766008, 2.366390, -2.307591, 2.373812, -0.476952, 1.527025, -0.465218, 2.363796, 1.407915, 2.833245, -2.019148, -0.643388, 2.679688, 5.222930, -1.854012, 2.572557, 1.569281, -0.477092, 1.902260, 3.180279, 6.306729, -0.274454, 0.718802, -2.418252, 4.774065, -1.660186, 0.724020, 1.382207, -0.139734, -1.164008, 4.906361, -2.018308, 3.283710, 0.376118, -0.720316, -4.012407, 0.609760, -0.314082, 1.398531, -0.268018, 1.786567, -2.602562, -0.330188, -4.301880, 0.335341, 4.711812, 0.695186, -2.297723, 0.472864, 0.569482, 1.624462, 2.681783, 3.021969, -0.104063, -1.222748, -3.950817, -0.500057, -0.442732, 1.211303, 1.949326, 1.100488, -5.104456, -1.600076, 1.789613, 0.082084, -0.346776, 0.059769, 1.983953, 2.033277, 0.305617, 2.394171, 1.896901, 6.016169, 0.465130, 2.148189, -3.048455, -1.015694, -0.439929, -4.321608, 2.767549, -2.717652, 0.469900, -0.449218, -1.194940, 0.407131, 0.130723, -0.135469, 2.412801, -0.450518, -1.383515, 0.950040, -1.357381, 3.228314, 3.361074, 0.332888, -5.870872, -2.020130, -0.178779, -1.861960, 2.273883, 0.218612, -2.976734, -1.733326, 1.392309, 1.946056, 0.799269, 1.524984, 2.629119, 1.725265, 1.525396, -2.106735, 2.488540, -3.239527, 0.826539, 0.666062, -1.867959, 0.354207, -1.698327, -2.927954, 2.341576, 0.872006, -2.815492, 3.722144, 2.325369, 1.456939, 2.497020, 1.038249, -1.285768, -1.675847, 0.893679, -1.395089, 0.595987, -0.750167, -0.563275, 1.091686, -2.418337, -1.896940, -1.411161, -0.380414, -0.252327, 0.983415, 3.090965, -3.424773, 1.611721, -1.380765, 2.076887, -1.467507, 1.811858, -3.494227, -3.977528, 1.935094, 0.442732, 4.073760, 0.244701, 3.578159, -1.013995, -2.507242, 4.465893, 1.057177, -0.537271, -4.829791, 0.020822, 1.384580, -0.917904, -0.844748, -2.344644, -0.291398, -1.531452, -0.735474, -2.581677, 1.842135, -0.054317, 2.477489, 0.283114, 1.382037, 1.793713, 2.417314, -3.699762, -2.272878, -3.177811, -0.718206, 1.148291, -1.529859, -0.163814, 2.781793, -2.223887, 3.794726, -1.517594, -1.921671, 3.090774, 0.501923, -1.972393, -1.791274, -2.481785, 0.448921, -2.090209, 0.220071, -1.382417, -0.468002, -3.140492, 0.560268, -3.443594, 0.177877, -1.951589, -2.809496, 0.517052, -1.871615, -2.020593, -2.291572, -1.610467, -0.813363, -3.394322, -2.246128, 1.871319, -3.381363, -3.987760, -2.516682, 0.588713, 3.890212, 2.469473, 0.362648, 0.561302, -1.753007, -1.627349, -0.349749, 2.216681, 0.383506, -2.713041, -0.593570, 3.251638, 2.689493, -3.748345, 0.691237, -0.569738, -5.734797, 5.413714, 2.996217, -3.365990, 1.122713, -0.547549, -2.552277, 3.383361, 5.260242, 2.122134, -0.637885, 2.626056, 1.089457, -2.199468, -2.468286, -0.702738, -0.091756, 0.488842, 0.760422, 0.146176, -1.661150, 2.917534, -1.514377, -0.663864, 2.496569, 1.366415, -2.151349, 1.885142, -0.772022, 2.252527, 2.092843, -2.031601, -0.813497, 0.407781, -3.519864, 1.166523, 1.347374, -2.144313, 1.894274, 2.548582, 2.886257, -2.012259, -0.423815, -1.669818, 4.414027, -2.174315, -0.024239, 3.345891, -3.157781, 3.547366, 3.083466, -2.391912, -2.350487, 1.912315, 2.167852, -1.419118, -0.530295, 1.211723, 2.307965, -3.893567, -1.757907, -2.166578, -0.312290, -2.318677, 2.761449, -2.065255, -2.718894, -5.395889, 1.222173, -1.110870, -1.290285, -1.207875, -0.060934, -2.584382, 1.737085, 1.880211, 2.757859, -2.688734, -2.082012, 0.599087, 2.687817, -2.736868, -2.483763, 1.212993, -1.285007, -3.371017, -4.127361, -2.956272, 1.772995, -1.661328, 1.707176, 0.474840, -0.346689, 1.808019, -1.051578, -3.156462, -3.182742, -2.470137, -0.530100, 1.256876, -3.449591, 1.078454, 3.284560, -1.910391, -0.292548, -1.595873, -0.838040, -0.740394, 1.511373, -3.185318, 0.501699, 2.445569, 1.670159, -2.597500, 3.368602, 4.472101, -4.204967, 4.533583, 0.047076, -3.118933, 1.523812, -1.076397, -3.584929, 1.941660, -2.212678, -0.267279, -2.351005, 3.205257, -1.552729, -2.789754, 2.306149, -4.425440, 2.926135, -1.727820, 4.657523, -3.581824, 1.874127, 2.977703, 3.189226, -1.316201, -2.322132, 0.326548, -2.243823, -0.581202, -2.148850, 1.752387, 3.638435, 1.993092, 1.322640, 3.349422, -4.158694, 2.212160, 3.103670, 2.673969, -1.280546, 1.500715, 2.859776, -0.207381, 4.820920, 2.770439, 1.037140, 2.432384, 1.314973, -3.747844, -0.571894, 0.404972, -0.784428, -4.287580, -0.850005, -0.596455, 3.271060, -2.868608, 3.010384, 2.683059, 4.961196, -1.193018, -2.061228, 0.384417, 0.839436, 1.829330, 1.032372, -2.438330, 0.873286, -5.543126, -1.238505, 4.828424, 4.276352, 2.086277, 1.759339, 3.404096, 1.391880, -2.645986, 1.286137, 2.212531, 2.400476, -1.635331, -1.826915, 1.386136, 0.734565, 3.339298, -2.710637, -3.658201, -0.764404, 3.978921, 1.805965, 0.612052, -0.069873, 1.873704, -0.291247, 0.808841, 1.046961, -2.339138, 2.548506, 0.116501, -1.358457, -2.428479, 0.405364, 0.049829, 2.078218, 0.630792, 2.682093, -0.111304, -1.323117, -2.635784, 1.362048, 3.085995, 2.625044, 1.898022, 2.371150, -0.770906, -0.275337, -1.812634, 0.451130, 0.030746, 1.921646, 2.202017, -0.176585, 0.895559, -0.876122, -3.598070, 0.757027, 0.715836, -3.268029, -0.826421, 1.655850, -0.267985, 0.701595, 0.129097, 1.087256, -2.069307, -1.865456, 1.049002, 1.045302, 1.117406, -0.491220, 1.752171, 0.180681, 3.506818, 1.705417, -3.404651, 2.519688, 2.092036, 2.918577, -1.644121, -0.248030, 0.368383, 1.397353, 0.468383, 2.009109, 2.615247, 1.785939, -2.678337, 1.617231, -1.236114, -0.461577, -1.838622, 1.968311, 1.328617, -2.911062, -1.685913, -1.801324, 1.286906, -4.115755, -2.113906, 9.896759, 0.374081, -0.388752, -1.223068, -3.063830, 2.229397, 2.612700, -1.033353, -1.022675, 3.386715, -1.121697, 1.809654, -2.561433, -2.942688, 2.570541, 2.310937, -1.491891, 0.108844, 1.215748, -1.947517, 0.629480, 0.692682, -0.667800, -2.369210, 1.485672, 0.038969, 0.029837, -0.422000, -0.252593, -0.688025, -1.452450, 4.435425, 3.307030, -1.265886, 6.128372, 1.978850, -3.929060, -0.298456, 2.120240, 3.654680, 1.866614, 2.054912, -1.318845, 1.181852, -0.038600, 3.942681, 4.401767, -1.473351, -0.744864, -2.225775, -0.871413, 1.849269, -1.780350, 2.124613, 2.299182, -0.689189, -0.497090, 0.518188, -2.052712, 3.142647, -2.284230, 4.709138, -0.259247, -0.405531, 1.163305, 1.855179, -0.130810, 2.581027, -3.659903, 2.676016, 1.587614, -2.458660, 1.384374, -0.823230, -1.355044, -2.172476, 1.655872, -0.893984, -2.042878, 1.969854, 0.140095, -5.839048, -1.551351, 0.726457, 0.911761, -0.065500, 0.825390, 1.000783, 2.737443, -4.026385, 3.902370, -1.769917, -0.966249, -0.816267, -0.570867, 1.182167, -3.117018, -2.331257, -2.639359, -3.660809, 1.104480, 1.201332, 0.084591, -0.062481, -1.295772, 1.735669, -0.666152, -0.181148, -0.707571, 6.112964, -5.684958, -1.611841, 0.320344, 1.758454, -2.201783, -0.981238, -2.890517, -4.545226, -2.521300, 0.036706, 0.128801, -2.098386, 2.333826, 2.365282, -2.611004, 2.553960, -1.038355, -5.702994, -2.997832, -2.794518, -2.389665, 2.133088, -1.820822, -2.890388, 2.260902, 3.086394, 0.570292, 1.511309, -3.697427, -0.810149, -0.843711, -1.795822, 0.495115, 2.823005, 1.649202, 8.319246, -0.411636, -2.392526, 2.539201, 2.370214, 4.756253, 0.828395, -1.034127, -2.109726, -1.572361, 5.465284, -3.657965, 4.423150, 2.149361, -7.255781, 3.108456, -2.962606, 2.280480, 1.510164, -0.990311, -3.053618, 2.630822, -1.767082, -0.539244, -2.688886, -1.095631, -0.003539, -0.584830, 0.941815, 1.863402, -0.325740, -0.093675, -0.985750, -2.204835, -2.334163, 0.659979, -1.940781, 0.892017, -0.047762, -0.801082, 5.068970, -2.416367, -0.054956, -0.609716, -0.925279, 3.985359, -2.428799, 1.385789, -2.310544, -0.390019, -3.638839, 1.571243, 0.829102, -1.987846, -1.974930, 0.148836, -2.629555, -1.189142, -1.211454, 1.648244, 1.809920, 0.444209, 0.857393, -0.552437, -1.938646, -1.704941, 1.153294, -7.150531, 2.521330, -4.333313, -1.203517, -6.791515, 5.610275, 2.526631, -2.193799, 0.316134, -2.975452, 0.043736, -2.145086, 0.947292, -1.770846, -1.070288, -1.937447, 2.345026, -3.003102, -0.961535, 4.269505, -1.473615, -2.028548, -1.439545, 3.920681, 2.086540, -0.087132, 2.692583, -1.969973, 0.640000, 0.759344, -0.851917, -2.110132, -1.116281, 1.290830, -0.036435, 1.081315, 3.280384, 1.031457, -3.773770, 2.426805, -2.144011, 2.387061, -1.523329, 1.364333, 0.632966, 1.254096, 0.397885, 1.691442, -0.060497, -2.736957, -0.955565, -1.479424, 1.189433, 4.488828, -1.039956, 4.110622, -4.271568, 3.211975, -0.931945, 2.384370, 1.593602, -0.530324, -1.894910, 0.872313, 0.775410, 0.611552, 3.815946, 5.441214, 2.328136, -3.694026, 0.856822, -0.358736, 1.209829, 1.578911, -1.869051, 2.328872, 1.902640, 0.171978, 1.517722, 0.237618, -2.074708, 3.189997, -0.351501, -3.595257, -3.138206, 0.722451, -0.521091, 2.513267, 0.863403, -3.036732, 0.966440, 0.639682, 3.953517, 1.480138, 2.872857, -1.725542, -0.765049, -3.056712, 0.961703, 1.613330, -3.012677, 1.984263, 0.603873, -0.146085, 0.401721, -1.095984, 1.858853, -2.949477, -0.214432, 0.300439, -1.510662, -1.520183, 3.227612, 3.277660, -0.816754, -1.828793, 1.171779, 2.029778, 1.869878, -2.729131, -2.555247, -3.633353, -3.591051, -1.858859, 2.480379, 0.767383, 3.103071, -1.344633, 2.634771, 4.324559, 0.995620, 1.743588, -4.545441, 3.179331, 2.600043, -1.224859, -0.539547, -2.826778, 2.330503, -2.919528, -2.707401, 3.279189, 0.743885, -1.030735, 1.944464, 1.497105, -0.072477, 9.878094, 0.991437, 1.252940, 0.908459, 2.009145, 2.427585, -2.123645, -2.284483, -0.918611, 2.746569, -2.033646, 0.837004, -1.525458, 0.870853, -3.641463, -2.634501, -0.521927, 0.874221, -1.667083, 1.903460, 3.153025, -1.037110, 2.270567, 1.085054, 0.564745, 1.921359, 2.154752, -1.015654, 2.087633, 3.454697, 1.214256, -1.918541, -1.607991, 1.002951, 1.865858, -0.964412, 2.511755, -2.399036, 1.691220, 2.473380, 0.841120, 6.731910, 2.276564, -0.619042, 0.109877, -1.801440, 2.544270, -3.653917, -0.909440, -1.673509, -3.541651, -1.475499, 0.974824, 1.341343, 3.371499, 2.327110, -4.265070, 2.730826, 6.252456, 0.442818, -2.496783, 0.133818, -1.569381, 1.881387, -1.776341, 0.944535, 1.976379, 1.078518, 0.454756, -0.553306, 1.956321, -2.193647, -3.906279, 0.997558, -2.133405, 0.813385, 0.066720, 0.675073, -1.305464, 1.049415, 0.058389, -3.035835, 0.971060, 1.673939, -1.002198, -2.784204, 1.691542, -22.129820, 0.332251, 1.568601, -1.923303, 0.294346, 1.423302, -2.935637, -1.775013, -3.467926, 2.910514, -1.682199, 1.219355, 0.793816, 2.940609, 2.325723, 1.023719, -2.592135, -1.507832, -1.641595, 1.456100, -1.044729, 4.106136, 2.221826, 2.339298, 3.302754, 1.938588, 0.728833, -1.555544, 2.474434, -0.518467, 0.533135, 3.680743, 1.677448, 2.104536, 0.646286, 0.555699, 2.477109, -3.550225, -3.647983, -2.079800, 4.112247, 2.804218, -0.919420, 1.293995, -3.565257, 0.565476, -1.367818, -1.903385, -1.448610, 0.866282, 2.322896, 1.147820, -0.627827, -1.059485, 4.482340, -2.255801, -6.158365, -1.201316, -2.617374, 0.849946, -3.063239, -0.336349, 3.439678, 1.400132, 0.616048, -2.214755, -0.108316, 1.013055, 1.614219, -2.119207, 2.570008, 1.538140, -1.337971, -2.840325, -0.002898, -3.976804, -0.981789, -2.240550, -1.108801, 2.162783, -0.593558, 0.575897, 2.254664, -1.405173, 1.071949, 2.099885, -1.849970, -0.874483, 2.513324, 2.759261, -1.591699, -2.328701, 1.031207, -1.108524, -12.576934, -0.212726, -0.139737, 1.239497, -1.242406, 1.892567, -0.104317, -1.160928, -0.283431, -1.366523, -0.203520, -0.691903, 1.787807, 0.201596, -4.096425, 2.446693, -3.331512, 2.033213, -3.020568, -1.573980, -0.388636, -1.998314, -1.966732, 1.677706, 3.899530, 2.133426, 0.836722, 0.754416, -0.003854, 0.806098, -0.711075, -0.452569, 0.355189, 1.279910, -1.450249, 1.334980, -0.518858, -3.259822, 3.693094, -2.071981, 0.501570, 0.056253, -2.921665, -0.392897, -0.867844, -0.986845, -1.664861, 1.737591, -3.061367, 0.302886, 2.094111, 0.045325, -1.595525, 0.674877, -1.471849, -3.893967, 1.412523, 1.773032, -2.750422, 1.035602, -0.516791, 0.286685, -0.128167, -3.022458, 0.957886, -0.954299, -2.147751, -0.348507, 1.870028, 4.274771, 3.953058, 1.525810, 4.137014, -2.523398, 4.372852, 0.338937, -0.623839, -1.676739, 9.756101, -3.779910, -1.706488, -2.624722, 1.853259, -1.887885, 0.109111, 2.570731, 1.300253, 0.520757, 0.706603, -4.339977, -5.510975, -3.262259, 0.894060, 0.797520, -1.387151, 1.952152, 3.178773, -0.055920, -0.988293, -2.944663, 0.906312, 1.589234, -1.910973, 1.550346, -3.043045, 1.202343, 2.156839, -3.822891, 1.310066, 0.109305, 0.393427, 0.848353, -0.702017, 1.135580, -0.544394, 0.898604, 2.183110, 2.875517, -2.742109, -2.914018, 2.957861, 1.904441, 0.188540, 3.428993, 4.164488, -3.325378, -0.339448, 1.521152, -0.671186, 0.412715, -0.244523, -0.876270, 1.905240, 1.004341, 1.861009, 3.023627, 1.829536, -0.710838, 2.065416, 3.122601, 2.638688, -2.839584, 2.718982, -0.912577, 3.302638, 2.473169, 0.684764, 1.555684, -1.825787, -1.142594, -0.022323, 3.818839, -3.767341, -2.613019, 0.704796, 2.798040, -5.006337, -0.892998, 2.696245, -2.926975, 2.355693, -0.578362, -1.544222, 3.422838, -2.041148, -0.578029, 0.011704, 1.837247, -1.490923, 2.378649, -2.392335, 3.045223, -3.292506, 0.487439, 2.134249, -0.827777, 0.996064, -1.616960, -0.817308, 1.334069, 3.380960, 2.541771, -0.265344, -0.917402, 2.407108, 1.253521, -1.562716, -1.517113, 2.796017, 0.775317, 0.575597, 1.323530, 1.529689, -16.116417, -2.998111, 0.450969, -1.814111, 0.349480, 2.782877, 1.295974, 1.560289, 1.718585, 1.898549, 2.660482, 0.864414, -2.238849, 0.737516, -2.422750, 0.247123, -2.743533, 4.227241, 0.526669, 0.501835, -2.821635, -4.300069, 1.813544, -1.345552, -3.812800, -0.391366, -1.456246, 2.738957, 1.825692, -2.190766, -3.353376, -1.940057, -0.862514, 0.672473, -1.073232, -4.133321, -0.304357, -2.691906, 4.606014, 2.997205, 1.211570, -1.067043, 0.287098, -2.027639, -2.065545, 3.585075, -1.943740, 1.839725, 0.222802, -4.045444, 0.650917, 0.830172, -2.850742, -2.450385, -0.215431, 1.970153, 3.219507, 0.619687, -0.603187, -2.238926, 2.604592, -1.533692, 1.733842, 1.440334, 0.821643, 1.787324, -3.166633, 0.036767, 5.421640, 1.393930, 3.653860, -1.986755, -3.910746, 1.559711, 0.495545, -0.567174, 3.627657, 0.452299, 0.548954, 4.050268, -3.818210, -0.528180, 1.463299, 1.876337, -3.187659, 0.062530, -3.048895, -0.426209, 1.873657, -0.866349, -2.653108, 3.005883, -3.116135, 1.542959, 0.393652, 2.553407, 0.303429, 2.347852, 1.130731, -1.145539, -0.604549, 0.910890, -2.142335, -0.729891, -2.702992, -1.919118, 0.967466, -2.112257, 2.792009, 0.179999, 2.758096, -1.864986, -3.021833, -0.889339, 1.378984, -0.177384, -1.409245, 1.099375, -0.886126, 1.222357, -1.046859, -1.496866, -1.473668, -1.838376, -2.917238, 0.983777, 2.499777, -1.375984, 2.721064, -2.649018, 1.615053, 0.037384, -2.364583, -0.474144, -1.747913, -2.922879, 0.293949, -1.100694, 1.720940, 2.906822, 0.333392, 3.154211, -2.480198, 1.405208, 0.162729, 1.072986, 1.403922, -2.553919, -1.325863, -0.578505, 1.225022, 1.082428, -1.728628, 2.811140, 3.167954, 2.595941, -0.468482, 0.676855, -3.399291, -2.395138, 2.171390, 5.313373, 0.593171, 1.174135, 1.041957, -3.592760, -1.115999, -1.609447, 0.996160, -4.086498, 1.880493, 2.586645, -1.809857, 1.197866, 3.047547, 2.049238, -5.638014, 3.052633, 1.992575, -0.746054, 2.904683, -0.634189, -1.680446, 6.441619, 0.607398, 1.153746, 0.197727, -0.397482, 3.676062, 4.964014, 3.061106, -1.133064, 0.068444, 0.732943, 1.284233, 0.121063, -2.615230, -0.237515, -0.004988, -0.984315, -0.205606, 0.886206, 2.857014, -1.104957, -0.110875, -1.748340, -1.275044, 0.566729, 2.455837, 0.216097, 1.074522, -1.081340, 1.013576, -0.510011, 0.004325, -1.918031, 2.226862, -1.485101, 1.199568, 3.146095, 2.427711, -3.011320, -2.449924, 1.329107, -1.326373, 0.970822, 0.849875, -1.601499, 2.314740, -3.488747, 1.154833, 0.087346, 0.803074, -0.903432, -3.312709, 2.299358, 2.803974, -3.286255, -1.573510, 2.070856, -0.101941, 2.171456, -1.329902, -3.907687, -3.720195, -2.571179, 2.813491, 1.830530, -2.511291, 1.457561, -2.576567, -2.362876, -1.616081, 1.032160, -1.181725, -0.276253, 2.621583, -0.266221, -0.139193, 0.748835, 1.204025, -1.065338, 0.172388, -0.772580, 1.074110, 1.518136, 1.267711, -1.981061, 1.981899, 2.800466, 2.231948, 0.890822, 1.847108, -2.885270, 1.300715, 4.157722, -3.683444, -3.738341, -1.370671, -0.604141, -1.896416, -3.395854, 5.339676, -3.188531, -1.775950, -1.851381, 1.067175, -1.844483, -1.300046, 3.536031, 1.521053, -3.789303, 0.140510, 1.047382, -0.060062, -2.875290, -1.034876, -1.860420, -3.490496, 0.724952, 0.459753, 2.903262, 1.382170, -1.717902, -1.420321, 3.068552, -2.760029, 0.564609, 1.620253, 0.562308, 0.044785, 0.964442, -2.613660, -1.785604, -0.199799, 1.718101, 1.127491, 2.559109, 0.299823, 0.551968, 2.438937, 0.701297, -0.019573, -2.164789, 1.922659, -0.866623, 0.326152, 2.629571, -0.972274, -4.290656, 4.196831, -3.415634, 0.356076, 3.115250, 1.733862, 1.982695, -3.026798, -2.023067, 2.122068, -2.348504, 0.413962, 0.148813, -3.872636, -1.810721, 0.762815, 0.159291, -0.137686, 2.732738, -1.524898, 3.947837, 2.365946, -2.701674, 1.253832, 0.194691, -0.156369, -3.185858, -1.945729, 2.551816, 3.177832, -0.077237, -2.811372, 3.560195, -1.534038, -1.941651, -1.060778, -1.237860, -2.827729, 1.169655, -1.817541, 1.247796, -2.044961, -2.042051, -0.980262, 0.946876, 1.271373, 3.471315, -2.489898, 2.386957, 0.589401, 0.960726, 4.046332, -0.806419, -0.904894, 3.152041, -5.319210, 2.374078, 4.134824, 1.937350, -0.497493, 2.662346, 0.755571, 4.101086, 0.119435, -2.297841, 1.126628, 1.798876, 1.621213, 3.012361, 0.051148, -2.327565, 0.777902, -2.543133, 0.416235, 1.116755, 3.242164, 1.235808, 0.284160, -1.247335, 1.107888, -1.079982, 4.659342, 2.768052, 1.012518, -3.917892, 0.223945, 0.545874, 4.944964, 2.193990, 3.037240, 1.796157, -1.105980, -2.231532, 2.658256, 2.005485, -3.134518, -1.135793, 0.886341, 2.554361, -2.021382, -2.364516, -0.521016, 1.373525, 1.369994, 1.462440, -0.309788, -2.899583, -3.485990, -0.327034, -0.483858, -2.365628, 0.571344, -3.879119, -0.263073, -0.500833, 0.611228, -1.481678, -2.642653, 1.774129, -3.965195, 2.924938, 0.561954, 2.093593, 2.242171, -3.799051, -0.805658, -2.153137, -0.395024, -3.239681, 2.431444, 0.556913, -3.113872, -2.407112, 0.681333, 1.893931, -2.904344, -0.771003, -1.218895, -1.536474, 2.928406, -4.104526, 1.685589, -1.799101, 2.594079, 1.383285, -3.314305, 3.277896, -3.116689, -0.780264, 2.376668, 2.787229, 1.384615, -2.993711, -3.415761, 3.395609, -1.612519, 5.627576, 3.240973, 0.276432, -2.974000, 5.787770, 2.183374, -0.118661, 2.462688, -1.815760, 3.092844, -1.904415, -1.680902, 3.111355, 2.789873, -3.434824, -2.328745, 4.971395, -0.408980, -1.288155, 1.367070, 0.403338, 0.400134, 1.119727, -0.911876, -1.399861, -3.973023, -2.791090, 3.630412, -1.710031, 2.134027, -0.959766, -0.329367, -0.732914, -0.993124, -2.181825, -1.024484, -1.577234, 1.415553, 1.184569, 0.033538, -0.182342, -1.553549, 0.714704, -2.457281, -1.513178, 1.668061, -1.185722, -0.021128, 0.509903, 0.016285, 2.772515, 3.466581, 2.642076, 2.103426, -0.777547, -3.492056, 0.086423, -1.734952, -2.397410, 0.079009, -0.129437, 1.052238, -0.897817, -1.193079, -2.432753, -0.610183, 0.759082, 2.856251, 0.527911, 2.582767, -1.680882, -0.903450, 0.290197, 1.900693, 3.820286, -1.310768, -2.894516, 0.749423, -2.294792, 1.678832, -1.254640, -2.198882, -1.182828, 2.383785, -2.344207, -3.577337, 1.876101, -3.005041, 3.585013, -0.243899, -2.782204, -0.227220, 0.830313, -0.217250, -2.924293, 1.900393, 1.335294, -0.480179, -2.973958, -3.055072, 0.840056, 3.724781, -0.353757, 0.307534, 0.540906, -2.654098, -0.348129, -0.920658, 2.132657, -2.957996, -3.357490, -1.639126, 0.272579, -1.003739, 1.129556, -0.414330, -1.633175, -2.019671, -1.955855, 1.487665, -1.117603, -2.708709, -0.485532, 1.924051, 2.907635, -4.480110, -1.866212, 1.161165, 1.436610, 0.408536, 0.190726, 1.837151, 3.994542, -1.868298, 0.735382, 3.357700, -3.131468, 1.535451, -2.268453, 1.079755, 1.087334, 2.349835, -2.999949, 0.427312, -2.889404, 1.677031, -0.963640, -0.438927, 3.972819, -0.491260, 3.205900, 2.318801, 3.833720, 1.883632, 0.465372, -2.379404, 2.530885, -2.490902, -1.817383, 2.394443, 2.693515, -2.303585, 2.991621, -0.318521, -2.124806, -1.568704, -1.105953, 1.689304, -1.503239, -0.745483, 1.219584, -0.008324, -2.262623, 2.536577, 0.300485, -0.126827, 0.379301, -2.164514, -1.050668, 2.279346, -0.929581, -2.010704, -1.761641, 0.769421, 5.870451, 2.409795, 1.549106, 0.406771, -2.387406, 2.617296, 2.226446, -2.372685, -2.909598, -4.472449, -4.199341, 0.783240, 0.096103, 0.230260, 2.392704, 0.130306, -1.671189, 0.000460, 0.859836, -2.856006, 0.879900, 2.923421, -3.459907, -2.481248, -0.380551, -2.122747, 2.509279, 1.280891, 1.923709, 0.026194, 0.975075, 0.535158, 5.102134, -3.227482, -0.724829, -0.011868, -2.003006, 0.030337, 1.065228, -0.093162, 4.732137, -2.835727, 2.164323, -3.163384, 1.464387, -9.556238, 3.294225, -1.643234, -0.544956, 1.026036, 1.801332, 0.531682, 0.875548, 2.275990, -5.821035, -2.165171, -2.959899, -3.164179, -0.533030, 0.882995, -3.908379, -1.524249, -3.931026, -1.867308, -7.501748, -3.151053, -3.374834, -1.482490, -0.023572, 4.518243, 0.564633, 4.339701, 3.183874, 4.368966, -1.674653, -1.995406, -1.902929, 2.051385, -1.286106, 2.168142, 1.894793, 3.113796, 16.570700, -2.124722, 0.712696, -8.329745, 1.154208, -1.029663, -1.493598, 0.788374, 0.373883, 2.354367, -2.372883, 2.818204, -3.362396, 2.617382, 4.474661, -0.396233, -1.267438, 2.482839, 0.489908, 2.244136, 1.239966, 4.426870, 5.876467, 5.865771, -2.613004, -0.572432, -0.640416, 1.465602, 0.965603, 1.523670, 3.018555, 2.185856, -0.685083, 1.380377, -1.892750, -3.034616, -1.156111, 0.307927, 4.088897, -0.267865, -0.748823, 2.313815, 1.472056, 3.863946, -1.217377, 0.140903, 2.896098, -1.744379, -1.723044, -0.629762, -0.662425, -2.445518, 0.152551, 1.374967, -1.386010, -2.313100, -4.004970, 1.415225, -4.312975, -1.410863, -2.737320, 1.355401, 0.709947, 1.922767, -0.346310, 0.500168, -0.085046, -3.296778, 1.980763, -1.139568, 1.725973, 3.104150, 2.869179, 0.393733, -0.193732, 3.668149, -2.017278, 0.133616, -0.618817, -1.574684, -3.663116, 0.169122, -0.268842, -1.601942, 0.325557, 1.805874, 2.177101, -2.602196, -1.913941, 0.011868, -0.395234, -0.834341, 1.420007, 1.540699, 3.013688, 1.357066, 1.180240, 1.042086, 2.019283, -2.652542, -2.059645, 0.797553, -0.451454, 2.375256, 2.512674, 1.454196, -2.217665, 0.813907, 1.702392, -0.050121, -6.994725, 5.303644, 1.784331, 0.242761, 1.062950, 3.647518, 2.466606, 0.988599, -0.931017, 2.402037, -1.814979, -0.855702, -3.275341, -1.074031, -2.235515, -1.613209, -1.744707, -1.840088, -0.779895, -4.490575, 0.831230, -0.156800, -2.528859, -3.127295, 2.660365, 1.944716, -0.364012, -1.861313, 3.323568, -0.811558, 0.169003, 2.853401, 3.639811, -0.011596, 1.986325, -1.664246, -4.615153, 2.271653, 1.025806, -3.154962, -2.319878, 2.225085, 0.531775, 0.322223, 1.468869, -0.566937, 1.309493, -1.211456, -3.419137, -1.585997, 1.178995, 0.171657, 3.465562, -0.804610, 1.665783, -5.088279, 0.650534, 1.791170, 17.250408, -1.626894, -1.974006, -0.824130, -3.331557, -3.880378, 0.599182, -0.694505, 4.215900, -1.447300, 2.230931, -0.596584, 0.340238, 4.081385, 3.625847, 2.839355, 3.756844, -2.777805, 0.100937, -1.989295, 3.277542, -4.379092, 3.087123, 1.403637, -2.231328, 2.390607, 1.557257, 3.298607, -0.299247, -1.181531, 0.525384, -0.890912, 1.824297, -3.995151, -5.042112, 0.744427, -3.067411, 1.595778, -2.571892, 0.771376, -3.277676, -1.455596, -0.250049, -1.351680, 0.697943, 1.069591, 2.857038, -0.159182, -0.362470, 1.539649, 0.099324, -0.135235, 1.614402, -1.260010, -0.059322, 0.792264, -3.511580, -0.926815, -0.584613, 3.695359, -1.960196, 3.956642, -2.118308, 1.945329, -3.706782, -4.705505, -1.869684, 1.638058, -4.319848, 0.475216, 3.160248, 1.229439, 5.102840, 1.172704, 0.917782, 2.616993, 2.139804, 0.179050, 0.635365, -2.586541, 3.264217, 2.315623, -0.899583, -0.615601, -2.657061, 1.547319, -2.286578, 1.879086, 1.495636, 2.321740, -0.555475, -1.915786, 1.741425, 0.884898, 2.387746, 2.810685, -1.226168, 0.692660, 2.960999, 3.579106, 2.618080, 2.850123, 3.143281, -1.445385, -0.869172, 2.514671, -2.716269, -1.470776, 1.258983, -3.679865, -2.217870, 2.349101, -1.496878, -3.025600, -4.349153, -1.707175, 3.005045, -1.947022, 0.834064, 2.545904, 0.842057, 1.460314, -0.993782, -0.211882, -3.322590, 0.401144, 2.192401, 0.763440, -2.603230, 2.001291, 0.178745, -0.747411, 2.043627, -2.383173, 1.185634, -0.005095, 2.036742, -2.243795, -0.514880, -1.657962, 0.009154, -0.846694, 3.579744, -1.727863, 1.975658, 0.298879, -1.385962, 3.904871, 0.650457, -0.503490, 1.660019, -1.920314, 1.000721, 2.794953, 0.703100, 1.590531, -3.563776, -2.024322, 0.831120, -0.502338, 1.180401, -1.431693, -3.374807, 1.813871, 0.377378, 2.058053, -2.708455, -0.701740, 0.503954, 3.542927, 1.265793, -2.440325, 5.357823, 1.181739, 1.192558, 0.281461, 2.353562, -0.804080, -1.500842, 0.188305, -3.794515, -1.650937, 1.416369, -2.651186, 3.189369, 1.791879, -2.396369, -4.074958, -2.053515, -2.250571, -4.064375, 0.532033, 1.440506, -1.240853, 2.492684, -2.390124, -1.024883, -0.555563, -1.956213, 1.623435, 2.391842, 3.169787, -2.010656, -2.048950, -0.517389, 3.292646, 2.329223, -2.762945, -1.225980, -3.282871, -0.160141, -1.514427, 1.987719, 0.649399, 0.815709, -2.400442, -2.567317, -1.862416, -1.602954, 4.048628, 1.432278, 1.482120, -0.028168, -1.211920, -1.063073, 0.239240, -0.337652, 2.459109, 0.510653, -1.598884, 2.341745, -3.938889, 0.773527, 1.360329, 3.298347, 3.038607, -1.644866, -0.088439, 1.705638, -1.667633, 3.251128, -0.499025, -0.244674, -2.124416, 1.485698, 1.383972, -0.360520, -1.537704, 0.886796, 1.275228, 1.144015, -0.614845, 3.147204, -2.104358, -1.287056, -1.382660, 1.712896, -1.635278, -2.644013, -0.343970, 1.227356, -0.518890, -3.459199, -1.418720, 0.039872, -2.601501, -1.806357, -1.778686, 2.005520, -1.855919, 0.793212, -1.463252, 1.586207, -0.184600, 1.077291, -3.230673, -1.065376, -2.426338, -4.433507, 3.362201, -2.072861, 4.426600, -1.687956, -0.826343, -0.115859, -2.048442, 0.140565, 4.825446, -1.373407, -0.139355, -0.565521, -3.447233, 2.419903, -1.851389, -3.471667, 1.108885, 3.593758, 0.237525, 2.659810, -1.035963, -2.568973, 1.054946, -1.254823, -1.618325, 0.781841, 1.970722, 3.680123, 1.463840, 0.097049, 1.916009, -0.708227, -0.067044, -1.741496, 2.598838, 3.050692, -0.787915, 2.766759, 1.675597, -1.173892, 2.223792, -1.062551, 2.273316, 3.011199, -1.650119, -2.648605, 0.158647, -2.177063, 0.186268, -3.177166, -0.526156, -1.345172, -2.188882, -0.780098, 2.949754, 2.012983, 0.683741, 1.745923, 3.283293, -2.406209, 2.281897, 1.034458, 0.070274, -3.126071, -3.453828, 0.506103, 4.258954, -0.538088, -2.218058, 3.084713, 5.720852, 2.743039, -1.810076, 6.272157, 0.664128, 3.120400, 0.742479, 3.379241, -2.731331, -0.166025, -6.347305, -2.193065, 0.273632, 0.289647, 3.115033, -2.160245, 0.387171, -0.448133, -1.982356, -4.029318, 1.811508, 3.046365, -2.964187, 2.094236, -3.426285, -3.485981, -1.995775, -1.983477, -2.634778, 0.562914, 1.780642, -2.165194, 0.609340, 0.511937, -2.450258, 3.387038, 2.438292, 1.092778, 1.200358, 2.149229, -3.404688, -3.018096, 4.586117, 1.999421, -1.138862, -1.361518, -0.968755, 1.716326, 2.557005, 1.137697, -1.860175, 4.096773, -2.173637, 0.623251, -0.005716, 0.244621, 2.366596, 3.502475, 1.700907, 1.084500, 3.542811, 0.157646, 0.709190, -1.301711, 1.101328, -1.765094, -2.921493, -0.108164, -0.767839, -1.458841, -3.287886, -2.644441, -0.287442, 0.930482, -1.790940, 2.306692, 2.406610, -3.171745, 3.804529, -2.363433, 2.127100, -0.969682, -1.289734, 0.080165, -0.030003, 3.477387, 1.284148, -0.626319, -1.433724, 1.803858, 3.479468, -0.094739, 0.400459, -7.394617, 0.835967, -2.398387, 0.867873, 2.937330, 0.751119, -1.297753, 0.425168, -1.585854, -3.647148, 3.178249, 1.690364, -1.492471, 2.513674, -1.554596, 3.298636, 0.609163, -2.723015, 2.000729, 1.757294, 0.908518, -3.078389, -2.418847, 7.257916, -1.492727, -3.191773, 2.915990, 1.081085, -1.266900, 1.227162, -3.655681, 0.880118, -0.415529, 4.598647, 0.800119, 3.403203, 1.137017, 0.396906, 0.119401, 1.960427, 3.879504, 6.221431, -3.068269, -2.315856, 0.694096, -0.145249, 0.976113, 1.862161, 0.650839, 0.130839, 0.288261, -4.273160, 0.193502, -4.425699, -2.068643, -1.223312, 1.958007, -0.721088, 1.534529, 2.555326, 2.301806, -1.540642, 2.078291, -3.631954, 4.562294, 1.094564, -0.621842, 1.441936, 2.677450, -1.552978, -0.132519, 0.788240, 1.321109, 3.038334, -5.551532, 2.999269, 0.364990, -0.272237, -2.743863, -2.561628, -1.798099, -0.921997, 2.971652, -1.706339, -1.688948, 0.070037, 3.021770, -3.601783, 2.029795, -0.783538, 1.702649, -0.212672, -1.084160, -0.049358, 5.279011, -1.903958, 2.085022, -3.444110, -1.312608, 2.920285, -1.673897, -0.170632, 0.140543, 0.217857, 0.993657, -0.147226, 4.131040, 1.917654, -0.402115, -1.762819, 4.605012, -0.253315, -0.367822, -1.636496, 2.923368, -3.571045, 1.615221, -0.274513, 1.293480, -3.736974, -2.906220, 1.564071, -1.271615, -4.054400, -0.437894, -2.497492, -4.102562, 0.335778, 4.307206, 6.871069, -1.353800, 1.235324, 1.515420, -0.432864, -0.915976, 1.982390, 1.529526, 1.321939, 0.553168, -0.996906, 1.229996, -1.395458, -0.720184, 3.174892, -0.654703, -5.431517, -2.564596, -2.954030, -1.324461, 1.512441, -2.701057, 0.388167, -2.322500, 2.430663, 1.636868, 3.658733, 0.276975, -2.526526, -2.830755, 0.008060, 2.742812, -1.348127, 1.398821, -1.734482, 0.392166, -0.090562, -1.914905, 1.780919, -2.045624, -1.376605, -1.993155, -0.685609, 2.082545, 2.854451, 1.075901, -1.963066, 2.440422, 0.654377, 1.624016, 0.347185, 3.520130, -3.108392, -0.240567, -3.161610, -0.800270, 0.307272, -6.462657, -2.478560, -1.957913, 1.711907, -0.733501, 0.139884, 1.806551, -1.349770, 3.346692, -7.009873, -2.372363, 0.695869, -0.527927, -0.040519, 3.153802, -0.830838, 2.495985, -0.613522], "minicpm-v:latest": [0.373405, -0.434066, 0.402574, -0.150118, -1.009159, 0.055999, 0.578607, 1.257351, -2.809614, 3.972482, -0.813682, 1.450852, 0.222104, -1.309440, 0.825239, 1.990135, 1.091002, -1.035189, -0.127123, 1.494732, 4.583488, 2.152238, 1.950469, 0.040962, -1.026761, -2.280671, -0.764667, -2.645734, 0.738428, -0.337854, -0.514919, -2.169918, -1.606695, 0.726714, 2.363379, 0.812033, -1.017132, -5.906273, -0.068586, -3.161905, -0.668377, -0.930418, 0.602283, 1.411093, 0.929184, 0.361665, -7.188779, 0.122860, -0.208246, -0.857899, -1.278283, 0.683655, -1.143659, 3.433253, 0.186680, 0.858210, 0.086462, -1.969538, -0.585241, 2.501438, 1.663121, -1.452310, -2.782187, 2.376974, 0.011909, -2.539715, 0.295956, 0.566496, -1.961629, -0.696331, 2.636359, 1.223098, -0.417314, 0.676321, 2.153070, 0.455967, 0.085189, 1.253216, -0.196767, -0.790615, 2.291733, -0.371285, -3.192962, -1.635582, 0.242907, 17.824533, -1.191271, 0.693028, 0.503808, -2.743020, -0.241273, 1.333885, 1.035344, 1.961190, -0.152041, 1.285618, 6.496006, 0.763207, 0.087968, 2.870465, 0.356444, -1.683651, 2.216422, -1.844649, -0.691816, -1.050912, -0.499342, 1.075545, -0.171938, 0.909891, 1.241483, 1.092643, -2.173166, -12.015554, -2.856367, -20.698908, -0.609858, -3.207479, -0.977120, -0.093685, 2.417937, 2.442160, 0.848334, -2.646831, -1.455137, -0.189020, 3.002412, -1.256232, -21.787876, 0.788017, -0.381844, -1.949451, -1.868758, 0.008738, 2.679282, -0.296270, -1.594275, -2.922423, 0.540774, 0.256479, 3.338656, 0.494479, -2.322596, 2.338914, 0.057826, 1.467675, -1.584482, 0.633437, -3.083071, -0.641232, -1.212952, 0.981840, 1.525307, 0.569190, -2.707587, 1.231240, 0.272207, 0.788035, 0.123544, 1.150698, 0.283472, -2.258527, -35.561195, 2.163777, 6.679248, -1.705841, -0.327031, 2.162935, 1.658646, 0.227390, 0.758502, 1.687766, 1.669509, -0.139092, -0.986089, -0.276972, 1.080841, -0.210213, -0.943579, -2.606741, -4.459008, 0.478328, -1.479767, -3.058852, -8.545797, -0.395627, -1.164287, 2.495408, -0.183550, -2.715141, -2.877285, 0.344198, 0.790495, 0.623198, -1.037656, -0.456738, 1.659658, -0.046735, 2.193197, -0.659875, 2.066572, 2.893802, 0.885306, 0.174187, 1.547745, 1.911764, -1.608837, 0.710351, -1.673273, 9.898326, -2.634362, -0.044190, 0.545811, 0.707455, 3.859440, -3.805061, 2.133735, 4.158154, -0.054382, -0.429803, 0.409739, -0.137998, 2.086717, -0.399401, 2.022264, -0.645827, 1.551178, 0.694332, 0.139730, -0.094900, -0.723024, -2.779635, -0.507423, 1.105360, -0.154632, 0.736738, 0.161376, -0.676730, 15.677238, -0.188134, 1.856025, -1.736571, -0.953508, 0.939773, 4.345387, 0.713690, -2.220577, -2.284271, 2.869463, 1.600713, -0.709076, 0.347308, -1.171167, 1.196288, -2.898170, -0.818068, -0.079971, 1.877244, -0.987878, -2.073211, 2.558323, -1.109507, -0.980287, 0.290609, -0.468093, -1.300264, -2.062796, -0.978946, 1.764146, -0.106737, -1.404154, -0.090169, 0.652458, -0.804112, -2.321134, 1.668066, 0.430334, 6.668191, -1.203562, -0.704168, -0.549548, 2.036619, 0.800234, -2.810486, 0.518899, -0.560133, 1.006556, -2.317722, -0.750900, -0.774626, -2.130997, -3.248784, 0.649599, 1.748983, -0.228464, -0.605739, 0.412901, -0.329177, -1.682082, -0.877227, 0.548816, -0.893511, 0.946567, -2.476345, 1.110094, 1.972772, 0.139028, 2.039542, 1.314990, 0.706638, 0.704419, 1.276759, 0.549634, -0.312816, -6.987369, -2.040702, 0.164418, 0.289892, -1.125816, -0.748568, 1.327388, 0.417259, -1.186356, 0.290069, 0.164743, -1.728422, 0.379547, -1.688991, -1.351178, 2.636830, -5.866957, -0.173038, -0.586063, 2.464278, 1.674203, -0.039280, 2.671825, -0.426402, 0.797512, 1.504823, -4.485917, 0.373612, 0.854094, 1.092571, 1.066606, -0.742925, -1.399240, -0.229709, -0.043383, 1.223743, -0.173675, -1.400634, -0.751707, -1.184195, -2.087879, -1.150729, -3.714264, -1.186671, -2.254113, 0.601697, 2.189452, -2.337301, 39.923878, 1.442919, -6.136178, 0.997319, 0.443302, 1.893847, 2.134698, -1.521084, -2.211249, -0.605163, 2.920052, -1.224859, -0.488684, 0.174961, 1.765274, 0.043963, 1.291592, -0.027254, 0.362563, 0.535911, 0.055040, 0.965669, -0.998588, -0.965053, 0.910799, -0.635001, -0.755441, -0.216526, -0.230160, 1.201640, -1.004903, 4.072732, -3.240913, -0.649104, -0.221394, -0.652424, -0.344757, -0.897428, 1.605987, 0.713921, 1.132797, 3.124265, -4.620122, -1.233402, -2.835165, -1.103005, -0.272193, 1.986733, -2.367633, 2.668632, -0.008239, -0.637520, 1.968715, 0.809258, 1.002304, 0.056953, 0.215855, -0.626684, -1.733515, -0.024491, -1.386338, 3.167961, 0.568686, 1.992621, -0.684541, -2.265276, -4.273585, 3.601405, 0.195045, 1.275817, -1.411008, -1.897714, -1.553356, 0.561319, -1.729886, -0.169032, -2.529392, -1.037613, 3.703908, 3.448970, -1.749579, -2.396593, 0.606615, -1.292753, 0.056893, -1.645130, 1.672513, 0.194815, 0.816912, 1.116137, 1.635205, -0.354506, 0.295139, -1.623365, 1.817265, -0.269048, 1.538462, -1.011906, 1.269872, 3.067373, 0.211793, -0.732102, -0.391167, -2.233507, -2.231629, -2.095895, 0.727035, 1.411009, -0.877170, 0.340414, -0.075507, 0.745245, -1.728194, -0.083872, 3.342732, 3.725381, -0.929764, -0.378220, 0.475852, 0.933637, 0.376741, -0.074854, 0.588735, -1.907476, 2.541552, -0.915148, 1.144570, 1.298011, -1.079804, 0.938594, -0.981594, -1.589298, -0.491617, 0.186467, 0.123616, -2.508697, -1.810076, -1.044597, 1.582594, 0.258120, -0.006478, -0.524830, -1.418215, -2.215704, 0.407240, 1.460952, -1.709083, -1.259769, -1.154309, 1.705392, -0.556009, 1.421965, -0.456303, -2.493070, 0.633029, -0.641055, -0.343872, 1.294190, 2.638387, -0.840192, -0.314008, 11.737316, -1.054496, -0.697600, 0.080303, 0.745963, -0.622276, -1.977137, -1.301768, 0.361449, -1.011482, 0.635370, -2.172765, 1.191688, 1.330068, -3.170839, 0.580865, 1.834052, 4.704100, 3.587384, -3.430662, 2.697701, -0.127231, -1.113158, -1.443183, -41.022354, -0.110150, -1.822315, -0.207086, -0.767085, 1.362608, 0.168367, 2.363555, -0.680236, -0.087632, 1.561876, 1.680736, 3.146271, 1.144814, -0.160254, 1.950765, -1.504552, -0.332065, -0.360760, 1.558014, 0.636103, 0.785471, -1.704202, 0.857587, 1.007262, 1.093875, 2.384038, -3.869771, 0.113459, -0.496516, 1.827181, -1.552594, -2.527205, 0.675593, 0.022145, 1.097947, 0.381638, 1.416903, -1.212677, -0.577768, -1.223361, -0.348717, -0.045128, 0.010273, -1.609390, -1.338410, -1.802680, 0.713390, 1.108162, -0.479905, -2.831465, 0.522394, -0.296082, 2.357434, 2.456294, -1.470178, -0.920917, -2.398948, -0.204510, -1.523816, 0.510649, -0.536751, 1.747033, -1.597472, -0.743485, 0.238079, -1.569961, 1.913989, 1.672282, 2.897475, 0.532682, -2.069995, -2.510436, 0.142607, 0.483538, 0.606211, -0.185770, 0.913036, -0.824144, -2.868243, -1.664191, -0.130939, -1.009496, -2.834906, -2.953358, 1.251532, -2.528788, 0.715784, 0.619456, 1.275110, -0.968184, 0.420924, 0.751423, -0.012198, -3.663082, -0.932058, 2.118438, -0.064653, -0.722701, -0.739168, -0.247635, -0.043738, -0.452323, 3.432029, -0.554806, 2.927828, -0.212855, 0.003955, -0.357271, -2.083921, -0.759885, 1.128178, 0.407497, -0.346585, 0.254764, -0.036644, 1.292589, 0.889476, 4.094336, -1.213452, -0.292327, 0.835652, -1.123920, -1.357339, 0.583533, -0.212889, -2.109420, -2.039322, -2.049470, 1.393434, 0.227105, -1.943285, -1.751864, 0.368406, -0.076198, -0.553293, 0.431516, 1.632055, -2.800219, 0.425088, -0.216394, -0.355467, 3.298864, -1.703126, -0.529324, -3.259636, -0.005124, 2.216942, 0.056490, 1.522583, 0.341275, 0.279314, -0.579731, 2.031629, -1.494830, 1.357635, -0.870218, 0.445755, 1.674716, 0.256306, -0.997950, 1.777514, 2.048767, 0.416811, 4.965404, 0.141465, 0.773394, 1.545753, 0.027247, -3.331308, 0.706755, -2.109440, -1.031737, 0.457077, -0.085427, -0.124898, 2.836118, 0.668133, -0.377359, 0.098304, 0.442726, 0.469875, 2.438758, -0.113918, -0.708185, 0.850951, -0.269033, 1.265060, 1.252891, -0.723365, -0.106239, -0.572127, -1.127482, 0.805131, 0.789896, -1.664021, -0.853497, -1.080191, -1.777647, -1.330180, 0.512299, -1.983276, 0.702110, -7.340586, -2.208733, -3.390751, 0.570193, -0.615102, 1.125478, 1.215835, 0.822542, 0.171175, -0.069884, -0.123099, 1.784375, -2.423002, -0.737408, 0.343006, -0.787060, 1.550688, -1.225983, -4.489320, -0.243526, -1.443461, 0.465390, 1.505774, 1.369661, -1.840881, 5.110126, 2.507883, -0.404607, 0.081059, 0.555840, -1.737800, -0.521636, 0.646257, -0.794755, 1.622429, -1.143865, 0.247397, -3.387873, -1.999178, -0.475409, 0.254099, -1.216121, 0.947898, -0.436287, 0.588959, 3.871037, -2.230380, -1.394659, 0.590486, -1.651382, -2.237124, -3.214393, 0.829730, -2.667794, -1.414842, -2.266326, -0.213557, 3.285220, 0.244696, 1.276757, -1.010511, 0.954373, 2.183401, 0.428658, 0.591693, 0.545552, -0.059087, 1.103354, 1.915429, 2.638312, 0.722118, 0.093238, 0.864807, -0.179161, 2.299291, -4.168321, -0.407841, -2.142166, -0.239048, 2.303017, -1.151473, -0.927492, -0.527071, -0.511980, -0.261069, -0.298833, 1.963749, -0.033439, -1.881962, -0.522094, -2.649604, 1.647931, 0.776876, 0.432121, -0.191135, -2.145623, -1.231426, -0.372664, -0.452278, 1.274758, 0.170175, 1.026496, 0.073504, 0.640993, -1.226367, 0.783692, -0.448901, 1.416342, -0.930619, -0.322302, -0.675186, 1.486036, -2.058913, 1.081755, 1.013078, -0.047287, -0.117357, 0.994664, 0.758084, -0.193374, 0.876904, 0.891312, 1.532211, -0.034118, 0.127642, -0.266024, -0.440258, 0.074581, 1.515373, -9.780887, 0.753208, 0.330636, 2.945389, -0.220795, 0.140223, -0.131821, -1.725950, -2.171350, -0.204008, 0.595772, -0.116645, 1.457196, 3.947017, -0.784874, -0.888277, -1.782836, 0.817804, -0.192799, 4.299161, -7.505760, 1.463305, -0.106789, 1.174378, -0.811028, 2.220318, -0.583385, -1.241430, -1.802632, -0.958205, 2.339732, 2.392081, 0.028450, 0.596011, -1.068670, -0.249349, -2.410244, -0.316100, 3.512356, 1.815074, -2.563461, -1.023207, -1.255088, 0.645512, 0.079494, -0.294559, -0.094625, 3.387850, 0.226355, -2.717346, 0.559185, 1.418335, 0.593394, -1.490641, -0.128680, 0.076089, -1.944839, -1.451210, 2.537951, 0.006042, 0.122956, 1.893836, -0.406169, 0.839640, -0.788404, 4.068511, -0.677647, 1.193237, 1.380761, -1.895364, 1.780263, -0.083210, -0.052489, -2.717657, 0.885126, 0.945189, 0.714716, -0.331114, -0.276344, -3.294456, -1.361389, -1.806947, 0.547034, 0.398358, 2.003053, 0.901725, 0.410779, -0.864764, -1.618749, -1.349252, -0.911480, 2.053375, -1.349382, 0.849684, 1.225166, -0.703413, 1.470940, -1.646121, 1.467508, 1.322925, 1.738584, -2.498531, -1.925045, -0.384096, -0.101580, -2.523852, 0.767522, 0.719983, 1.575168, -0.290012, 1.028273, 1.360790, -1.643553, 0.725088, -3.634740, -0.672812, -1.429943, -0.466053, -1.564547, 2.244541, -0.990287, 0.477694, 0.142603, -1.236258, -0.260318, 0.138124, 0.848498, 0.986988, 1.307112, 0.753924, -0.791700, 0.239069, 1.626977, 0.944337, 2.356667, 4.252893, 1.440436, -0.834279, -3.924565, 1.486321, 1.781534, 1.588995, -2.617434, -1.426156, -1.792942, -0.477611, -0.760941, 2.425961, 1.197546, 4.233758, -2.818418, -0.567543, 1.363195, 0.341048, -2.289856, 0.309621, -0.637264, 0.347269, -0.461381, 2.523387, -1.344993, 0.216397, -10.708355, 0.919107, -2.543254, 1.904163, -0.058510, 2.088098, 0.810954, -0.385477, -1.210636, 0.094202, 0.535237, -1.502995, 0.344693, -1.197936, -1.386554, -0.128568, 1.160851, 2.858701, 1.907802, 1.539349, 2.433446, 2.448377, 2.303145, 4.565366, 2.033096, -2.155319, -0.801175, 0.520414, 1.972140, -2.310103, 0.173755, 0.974195, -1.724003, -1.241399, -2.017481, -2.318946, 0.835495, -1.525946, 0.359016, -2.165900, 0.091703, 4.673479, 1.606368, 0.459343, -0.392127, -1.453846, -0.353931, -2.310848, 1.236771, -0.827690, -1.652173, 8.877085, 2.143196, -1.679855, 0.654059, 2.409938, 0.361839, -1.841513, 0.169567, -0.140878, -0.171590, 1.312745, -0.134062, 0.724259, 2.047288, -0.841137, 1.260102, 1.002513, -0.647772, -5.369482, 5.233755, 0.599639, 1.172586, -1.073408, 0.885488, -1.003686, -3.339315, -0.038161, 0.559816, -0.626176, -1.193738, 8.828257, -1.190498, 0.449845, 1.863093, 0.556650, 1.450109, -0.126581, 0.105580, 0.397996, 0.313131, 0.914442, 0.404302, -3.006561, -1.363588, -0.628297, 0.217007, -2.647120, 2.862575, -8.684146, -0.418185, 1.437103, 0.828107, 0.458347, -0.435039, 1.646789, 0.926643, 2.910391, 0.709955, 0.771045, 0.037429, -4.088374, 0.162264, -3.077456, 1.285309, 1.316735, 1.387373, -1.066935, -0.645049, 1.368517, -1.300493, 1.187293, -0.031566, -0.225020, 0.646678, -0.172955, 3.641774, 0.928097, -1.952956, 0.098479, 0.563732, 2.789780, 1.441335, 2.300636, 1.045073, -1.739837, -0.522630, -0.936236, -0.113540, -0.295716, 0.952094, -12.808439, 0.464352, 1.135275, -1.872086, 0.042147, -3.117124, -0.354251, 2.127711, 0.385427, 1.285458, 0.280779, 1.432706, 3.229187, -0.091171, -2.339383, 2.241873, 0.316786, -2.255201, -0.556684, 2.200058, -0.417242, -1.281321, -0.334675, -0.578597, -1.088588, 2.267492, -12.715529, -0.348637, 1.471534, -1.676870, 1.577533, 0.676559, 0.080321, -1.086842, 0.430531, 0.437493, -2.491856, -1.025787, -5.674278, 0.392208, -4.118344, 1.785830, 3.346591, 0.304010, -2.162539, -1.001513, 0.018220, -2.174213, 0.871597, -0.414440, 0.716925, 0.457252, -1.066866, -1.192133, -1.134685, -0.413272, -0.989214, -0.402608, 0.400279, 2.370824, 0.565737, -1.827672, 0.184565, 1.494704, 1.196620, 1.353731, -1.431783, -0.185922, -0.168908, -0.945395, -1.528083, 0.495618, 1.339592, -0.752455, -0.017693, -1.226821, -1.587509, -1.819557, 0.250778, -0.129144, -0.049336, -0.556312, 1.388641, 5.209940, 7.888215, 2.105105, -4.130414, -1.113280, 0.696917, -1.569371, 1.314536, -1.086163, 1.195435, -0.062427, 1.109358, 2.636901, 0.217930, 0.296666, -1.974165, -1.735777, -2.618426, 1.400363, -0.096162, 0.028053, 2.971062, 0.302502, -1.890169, 0.857939, -2.612601, 1.933837, -2.467517, 1.603533, 1.537552, 0.085175, -2.052856, 0.524604, 0.883950, 0.739990, 1.870276, 1.625467, -4.919993, 1.174270, -2.426991, 1.683943, 2.972154, -1.803723, 1.008075, 0.248989, -0.385458, -0.196188, 1.270961, -0.229944, -3.168777, 1.614013, 1.595189, -0.410707, -1.044544, 0.778994, 0.653459, 0.350870, 4.649518, -1.140979, 0.838352, 1.230573, 2.527621, 1.062342, -0.298138, 0.888464, 0.313968, 1.429981, -1.374527, -0.592750, -1.111212, 0.250075, -1.495723, -0.555394, 1.749684, 1.169427, 0.184614, 0.222946, -2.256650, -1.231119, 1.744124, 2.778740, -0.821602, 0.101260, 1.169010, 0.798608, 0.031937, 0.170795, -7.833639, -0.522136, -0.191751, -0.644957, 1.360851, -0.476764, 0.153982, 1.444253, -0.697542, -1.839324, 1.011768, -1.918130, 1.279057, -0.911504, 1.704320, 1.411318, 0.101799, -1.732975, 1.152074, 1.164443, -0.210665, -0.760696, 0.550097, 0.398409, 1.169000, 4.403631, 0.044010, 0.429579, 0.020382, 1.313668, 0.072535, -0.443568, -0.970095, 0.235293, 3.075127, -0.917624, -0.191897, 0.649122, 0.604821, -2.162972, 1.809951, -0.566550, -0.368535, 0.561067, -1.963957, 0.591505, -1.989305, 2.010246, 1.025328, 1.118531, 0.765658, 1.132761, 0.590744, 0.309546, -0.072684, 0.927601, 0.402006, -0.279634, 3.336041, 0.363340, -0.084176, -0.376793, -0.792103, -0.910649, 0.842618, 0.704083, 1.336435, 0.633091, 1.232460, -0.126867, 1.860313, -0.582771, 2.027243, -0.734689, 0.396841, -4.301981, -2.570962, -2.346615, -0.294627, -9.963593, -0.950176, -0.416823, 0.185868, -0.575314, -0.248928, 0.115231, -0.267261, -0.893238, -0.038151, -1.312479, -0.859603, -0.981597, 0.905031, -1.797287, 1.575514, -0.293734, -2.374883, -2.280471, -0.114908, -8.689425, -3.053759, -0.254290, -1.284394, -2.571275, -0.515252, 1.703866, 0.979483, -1.641806, 0.677989, 0.632244, -0.935906, -0.334838, -0.395663, -1.000913, -1.619586, -0.240378, -0.943641, 2.728803, -0.003069, 1.764210, -1.329910, 2.901473, -0.663974, -2.739002, -0.557714, -1.467272, 0.430610, -2.140990, -1.018700, -8.117669, 2.187021, 1.585709, 0.303729, 2.666512, -0.322125, -0.510872, -2.895965, 0.332371, -0.563060, -4.857492, -0.213120, 0.974745, -0.822440, 1.557947, -4.240340, -2.603575, -0.108165, -1.109751, 2.159279, -3.169286, -0.512767, -0.863623, -0.091780, -1.966987, 2.861250, 0.022738, 1.359011, 0.319737, 1.852149, -0.047252, -0.426285, -0.759734, 1.233245, 1.222208, -0.428217, -1.377964, -0.014197, -1.666247, -0.693576, -2.364216, -2.303179, -1.875806, -2.193077, -2.161515, 0.730956, 0.106795, -0.440026, 0.882222, 1.271376, 0.223913, 0.026457, 0.875005, -3.332191, 0.711084, 1.622856, -1.125061, -0.091555, 0.591242, -1.053302, -0.374932, -0.451309, 0.116873, 0.916229, -1.526904, 1.396491, -4.383001, 0.277899, -0.744718, 3.206128, -0.688903, -1.531305, 3.416818, 2.721693, 2.085379, 3.218796, -6.358753, 1.035356, -2.443428, -1.629758, -0.956840, -2.862966, 1.097265, -0.132868, -1.223482, -0.695732, 0.078407, -1.925692, -1.591343, -0.446446, -0.413490, -3.946516, 0.004555, -0.718744, 0.671976, 0.724541, 0.277439, -1.528037, 0.149476, 1.957371, 0.107017, 1.056521, -0.852286, -1.079987, 1.865555, 1.076413, 0.002297, 2.019732, -3.465175, -1.121418, -0.474052, -1.834379, 0.894442, 0.671292, 1.058087, -0.456843, 1.975728, 1.927830, 0.147125, -1.154134, 0.253917, 3.004951, 0.158338, 0.203460, 0.986412, 0.876946, 3.303209, 0.924066, 2.434523, 1.368330, -1.493254, 0.657588, -2.332864, -0.716405, -1.623935, 0.734781, 2.475377, -3.862651, 0.639123, -1.024320, -2.099458, -1.505555, -2.106091, 0.105197, -1.276832, -1.604650, 11.007366, 16.257780, 1.212517, 0.889604, 3.437125, -1.665995, -0.542497, -0.793582, 0.072068, 0.891223, -1.907271, 1.315802, 0.976274, -2.103378, 0.598498, 1.317193, 2.073358, -5.785201, 0.120337, -0.128886, -0.625547, 1.823134, -0.627097, -0.074800, 0.489213, 0.000787, -0.695088, -0.151693, -1.164222, -0.719642, -4.420443, -1.641346, 0.223747, -4.239794, -2.342813, -0.701424, -0.777337, -0.055989, 2.989524, 1.142629, -1.108115, 1.921261, 2.104734, 0.713200, -1.836161, -0.769459, -0.096665, -1.405661, 1.133891, 0.677240, -0.079669, -0.786968, 0.757151, -1.115593, 1.729968, 3.429117, 3.333342, 0.889537, 1.064027, 2.452963, 0.146189, 1.487560, 0.159354, -1.003925, 3.638438, -1.171568, 0.858463, 1.559678, -1.309983, 0.294997, -1.911492, -0.658933, -3.117077, -1.047560, 0.714687, -2.235785, -0.924417, 1.562038, 1.057431, -0.458399, -0.513814, -0.561772, -1.739037, 1.467458, 0.620015, -0.726114, 0.533329, -0.823658, 0.602171, -4.322502, 0.547450, -0.053952, 5.291243, 0.374018, 0.543503, 0.658229, -1.971978, -0.861369, 0.358919, 0.152244, 0.556775, -2.041901, 0.755251, -0.084016, 2.247265, -1.645395, -0.975744, -0.170778, 2.687211, 0.454293, -0.637791, -0.689573, 2.783953, -0.462319, 1.623681, -0.080127, -2.483746, 0.395303, 2.702187, 0.863832, -0.829897, 1.051659, -0.179475, 1.012557, -3.725295, -2.425763, 1.506878, 0.261926, -0.952743, 1.053141, -0.772923, -0.683897, 3.952535, 2.515576, -1.145893, -1.876673, 1.160748, 0.543679, 0.466676, -0.385127, 1.240535, -0.393820, -2.291663, 0.888950, -1.278874, 0.038750, -0.752375, 2.060377, -0.336541, -1.480818, -1.426695, -1.427118, 2.019039, -2.795073, 4.602587, -0.728483, 0.066261, 0.140672, 0.995434, 6.288233, -0.336479, -1.012862, -0.521627, 2.346851, 1.099883, -1.747922, 0.289940, -2.412572, -0.726209, 1.113508, 1.037497, -0.935774, 1.114324, 1.137784, -0.378494, -0.369090, -0.495506, 0.839105, -4.885697, 2.921585, -0.534774, 2.706858, -2.904579, -3.026774, 1.074224, 0.988017, -1.126216, -2.458224, -0.514051, 0.257682, 1.672144, -3.153304, -0.512526, -1.063563, 0.611227, -0.169314, 0.288846, 0.492560, 0.557903, 1.448651, 1.216152, 1.514556, 0.348303, -1.505949, -0.923314, 3.453034, 1.904600, -0.675914, -0.474720, -0.985341, -0.153247, 1.354074, 14.742016, 0.398167, 2.527965, 1.902573, 0.569317, 0.713961, -0.157248, 0.583093, 1.446733, -0.671570, 3.754446, -1.230091, 2.326427, -1.630548, 1.877837, 0.963890, 2.939709, -0.352764, -2.149770, 0.465506, -0.063179, -1.710148, 0.230100, -0.749501, -1.599947, 0.353103, -1.376222, -3.326641, -0.228345, 0.709135, 0.279200, -0.913855, -1.260106, -1.632902, -0.528416, 0.245233, 1.025672, -1.362333, -1.633941, -0.171049, 0.478239, 2.395704, 1.798306, 0.518881, 2.364272, -0.769217, 2.540560, 1.193590, 2.762551, 1.171583, -0.288009, 3.315905, 0.178900, -0.414532, 1.629692, 3.644429, 0.047435, -0.409813, 1.277344, 1.235892, -0.370595, -1.994537, -1.594649, -2.062027, 4.117460, -1.693915, 0.134951, -0.276196, 1.733694, -0.138643, 0.993175, 0.659948, -1.959156, -0.568687, -0.549599, 1.432800, -1.232484, -3.082007, -0.631581, -2.420118, 2.023217, 1.324583, 0.041360, 3.613342, -4.568541, 0.285068, 0.292985, -0.218380, 3.284291, 1.405751, -0.301350, -1.727242, -1.090432, 2.118229, 0.914769, 0.380845, 0.296591, 0.511121, -0.837049, -0.388810, -0.239591, -4.444277, 4.205221, -0.768113, -0.578623, -1.128701, 1.552697, 0.367937, -0.099581, -1.281450, -0.908604, -0.632442, 13.450798, -2.382877, -3.409620, -1.395153, -0.904329, -0.477906, 2.242418, 1.626651, 1.743318, 0.201355, -1.874433, 1.069358, -1.122438, -0.513217, 0.798678, -2.774002, -3.890982, 1.194817, 0.417884, -2.555126, -1.121142, -1.795019, -0.905862, 0.113841, 0.568253, 0.533732, 0.223853, -0.951292, 0.625884, -0.039080, 0.570316, 1.236212, 0.025467, -1.700714, 5.611750, 1.285293, 0.344259, 0.667707, -0.425748, 0.564495, -1.901882, 1.431199, -1.215952, 1.253814, -0.354065, 0.312732, -3.331754, -1.358605, -3.347685, -1.173771, 2.645103, -2.667670, -1.189343, 0.628767, 0.290056, -1.889051, 1.207483, -0.013234, -1.082175, -0.463480, 1.240325, -0.788110, 3.318880, 1.457310, -1.047825, 0.244308, -1.829554, 3.589651, -0.124967, 2.177041, -1.269532, 0.687182, 0.477322, -0.286502, 2.742045, -1.479311, -0.366762, 0.130173, -3.005942, -4.997714, -1.074178, -1.553302, 2.917926, 0.381329, -1.468071, -1.214693, 0.776294, 0.101943, 2.781591, -0.340745, -0.709129, 1.041831, 0.696277, 0.774845, 2.691426, 0.668740, -0.311207, 1.233782, -1.170120, -0.034383, -1.132921, -1.765879, 0.484065, 0.475109, 0.550853, 0.381090, 0.574175, 0.871582, 2.628418, 1.365183, 1.224013, -3.376759, 1.086175, -0.021419, -2.019457, 0.234232, -3.502301, 1.270517, 4.130678, 1.725621, -2.442235, -10.042397, -0.838784, 0.022949, 2.967072, 0.118950, 2.288779, 1.842968, 2.072146, -0.317232, -1.301560, 0.340098, 0.647872, -0.179999, 0.620119, -0.987134, 1.012957, -0.834723, 0.834885, 0.751951, 1.262253, -1.385338, -1.551879, -1.337877, -1.756028, 0.510630, -2.394951, -0.205486, 1.575394, -0.480835, -0.409149, -0.437463, -2.282324, -1.881278, 1.116361, 1.136089, 24.378939, -0.393524, -3.075753, 1.290074, 1.087091, -2.154333, 0.970394, -1.926258, 1.618542, 0.412278, -1.739195, 1.623174, 1.373538, -0.979996, 0.275235, 0.773335, -0.732338, -2.702239, 0.798534, -1.287799, 35.395630, -0.367900, -1.554014, -3.089915, -0.288484, -2.492246, 1.115393, -1.127128, -3.936405, -0.125779, 0.485530, 0.389982, -1.279955, -1.730957, -0.593967, 0.948370, -2.064061, -0.272153, 1.658313, -1.792256, 1.410611, 1.607257, -0.524023, -1.193940, -0.477432, -0.203510, -5.396678, 0.878312, -0.600852, -0.958319, -1.770715, 2.374568, -0.837991, 0.792972, 1.333605, -2.388574, -0.201241, 0.498677, 0.179003, -1.049678, -1.267674, 0.086085, 1.527040, -0.410888, 0.113187, -0.560847, 0.438254, -0.419042, -8.495057, 1.416575, 0.312171, -4.181562, -3.007250, 0.894191, 0.282944, -0.465586, -1.058151, -0.204914, -0.380061, -2.493015, -0.319261, -2.098388, -2.529561, -2.387559, 1.457654, 2.064481, -0.206003, 0.331642, -0.909741, -0.725373, -1.512754, 0.902448, 1.214116, 0.187296, 0.355552, -1.346207, 3.032812, -0.008650, 0.665337, 0.523351, 2.378387, -3.201352, 1.975095, -0.343192, -0.887129, 0.839616, 2.507690, 0.034270, -0.425381, 0.244614, -1.063135, -0.416553, 0.151447, 0.544476, -0.904222, -1.981529, 0.008429, -1.530195, -0.380718, 2.639323, -1.974096, 2.537336, 0.128485, 0.057991, -1.839553, -2.247260, 0.850893, -0.609513, -0.073828, -6.912158, -0.139270, 0.179913, -1.241016, 0.320185, 1.688670, 2.298871, -0.673911, 1.046304, -2.560003, 2.457667, -1.773535, -0.873110, 1.303595, -1.350625, -2.650466, 1.821359, -0.091552, -1.048395, 4.371029, -1.531572, -3.530663, -7.938312, 0.371839, -3.025643, -1.249872, 2.221552, 3.361130, -0.400779, 1.653993, 1.089241, -0.370755, 1.477701, -0.303671, -0.078472, 1.358668, -0.792539, -1.049296, 0.496935, 0.733483, 1.849233, -1.298069, 1.658483, 1.070524, 1.064211, 2.341386, 2.022804, -0.712600, -0.828958, -4.590330, -2.193731, -1.187169, -0.390252, -1.434384, 0.475188, 0.472390, 1.016687, 0.816242, -2.084652, -0.264609, 2.821970, 0.642034, 0.144703, -0.085207, 3.882366, -0.335585, 2.945193, -8.780211, -1.788703, -0.816980, -0.324785, -0.251749, 0.550231, 1.536315, -0.976531, -0.001636, 0.025487, 2.723643, -1.050987, 1.859178, 1.747509, 0.632397, 0.674326, 1.942476, 0.170618, 0.628320, -2.931181, 1.509731, 0.052616, 0.396503, 2.053801, 0.786840, -1.765508, -2.049108, 1.391213, -1.846742, 0.302443, -0.235106, -4.113284, 2.388442, -4.504088, 1.407906, -0.792122, 0.238252, 0.770481, -2.253059, -0.076944, -2.216919, -1.822923, 1.290645, 1.323500, -1.547410, 1.743029, -1.069493, -0.496441, 0.411108, 0.256616, 1.004339, -0.964894, 1.061500, -0.336986, 0.609277, 1.396947, 0.389251, -0.800786, 2.296143, -0.148887, 1.665507, -1.662055, -0.582016, -2.904270, -1.138959, -1.115410, 0.504910, -0.440490, -1.917381, 1.905744, 2.691540, 0.668207, -1.137727, 1.715427, 2.627678, -0.848056, -2.206196, -0.485603, -0.104143, -1.581868, -3.105368, -0.136548, -3.890651, 2.893072, 1.311447, 1.207675, -0.110239, -0.015520, 3.073158, 0.722297, -0.317380, -1.639348, 2.564957, 0.707945, -2.089616, -1.077968, -1.993640, -0.184228, 2.603096, 0.495441, -0.690588, -1.996336, -0.615599, 1.252049, -1.305657, 2.258763, -0.367589, 0.744874, -3.983390, 0.641868, 0.682344, 1.534273, -0.078887, -0.638862, -0.436887, 1.282581, -0.474767, 0.933970, -0.562142, -1.196221, 0.620703, -1.794334, -0.201897, -0.273618, 0.759434, -4.578209, 1.045878, -0.396665, -1.393743, -0.258993, 0.909860, 0.065080, 2.534648, -0.621033, -2.463706, -1.267582, -0.949879, 0.684556, -3.153200, -1.322336, -0.846883, 0.756270, -1.246623, 1.325826, 0.269144, 0.655697, 1.508641, 0.955546, -4.049324, 1.236385, -2.208986, -2.164684, 0.483166, -1.556875, -2.399807, -0.071894, -0.598155, -1.504630, 0.909837, 0.441980, -0.142267, 3.186562, -9.105115, -0.706504, -0.904092, -1.254219, 1.707472, -0.484811, 0.243699, -1.906812, 0.557756, 0.399335, -0.668094, 1.150291, -33.314419, -4.579463, -0.827204, -2.219878, 0.864965, 0.843359, -3.120395, -3.499207, -1.056821, 2.018512, 1.058076, -2.007281, -1.928121, -3.263878, -0.079084, -1.419108, 0.705255, -2.721349, -0.210468, 1.977742, 0.119329, -3.372694, -1.065624, 0.044773, 0.949414, 1.982919, -2.992223, 0.043504, 0.458767, 2.387087, 1.143641, 2.626701, 1.635916, 1.130192, -1.146586, 2.295970, -0.108903, 1.978274, 0.249793, -1.822635, 1.072817, -1.037818, -1.654801, 0.506986, 0.223898, -0.801533, 0.471886, 0.821947, 0.165247, -1.029804, 1.522923, 1.681942, -34.283497, -1.443581, 3.120758, 0.109961, 1.252267, -1.838808, 4.105927, 1.430098, 0.795596, 2.285416, -0.350729, -0.183396, 2.008162, 2.535472, 1.442522, 0.067094, 1.726215, -0.465237, 0.228092, -0.898866, -0.351523, 0.943222, 2.762748, -0.150182, 0.072098, -3.666024, 0.861893, 1.339004, -4.193988, 0.934029, 1.222872, 1.534581, 0.536860, 0.175307, -4.649261, -0.155842, 0.486639, -2.415943, 4.921558, 0.074272, -0.523580, 9.426488, 1.450721, -0.273441, -0.685305, -0.208207, -0.141061, -0.689686, -0.430765, 1.203310, 0.457785, -0.636253, 1.884096, 0.180858, -0.601352, 1.059752, 1.353793, -1.144091, 1.012244, 0.111028, 1.282956, 0.539753, -2.136105, -0.362957, 1.343894, 1.246425, 0.270625, 0.846098, -1.814227, 0.723642, 0.896651, -0.492568, -0.483125, 1.335945, -1.528607, -12.522959, -2.514193, -0.385231, -0.627579, 0.507236, 1.104174, 0.487314, -0.776387, -0.105157, 2.839599, -0.512341, -1.799638, -1.330242, -0.425150, 0.598084, -9.454448, -0.575648, -1.355067, -1.314143, -3.041453, 0.646404, -0.381917, -0.618457, -0.122956, 1.836147, 0.686982, 1.346205, -0.397530, 0.177667, -0.312596, 0.959667, 0.490264, -0.229002, 2.652546, -0.523528, -0.295107, -0.722657, -0.770971, 0.501242, 2.509320, 0.498323, -2.765550, -0.476504, 2.484814, 1.173968, 0.066951, -0.033656, 1.939883, 0.595368, -0.492711, -0.485204, 0.524125, 2.012069, 1.902331, 1.441705, -1.873747, 8.485007, 0.809767, 2.354154, -1.817922, -0.857522, -1.486599, 0.853559, -3.121060, 1.469213, -0.247292, -0.377132, -1.203432, 1.732122, -0.968094, -2.430906, -0.646179, -0.096946, -1.891861, -1.449054, -1.289801, -0.931208, 3.503360, 0.109478, -2.145627, 1.445237, 0.036277, 0.486036, -1.857045, -0.748751, 0.455339, 0.367962, -2.402283, -0.665134, 2.883188, 0.057114, -2.703108, -1.443505, 0.250365, -0.242302, 3.324877, 0.508369, -0.850818, -1.103852, 0.660547, 3.891412, -0.431533, 0.720094, 0.351299, -1.011191, 2.850002, -2.083087, -1.732875, -0.878147, 0.754441, 1.108804, -1.122806, 0.565206, 0.396117, 3.356467, 0.028610, -1.452879, -1.373615, -0.454277, 0.141389, 3.266813, 12.898273, 1.677444, 0.586111, 1.705890, 1.566172, -2.538870, -2.231463, -1.102204, -2.527648, -1.085532, 0.478172, 0.187650, -6.529508, 0.846004, 2.000021, 0.900888, 1.543347, 1.928105, 0.956930, 1.169501, 2.499456, 0.512169, 1.320805, 0.813092, -0.862943, -0.914188, 1.019531, -1.543529, -3.052305, -0.000092, 0.458211, 1.184352, -0.189562, -0.504885, 0.651876, -2.231333, 0.150909, 2.522482, 1.546768, -1.469670, 0.283542, -0.038828, -0.221745, -0.613487, 0.986294, -2.087503, 1.868805, -0.862680, -2.028013, -1.342783, 2.996248, -0.964869, -0.889650, -1.940610, -0.491476, 4.659435, -0.629713, 0.880112, 0.013399, 1.543464, 1.811594, -1.919105, 0.579925, -0.054963, -0.611301, 1.213541, 0.044412, 2.428571, 1.650359, -0.011562, -3.940373, 0.219158, -0.250527, 2.196884, 2.333903, 2.026985, -2.534095, -1.620452, 1.754802, -2.421583, 0.130233, 1.375082, 1.599321, 1.592287, 1.072141, -1.522067, 0.396378, -1.969304, 1.184472, 1.064752, -0.051353, -0.429749, 3.536501, 1.289551, -1.872306, 0.962450, 0.012061, -0.633179, -5.554706, 3.344516, -1.520184, -0.216105, -1.681684, 4.008332, 2.174610, -0.961140, -0.686181, -1.402003, -0.401292, -0.713743, 3.171560, -0.512009, 0.219556, 0.559843, -0.798941, -2.584601, 1.708179, -1.446687, 3.116506, 0.701865, 0.474630, 0.662607, -0.341657, 3.770340, -1.970125, 0.079163, -0.033681, 2.390205, -0.802647, -0.419331, -0.633927, 0.158110, -1.025442, 0.967359, -0.777828, 0.594151, -0.833795, -0.444425, -1.413129, -0.685053, -0.072876, -0.182840, 0.716580, 2.238798, -0.586169, 1.660746, -1.904589, 4.398918, -1.573961, 0.223843, -2.490864, 1.426156, -1.309565, 0.297844, -0.122553, -4.125076, -1.175787, -0.808162, 3.899071, -3.727594, -1.969113, -1.416097, 0.433778, 0.566248, 2.189430, 13.170939, -0.437235, 1.563687, -1.015102, -10.265989, 0.488794, -1.776708, 0.143081, -0.918150, -1.271382, 4.251751, 2.227284, -0.468386, -0.818391, -1.076629, 0.803174, -6.778745, -2.332751, 3.314303, -0.854069, 2.525487, -0.036530, 0.712685, -2.247428, 1.711706, -0.839512, 2.300299, -0.371730, -0.143227, -1.000555, 2.392848, 3.181103, 1.384507, 0.174168, 0.332910, -0.797669, -1.976347, -0.308509, -1.638045, -0.427884, -1.578577, 1.571453, -1.381518, -0.068900, -0.008149, -0.877026, 0.584573, 0.371052, -0.273352, -0.496754, 0.173186, 1.495046, -2.134991, -1.174339, -2.233019, -1.859445, -0.982453, 1.005392, -3.032267, 1.807362, -1.046106, 1.126163, 1.836564, 1.041053, -0.354948, -2.419614, -1.992808, 1.194339, -2.727120, -2.960786, 1.990473, -0.249863, -2.641395, 2.517450, 1.227859, -0.836136, -0.104000, 1.062187, -1.290700, 1.327066, -0.019623, 5.412015, 0.280590, 2.486149, 0.634194, 0.961871, -1.225696, 2.310711, 0.309103, -1.714766, -0.548396, -0.683925, -3.823319, 1.304894, 1.286261, 0.745414, 3.101287, 0.891395, 1.379889, 1.123019, 1.053005, 2.294336, 1.014520, -2.056264, 0.847806, 1.838467, 0.661211, 0.911672, 2.108672, -1.296724, -0.994294, 2.204733, 0.060958, -3.965935, -0.375877, 4.837624, 1.608993, 0.728340, 0.086287, -0.533132, 0.689582, -0.234022, -0.732648, 1.564658, 1.145785, 1.684504, -0.951721, -2.480582, 0.639165, -0.163595, 0.932444, -0.321780, -1.511633, -0.700274, -2.130924, -0.589078, 2.558939, -0.031520, -0.466372, -0.661476, -0.310624, 0.919263, 2.138045, -2.980811, -1.367038, 1.232296, 0.824462, -1.153319, -1.915287, -0.933534, 0.126949, -0.378158, 0.226273, -0.199483, 4.345228, 1.491282, -15.042450, -0.682710, 0.315321, 0.360537, 0.098894, -1.003842, -1.970495, 0.370964, 0.917547, -2.802694, -0.426734, -0.872954, -0.628911, 0.498273, -1.146430, -0.303363, 0.423223, -1.333830, 3.313324, -0.368433, -0.129785, -1.392973, 0.117655, 0.907470, 0.862268, 30.468317, 1.916252, 0.630351, -2.087536, 2.504394, 1.090119, -1.520968, -0.478656, -0.055788, -1.631508, -0.471805, 0.263214, 1.737623, -2.067608, 0.341913, -0.636885, -1.703791, -0.276518, -1.013657, 0.735331, 0.755858, -7.211546, -2.701548, -1.511194, 0.885756, 1.038019, -0.747457, -1.262162, -2.229699, 0.619854, -0.506964, -2.198322, 1.772692, 0.917558, -0.345676, 0.773990, 0.077003, 0.222814, 0.570620, 13.318283, 5.154265, -0.133818, 1.382821, -3.381096, -1.348515, 1.883829, 1.128500, 1.431208, -2.484697, 2.832257, 2.581187, -0.643860, -0.449537, 2.258231, -0.865092, -2.880285, -4.159235, 0.895552, -0.865830, 0.576851, 0.660565, -0.503361, 2.477618, 0.888527, 0.642267, -0.203590, -0.203652, 0.518252, -0.481242, -1.399775, 0.071838, -1.687822, 1.408637, -1.714611, -1.262840, 0.550977, 1.361458, 1.093706, 0.218738, 1.694335, -0.816886, -1.712049, -1.776350, 0.478118, 0.873915, 0.157106, -6.387165, 0.410236, 2.402042, -1.496522, 0.605822, -1.822789, -1.357874, 0.902568, 0.476761, -0.997130, -0.898986, -0.416213, 0.628011, -0.191016, -2.507825, 2.000534, -1.100833, 1.287321, -0.746681, 0.251129, -0.452919, -2.379659, 0.467141, 2.813297, 3.019051, 0.894045, 0.513003, 0.247807, 2.164137, -0.861424, -1.187832, 1.431251, -1.880267, -2.707241, 0.882622, -1.083287, -2.503087, -0.331748, -0.253308, 1.218072, 0.465451, 2.239259, 0.033480, -0.988318, -1.066254, 1.289019, 1.244425, 1.395825, -2.000483, 8.393459, 2.063885, 0.351303, 1.736136, 1.678012, 0.466673, -0.735065, 2.319815, 0.587236, -0.219373, 4.404061, 0.069035, -0.615926, -1.054781, -0.281119, -2.315298, -0.598869, -1.699938, 0.399952, 1.414117, 3.159378, 0.971840, 1.468741, 0.738671, -1.217917, -0.005656, -1.837169, -1.033584, 1.689942, 0.296139, -0.159082, -0.282257, 1.146659, 0.589294, 1.239523, 0.584590, 0.732567, 1.415604, 0.229542, -1.803726, 1.402160, -2.028142, 1.227429, -1.287164, 0.215315, 0.091171, -0.837675, -1.208822, -0.082121, -0.389508, -0.755331, -1.751180, -0.248557, -1.661491, 0.068512, 2.337994, 1.096209, 2.914422, 1.787869, 3.141023, -3.783829, 1.704273, 1.094445, 0.711651, -0.020473, -0.531446, -0.418532, -0.951771, -1.334428, 3.076059, -8.316907, 0.220659, -0.314838, -1.326944, 0.409516, 8.980949, 0.336135, -0.271100, 0.718912, -0.735141, 2.867171, 0.454470, -1.684832, -0.667039, 30.702013, 0.883889, 0.531766, -0.452834, -1.168136, -1.349033, 1.579897, -9.245929, 3.323505, 0.975022, 1.259668, 1.623177, 1.048897, 0.252429, 0.494380, -1.923661, 0.298914, -1.570870, -2.745278, -1.792378, 0.748294, 5.594098, -1.439187, -1.536032, 2.820908, -0.333814, -0.420440, 2.032806, -1.904369, -0.007114, -0.871720, 0.579277, 0.640428, -0.765459, 0.173253, 0.786739, -0.283481, 0.956241, -1.522335, -3.105874, -2.173901, -0.282985, -1.197513, -2.059102, -1.084585, -0.648405, -0.228381, -0.374229, 0.766084, -2.314189, 0.978999, -0.764588, -0.302045, -1.880411, -0.740701, 1.435884, 0.545858, 1.537653, 1.470700, 0.422056, 0.317391, 2.463825, 0.103666, 1.313960, 1.550173, 1.734758, 2.650702, 0.980899, -4.777202, -0.605416, -1.772424, -1.299403, 1.382249, -2.957427, 1.906892, -0.161418, 0.556454, 1.740305, 0.006908, 6.763729, -0.278677, -1.766817, 0.121939, 1.601009, -0.562171, 1.424215, 0.542079, -0.740449, 0.396567, -1.856834, -1.074328, -1.226289, -0.696100, 0.464042, 1.506954, 0.080288, -2.100569, 0.921613, 0.558173, 2.382897, -2.477536, -1.884583, -0.054904, -0.519191, -0.969873, 0.959781, 0.231381, 1.963495, -0.388346, 0.184281, -0.207316, -1.240958, 0.680580, -0.335134, -0.166712, 2.305028, -0.148513, 5.842828, 1.085152, 0.373194, -1.860990, 0.415133, -1.615274, -2.057973, 1.233435, 0.705104, -0.175169, 0.365537, -3.113515, -0.644583, -1.924930, 0.268149, -3.299269, -0.044249, 0.769384, 0.747958, 0.502507, 1.253872, 2.567127, -1.534801, -6.195432, 0.239830, 1.061098, 1.498278, -1.147298, -0.012934, 1.279267, 0.256549, 0.932009, 2.801676, 2.601096, -0.057946, 0.010808, 0.225650, -0.888552, -1.055726, 2.577467, 0.214108, -2.550973, -2.023343, 0.168402, 1.144072, 1.188154, 0.007975, -1.752963, -0.353960, -1.058180, 1.402650, 2.647588, 1.578010, -0.623806, -5.160738, 0.940677, -2.087744, 2.504496, 2.207713, 0.738647, -3.554515, 0.701976, -1.332600, -0.665026, -0.837372, 0.036542, 0.498252, -0.463899, -1.281477, 0.037951, -2.718753, -2.519139, 0.797102, -0.969279, -0.007088, 1.170141, -0.555038, 0.601634, -0.725569, 1.991015, 1.571092, 1.635221, -0.142499, -0.239051, -0.623268, 1.279354, 1.173028, -0.863181, 0.221621, 50.890709, 0.459216, -0.634014, -1.232200, 2.451216, 0.997746, -0.070813, 4.072617, -1.105848, -0.707585, 0.129221, -1.666970, -2.221236, -2.830456, 0.259465, 0.248408, 0.067695, -3.389023, 2.130977, 2.272633, -2.333500, -2.583345, -0.533235, -0.838626, 0.675928, 2.638815, 3.564664, -0.836662, -0.464054, 1.113580, -0.739706, -1.093256, -0.590220, 1.229130, -2.069568, -1.136705, 1.287993, 1.863708, -1.156420, -0.803547, 1.157883, 0.605184, 1.073040, 0.025076, -1.252429, 1.357709, -0.065110, -0.653028, 0.069790, 4.635374, -0.414350, 1.573298, -0.352398, 0.538733, 1.097928, 1.345788, -1.038578, -0.814868, -0.190454, -1.798921, -1.403554, 3.546619, -1.749692, 0.060874, 0.027193, 0.516291, -0.682459, -0.085446, -0.505916, -1.123992, -0.330597, 0.062728, -0.334435, 0.210847, 1.488047, -1.092073, 1.356015, 3.290716, -3.166292, -1.558093, -0.793964, 3.183681, 3.087046, 8.326931, -2.114156, -1.133654, -0.425647, -1.151897], - "llama3.2-vision:latest": [-3.078805, -1.697646, 4.600920, 2.136139, -1.034632, -0.008391, -0.913782, 2.059704, 1.261938, 0.314856, -0.313179, -2.066850, -0.737111, 2.565375, -1.834170, 3.589492, -2.283495, 1.541524, -2.779084, 1.555042, -0.403128, -0.313654, 1.059721, 2.011516, -5.750325, -0.567921, 2.883536, -0.803768, -2.809469, -2.432648, 0.492391, -1.709821, 0.459170, -0.239044, 8.034880, 2.616068, -0.866570, -0.313675, -0.625120, 0.185346, 2.798899, -4.163560, 1.685675, 0.213714, 3.102599, 2.594100, 1.551703, 0.171097, 1.063220, -1.646748, -0.573196, 2.495553, -0.831499, -1.483251, 2.482893, -0.726516, 0.400158, -0.145284, -3.525773, -2.577699, -2.111852, 2.612575, -3.947710, 1.561014, 0.210879, 1.777725, -0.181272, 2.283506, 1.242015, -0.025549, 0.957746, -0.246194, -0.365402, -0.971253, 4.563991, 1.184461, -0.725158, 0.744156, 0.815184, -1.637800, -4.056623, 4.689488, 0.249045, 5.708639, -3.208293, -2.040814, 0.996086, -1.227924, 2.843411, 0.019621, -0.695887, 2.414440, -0.218883, 0.861006, 3.528445, -1.916659, -0.619786, 3.501029, -0.093414, -3.492441, -1.942818, 0.583332, -0.770895, 3.453049, 0.567662, -1.010252, 2.237015, 1.537082, -2.533475, 1.944466, -0.632733, -0.084942, -0.505062, -0.605591, -1.142169, 2.620798, -1.018153, 0.334478, -1.901000, 0.225330, 0.280273, -0.487890, -1.552703, -0.719227, -3.150110, 1.320974, 3.735471, 1.304873, 5.072580, -2.039533, -3.994229, 0.517429, 1.133909, -1.422137, -1.049232, 0.620438, 1.793169, -0.807552, 0.183158, -0.329035, -0.641384, -1.251977, -1.347493, 0.622846, -0.984846, 1.988598, 0.296134, -2.342518, 0.632345, 0.030586, 0.768713, 0.437475, -2.269910, -2.521681, 1.579402, -2.377503, -2.105729, 1.594941, -2.283249, -2.322279, -0.602992, -4.794743, -2.325619, -0.762731, -0.249267, -2.240785, -2.439840, -0.962278, 1.386412, 0.236573, -0.113179, 1.957054, -0.561917, -0.478650, -1.572174, -1.008380, -2.675298, 0.341852, 2.328920, -1.276262, 0.605668, 2.816123, -0.020307, -1.087129, 18.897930, 1.684022, -5.963904, 2.153839, 0.326535, 1.419691, 1.839680, 0.152121, -0.443793, 0.515642, -1.226702, -2.276216, 1.966202, 3.762245, 1.711866, 0.671341, -3.111674, -1.433167, 1.465878, -4.289611, 2.021384, -1.996823, 0.042214, -0.409925, -0.922436, -1.212982, 1.138044, -1.522454, -3.863219, -2.712918, 2.454779, 2.351706, 0.824271, 1.279557, 2.515165, 0.249954, -0.514616, -1.325547, -1.099375, -1.732998, 1.701366, 0.903615, 1.821438, 0.362362, -1.209264, -0.406613, 0.343815, -0.926961, 2.383206, -0.131069, 2.882418, -0.794012, 1.588547, 0.142149, 1.719791, -0.516556, 1.390384, 1.157517, -2.705374, 3.152814, 1.891767, -1.230698, -2.227765, -0.907071, -2.255760, -3.713498, 0.391423, -0.435654, -2.571611, 0.210851, -2.287234, -0.744805, -0.057202, -4.453759, -0.717052, 2.153277, 2.111250, 0.667674, 2.024823, -0.369146, -2.738459, -2.051172, -1.193694, 3.160493, 1.187114, -0.628924, -1.819261, -4.691652, -1.558387, 0.826536, 2.190940, 1.007532, -0.518849, -2.375483, -3.083109, 0.717804, 0.421144, 5.876964, -1.385087, 0.068475, -0.233602, -0.143602, -0.377704, 1.460576, 1.612279, -9.952719, -0.860141, 0.009690, 3.998900, -2.104997, 0.633108, -0.447114, 1.322761, -1.092791, 1.379224, -3.249701, 0.721551, -1.894614, -0.110262, -0.938271, -2.394038, -0.695555, 2.551907, -0.666710, -1.585857, -0.769654, 3.607650, 4.665731, 0.014892, 0.578692, -0.946742, 0.955832, -1.183421, 1.518056, -0.766180, -1.134929, -5.823644, 0.987941, -1.487348, 2.554693, -0.725647, 2.227943, 1.358883, 0.959986, -1.001250, 2.546376, -1.055410, -0.220511, -0.629391, -1.756114, 0.999128, -0.474006, 0.172467, -0.361360, 0.087746, 0.241353, -1.736117, 0.641458, 1.114634, 2.208075, 1.267806, 0.002817, -2.069037, -3.187589, 1.093324, -2.561264, 2.068681, 0.762202, 0.362883, 0.582804, -1.015167, 0.106795, 2.033842, 1.161639, 0.377664, 0.452618, 3.925405, 1.386955, 3.140606, 1.267758, -1.133554, -2.524645, -0.955275, 2.711424, 0.770822, 1.372985, 3.062992, 1.510188, -0.150879, 2.115181, -0.426800, -1.095572, -0.336918, -1.500960, -0.942303, -0.226177, -0.733918, -0.031548, 0.936019, 1.037607, 2.740880, 3.736936, 0.013158, 0.783627, -2.299813, -0.794128, 1.020153, 3.677419, -1.156537, 1.287886, -0.735975, 0.271875, -0.541371, 2.015721, -1.514776, 0.217231, 1.587353, 0.698595, -4.475769, -1.661238, 1.777915, 1.495372, 8.351849, -0.992776, 2.359664, -3.734209, 0.888680, 4.039773, 1.338669, -1.540372, 2.827198, 4.282894, 0.447615, -0.325227, -5.473198, 0.158931, -2.082453, -0.471492, -0.908210, -0.229417, -0.812437, -1.082374, -2.361205, -3.133377, 2.533987, -2.056405, 0.330013, -2.951894, -0.052760, -1.753070, 0.411255, 1.419914, 1.145815, -0.162324, 0.704823, -0.556056, 1.881800, 0.478933, 1.291103, -0.936796, -4.451693, 1.578889, 0.183439, -0.686146, 3.263305, 1.043620, 2.137517, -1.800166, 4.166996, 1.393071, 4.382342, -3.738013, -0.384553, 1.550048, -12.476042, 0.659573, -0.358259, 1.873343, -0.827457, 0.601567, -1.043456, 3.390819, 0.940549, 0.042124, -0.425845, 1.186888, -0.495413, -0.849585, -1.485713, -2.023389, 3.484549, -1.132264, 2.287027, -0.184956, 0.862702, -4.589354, 3.184383, -0.465502, -0.568603, 1.523978, 1.301721, 2.099811, 1.112086, -1.031363, -5.436310, -1.959793, 2.574366, -1.392580, -0.500089, -1.667079, 0.266578, 1.470267, -2.391178, -2.849307, -2.344163, 0.151920, 0.573642, -1.668947, -1.790319, -2.520195, 1.811686, -0.149748, -2.250818, 1.122893, -2.718899, 0.765947, 0.051963, 1.497161, -2.095890, -2.668485, -0.750255, 1.363428, 0.544781, -1.652589, 3.956085, -1.285231, 3.104691, 1.370670, -0.483672, 2.409342, -1.230793, 0.431622, -2.930366, -1.572196, -1.468563, 1.024296, 0.239093, -2.253988, -0.034438, -0.042014, 0.333263, 0.990264, 0.407541, -0.725304, -0.715123, 0.676341, 0.069297, 0.102975, -0.306056, -1.024400, -0.794437, -0.089673, 0.943760, 0.686709, -3.228787, 0.669426, -5.304088, 0.056853, 1.038435, -0.478588, -0.638576, 1.946829, 2.407026, 3.066535, -1.730606, -0.888373, 2.951096, 3.153862, 0.237999, 0.558091, -2.061528, 0.783972, -0.139891, 0.005644, -0.081209, -1.382260, 1.061232, 0.716218, -2.590728, -0.990372, 1.266114, -0.707920, -3.964265, 0.948937, -4.407448, -0.231940, 0.264556, -0.267132, -3.774856, 2.497759, -0.914523, 0.970749, 3.113668, 1.660681, -0.062968, 0.503072, 1.886241, 1.214914, -0.328673, -1.813600, 0.017637, -0.110344, 3.279947, 1.179452, -2.311359, 2.184004, -1.036836, 2.163837, 0.106519, 0.639714, 2.886490, -2.178060, -0.599363, -1.816274, -0.672925, -1.578791, -0.505086, 4.701667, -0.958578, 5.368207, -0.715092, 1.727489, 2.659025, -1.855298, -2.369414, -4.724561, -0.572714, 1.991303, 2.016687, 1.089363, 3.728837, -1.101324, 2.744128, 2.082385, -0.082763, 0.874934, -0.554436, 3.319520, 0.412499, -0.356465, -0.818570, 2.214372, -0.868555, -5.018708, 1.333610, -1.507039, 1.224960, 9.023772, -0.491060, 1.643431, -1.231728, 3.141759, 0.249819, 0.606284, 1.929034, 0.079260, 0.980546, -2.081781, 0.985507, 3.136811, 1.951684, 1.112473, 3.010893, 3.541259, 3.218399, 1.626346, 0.683307, 1.771878, -1.257802, -0.757362, 1.514842, -0.636001, -0.430890, 1.693416, -1.821938, -0.486594, 2.916702, -4.420597, -0.468898, 1.278626, 1.723206, 2.731259, 2.001235, -0.968886, -0.304650, -1.206299, -2.135083, -0.916858, -0.571528, 0.047041, 3.504944, -2.133256, 0.494332, -0.482203, -3.457354, -1.942995, -0.520856, -0.325685, 2.403196, 1.200350, 3.393925, -2.494207, -1.327450, 2.736926, 1.736917, -2.051530, -0.506557, 0.020822, 2.461713, 2.484358, -1.360792, 0.210698, 2.571049, -2.315030, 4.404229, -2.766439, -0.356564, 0.609894, -0.005560, 1.713071, -0.469658, 1.197658, 0.040987, -2.285223, 1.484401, -1.409689, 4.578481, 2.506946, 0.425305, 2.446197, 3.263529, -3.631989, -0.371354, -0.413407, 4.208085, -0.127080, -1.246227, 1.026809, -0.691294, -2.296336, 0.541863, -0.415415, -2.213721, -2.229713, 5.652533, 0.325638, 0.895194, 0.288386, 2.395745, 1.893746, 1.656826, -1.891253, -0.981483, 0.977041, -1.021206, 1.278371, -0.601382, -0.586926, 1.910359, -2.599929, 2.117822, -0.329484, 2.511219, -2.765614, 1.057920, 4.557327, -1.286663, -2.828605, 1.715522, 0.892571, 1.328999, -1.452793, 3.062454, -0.846668, 2.685005, 0.747479, 2.616977, -0.771560, -1.170870, 0.398534, 0.343821, -0.425957, -1.216428, 1.275198, -0.774221, -0.835212, -0.086740, 2.298670, -0.739717, 0.506853, 0.935785, 3.649465, -2.203318, 2.277899, -0.553957, 0.137781, 0.643336, -2.148784, 1.281616, 1.584459, 0.310634, 1.217097, -1.326976, -0.676382, 2.128299, -0.981715, 0.061699, 2.889244, -0.561955, -0.823019, -1.973488, -0.304435, 0.426746, 1.849452, 0.824656, -0.400681, 1.364501, 2.863911, -1.468426, -2.614557, 2.445089, -0.290599, -0.084631, -0.135070, 1.321539, -1.531789, -0.796952, 1.839522, 1.379722, 2.139342, -0.658808, -1.957961, 1.888349, -0.784370, -1.480167, 0.501366, -2.519140, -1.949068, -2.253042, 0.033506, 0.485347, 1.676559, -1.090220, -0.454366, 2.439051, 4.116050, 0.226810, -1.543618, 0.369009, -1.356788, 1.256831, 2.512082, -5.875708, -0.840281, 1.064519, -1.075835, 0.661105, 0.859369, -2.564416, 0.931348, -0.657630, 0.169598, -0.473798, -1.840275, 1.967578, -1.207092, -2.994502, 0.979585, -2.226941, -0.148251, -2.205890, 0.192183, 0.761379, 0.832209, -0.757570, -0.044144, -5.811568, 3.817863, -0.739124, 2.566487, 1.508290, 1.364889, 1.391061, 1.753602, 2.484661, 0.561939, -2.577687, 1.308320, -0.658141, -0.613247, -1.009616, 3.694613, 3.931254, 1.646078, -2.630128, -4.893660, -2.893271, -0.949201, -0.252491, 8.096486, -2.053660, -2.715685, -0.488898, -0.369804, 1.844169, -0.557177, -3.507263, 0.625425, -0.046343, 1.116201, -2.843462, 0.430182, -0.984149, 0.188606, -4.804567, -3.662477, -12.423755, 0.505854, -2.556453, -0.261161, -2.620847, -1.616541, 0.459177, 0.800119, 2.372884, 1.032130, -0.833381, 1.391556, -0.065205, -0.212230, -1.258538, -1.861240, -0.342623, 2.821825, -3.306765, 1.124912, -2.588235, -1.198842, -2.072963, 1.051923, 2.765494, 1.943674, -0.649781, -0.090553, 0.010493, -1.542902, 7.146881, 2.114514, 2.671738, -2.078392, 2.603645, -4.148600, -2.695807, 3.128589, -1.877043, -0.506973, 3.315228, 3.669570, -1.232590, -0.327188, -1.176315, 2.137116, -0.240232, -1.170810, -3.713598, -0.264318, -0.798505, 2.017999, 0.936734, 2.098501, 1.205739, 2.284950, 0.970075, 3.022959, -0.798790, 0.537358, 3.912182, 1.456274, 0.925978, 0.233295, 0.655816, -0.774384, 0.546590, -7.141086, 2.124636, -1.520990, 0.233711, 2.825353, 0.444932, 2.467602, -0.684698, -2.486389, -1.738088, -3.114126, 2.726388, -1.019151, 4.807260, -0.181204, -1.084334, 1.163428, -0.109641, 0.964845, 0.559943, 0.336826, 0.089202, -0.693184, -0.897094, -0.925852, -0.158974, 2.087272, -0.549617, 1.179534, -0.658688, -0.299359, -2.097598, 0.542013, 1.209597, 0.868445, 3.104147, -2.820510, -1.204089, 3.737103, 0.770909, 3.525891, -1.135699, -1.655476, -2.240339, 2.525290, 1.211621, -0.105237, 0.486678, 0.251381, -2.101022, -4.316065, 1.071823, 0.767139, 0.222749, -1.501596, 2.875869, 2.010502, 2.222627, -2.201579, 1.083319, -1.388173, 1.759563, -2.029091, -1.098797, -2.860517, -0.240031, 0.644087, -1.932882, 1.401318, 2.215424, 0.366056, 0.841219, 0.509652, 1.678419, 0.370270, -1.852237, -0.313416, -2.199522, -0.294784, -2.797149, -1.652146, -0.632833, -4.488834, -0.143081, -6.097857, -5.642374, 1.487491, -1.104113, 0.571834, 0.434285, 2.254967, 1.653841, 2.349760, 2.489481, -2.731234, -0.345902, -3.468615, 0.002949, 1.483104, 1.326606, 1.378591, 0.229062, -0.190730, -2.039232, 0.562338, 0.645633, -2.672769, 3.342596, 0.399291, 0.816063, 1.965873, 1.918716, -1.087965, 0.048307, -0.756520, -2.151526, -1.697711, 0.347615, -3.180706, -0.146576, -2.279339, 3.271249, -4.460975, 2.981601, -2.770810, -0.264376, -0.206536, 0.586737, -1.217057, 1.638721, -0.420798, 1.459608, 2.833084, 0.065077, -3.276213, -4.026783, 1.695028, -0.414748, -1.509182, -2.335618, 0.645046, -4.259199, -0.605817, -2.499427, 0.038157, -0.863183, 2.061970, 0.611956, -2.115386, -3.359080, -1.596068, 0.307425, 0.191455, 0.361744, 0.899717, -1.701859, -4.992336, 1.584893, -2.895921, 0.149289, -1.404445, 0.501034, 2.682879, -0.675067, -0.786691, 1.069297, -1.640006, 0.099055, 0.312075, 1.645067, -0.090822, -2.817954, 0.371475, 3.697231, -0.769372, 1.777448, -2.683017, -0.970097, 3.940720, 2.429331, -0.532402, -0.217043, -1.411375, 2.891897, 0.184554, -1.693691, 0.541098, 2.303737, 0.010059, -2.598566, -2.101953, 1.095212, -0.516849, 1.995777, -3.868278, -3.872117, -2.956234, -2.438829, 1.945905, -2.324775, 1.419934, 4.604033, -0.205442, -1.559651, -1.537539, -0.980349, 2.164563, -1.639275, 2.282555, 0.196895, 1.739233, 6.665299, 11.612860, 2.788843, -1.838532, 0.987776, 0.625764, -1.816129, 1.568187, 2.474130, 0.996020, 3.790505, -0.408657, -0.948997, -3.530251, -1.080017, -3.417130, -2.058712, -1.956407, -2.989211, 1.230417, -1.237799, 1.329787, -1.045655, 0.436384, -0.770900, -0.847061, 0.136578, 0.058188, 3.483011, 2.304829, -0.042124, -0.562030, -0.265483, 3.568855, 2.773987, 0.164709, -6.854160, 2.262440, -1.915387, -0.935466, -2.306059, 1.185262, -2.824844, -1.654353, 3.939613, 3.268545, -0.325672, 1.448018, -0.088753, 1.695897, 0.997020, -0.638259, 5.060489, -1.404778, -0.066517, -1.533605, 1.931493, 1.549286, 1.261844, -0.025092, -5.829014, -1.885451, 0.096401, -0.849005, 1.323331, 1.640723, 2.387309, 0.977560, 1.003233, -2.851422, 2.125631, -0.622645, -0.251111, 0.108200, -0.860243, 2.165220, -1.747716, -1.526603, -0.160664, -2.234724, -3.378771, -1.829369, -0.828802, 1.092334, -2.838180, 2.241061, -0.070938, -0.125325, -0.533843, -0.900653, -3.907611, 0.291193, 1.380849, 3.148268, 2.248498, 2.641789, 1.162023, -1.307409, -7.452969, -5.762230, 0.570644, 2.225794, 2.268607, 2.220694, -1.868391, 1.590621, -1.087466, -1.318701, 0.355766, -2.192402, 5.148515, -1.260565, -0.340788, -3.066326, -4.802943, -1.822399, 2.027146, 2.033635, 2.484090, 0.864079, -2.444166, -2.060366, 0.691616, -0.893325, 1.614154, 2.483134, -7.538818, -2.377872, -0.144321, -0.343106, -0.466051, -1.690653, -3.003664, -0.220598, -1.996093, -1.322911, -1.798978, 0.656198, 1.923437, -0.542048, -0.335357, -0.584021, 0.888298, -1.122800, -1.792932, 0.603619, -1.189901, 0.216393, -2.956912, 0.420574, -1.393399, -1.954148, -1.921767, 4.165194, -1.175791, 2.798207, -10.199874, -0.837972, 1.866549, 4.939780, -1.918338, -5.765625, 1.894684, 2.931093, -2.691194, -1.582052, -0.238202, -2.357203, 3.227861, 2.676049, -1.514403, 0.610364, -2.333433, 1.532472, 3.116742, 3.468613, -0.903491, -3.069241, 2.971029, 1.897196, -2.358795, 1.614014, -3.038753, 0.296462, -3.413616, 1.476674, -2.205294, -0.426963, -1.149141, 0.687046, 1.459785, -1.579808, -0.128799, 1.095678, -4.413157, -0.583210, -0.037564, -3.217687, -3.874143, 1.367258, -0.658092, -4.242138, -1.566653, 1.616565, -0.231850, -0.561782, 0.175397, 0.873898, 2.102082, -0.481464, -1.669100, -0.535181, 1.875431, 4.580070, -2.197358, -0.910670, -0.982496, -2.303578, -1.195334, -0.761116, 1.036271, 2.430554, -1.472700, 0.805644, -0.503828, -0.386881, -0.879527, 0.931079, -0.833261, -0.988179, -1.804157, 1.147231, 2.970528, 0.660973, 1.180153, 1.487581, -2.030739, 1.501949, -0.520467, 1.889769, 1.071903, 0.089176, 0.463579, -1.126580, -1.210711, 0.082390, -1.625102, 0.874795, -1.195674, -3.522628, -1.638729, 3.287249, -1.166453, -0.828902, -1.406641, -4.649971, -0.544490, -1.771114, -0.274173, -1.582270, -1.786603, 3.387163, -0.859107, 3.550398, 0.630316, -1.103999, -2.080497, -2.685435, 4.383327, -0.171877, -0.829238, 1.843727, 0.737404, 1.495080, -0.509761, -0.942684, 0.598197, 0.137417, -2.112840, -3.016319, 2.016804, -0.613345, 2.634976, 3.474667, 2.299239, 3.921109, -0.259313, 2.293547, -0.668590, 2.097881, 0.135472, 1.272800, 1.023938, 3.750861, 3.546648, -5.987249, 5.189409, -2.012692, -2.080787, 3.648663, -1.718726, -3.059613, 0.914977, -0.070629, 2.184004, -0.554282, -1.572647, -4.378010, -1.895564, 2.295761, -1.807297, 0.939028, 3.319588, 0.981454, 1.846804, -0.074813, -0.959925, 1.180125, 0.214293, -1.340116, 2.026832, -0.275083, -2.391880, -2.757796, -2.484968, 0.950552, -1.653520, -1.723869, 1.662331, 0.535817, 0.273211, -1.151067, 2.993492, -0.075704, 0.320338, -1.407653, -1.041616, -1.881412, -2.501787, 1.301393, -1.517647, -2.464800, 0.252449, -1.997724, -3.026290, 0.212039, 1.502458, -4.246790, 1.029287, 2.246881, 0.741828, 0.158541, -1.028801, 2.529005, -0.893468, 1.443282, -0.346494, 2.669097, -2.864852, 3.209790, 3.041502, 0.020234, 0.115243, -2.031523, -0.282542, 0.396332, -1.363705, 3.021288, -0.318689, -1.363607, 0.955074, 0.509633, 2.074913, -1.201262, -2.390836, -0.396984, -1.613606, 0.360301, 1.139240, 0.839843, -0.285194, -0.322745, 0.437023, 1.281257, 3.713523, -2.683018, -2.837252, 2.113353, 0.791203, 2.814537, -2.278843, 0.702416, -1.589167, 1.516128, 0.348960, 1.021993, -2.720528, 2.671391, -1.391619, 2.161263, 1.234758, 3.476467, 2.706942, -2.214298, 0.062933, 3.817600, -1.251961, 3.566387, 1.513068, 0.747667, 1.597302, 3.858117, -1.373156, 0.729746, 1.749554, 2.064092, 0.612888, -1.871164, -0.993573, 1.980641, -0.394882, -0.771789, -2.300255, -1.864526, -1.332325, 0.679321, -1.628580, 1.684532, 0.794181, -0.341031, 0.865944, 1.570801, 3.436136, 1.568980, 0.631498, -0.048965, 0.418611, 0.775022, 4.532741, 0.286989, 1.137858, 0.944515, -2.167217, 3.400816, -2.120444, 1.896021, 0.739563, 1.800057, 0.976187, 2.241848, 1.109004, -0.139124, 2.273750, 2.243190, -0.000260, 7.102096, -0.981638, 0.517041, -0.106855, 1.611364, -0.680890, -1.851746, -0.071771, -0.558747, -0.778105, 1.001288, -2.798460, 4.107984, -2.121557, 2.265193, -0.130054, 1.358769, 0.994828, -2.313165, 0.441207, -0.300783, -0.672292, -1.479555, 3.422815, 0.001658, 0.206383, 3.627066, -1.661040, -1.218103, -1.546490, 3.201838, -0.425097, -1.208788, 1.069553, 1.511283, -2.078781, -0.688469, -2.191390, -2.106024, 1.625274, 2.402612, -0.825243, -4.379784, -1.812769, -0.058542, 1.346276, 0.333447, 2.418711, 1.475576, -2.620394, -0.280505, 1.131126, 0.607015, -1.094618, -2.319695, 1.346097, 2.262267, -1.525486, 1.444179, 0.501193, -1.753824, 0.733545, -2.394020, -1.725506, 0.408842, 1.683417, -3.182556, -0.488479, -1.059273, 2.058362, 3.142289, -11.317917, -2.676118, 2.479192, 2.741839, -1.815452, -1.900564, 3.863253, 0.957858, 1.753211, -2.094190, 1.561213, 1.603490, 1.401700, -1.423863, -1.557342, -1.070505, -1.258466, -2.814864, -0.233111, 1.705125, 1.368987, 0.674738, 1.317454, 1.040991, 0.379262, -3.057135, 1.303494, -0.719236, -0.150981, -4.246114, 0.012882, 2.754486, 0.887157, -2.154360, 0.434403, 2.080772, 12.592675, -1.400895, 3.682422, 18.500687, -1.824731, -1.229960, -1.306891, 4.426475, 4.153411, 0.296232, 0.334915, -0.198403, 0.207817, 1.373839, -1.809203, 3.648643, -3.357051, -0.607143, 0.804729, -2.215043, 1.579057, -1.446862, 2.116117, 0.022489, -0.106560, 3.259476, 2.179984, -0.827409, 0.466238, -0.174084, 0.749496, -2.032082, 0.819768, -0.012092, -1.404604, 2.040248, -4.373999, 2.905264, 0.524823, 3.290520, 1.260755, -1.269325, -0.546155, 0.301613, -3.231517, -2.633416, -0.817336, -2.118700, -0.650470, -2.887156, 1.295805, 1.893788, -0.623003, -0.415323, 3.246930, 0.012536, -2.414260, -1.570900, -0.459119, 2.363632, -1.078479, 0.124088, 3.440009, 1.118437, -1.124332, 4.910594, -1.549336, -1.430183, 1.136268, -2.112774, -1.333619, 2.121091, 2.231785, 1.296702, -0.906199, 0.965406, 3.869365, -1.652183, 0.759212, -2.980289, 0.682221, -0.984810, 0.634504, -1.211771, -2.491594, -0.945581, -2.850454, -0.138457, -0.431604, -0.632796, 1.385889, -0.483601, -1.077885, -1.354634, 1.187483, -0.412081, -0.877794, -0.520237, 0.527379, -2.126064, -1.391075, -0.027463, -3.005242, 0.156661, 2.754908, -1.378801, 0.082020, 0.123864, -0.290277, -3.230394, 1.301592, 2.213902, 3.094155, -1.755002, -2.322603, 0.064931, -1.328647, 0.874909, 1.640637, -0.022136, -1.437375, -0.168740, -1.580431, -1.466243, 0.074297, 1.542410, 0.640596, -2.803895, 0.838751, 1.192846, -2.151078, -1.352274, 4.536688, 0.652296, -1.681036, -1.748077, 1.116713, 1.505466, 1.093051, -0.850750, -2.080233, -1.093351, 0.568372, 1.615424, -2.187084, -1.067516, -2.221610, 0.670807, -0.322722, -0.557603, -2.128840, 2.599048, -0.295556, -2.070664, 0.996682, 2.272573, 0.704867, -0.482912, 0.789556, 2.423104, 1.283592, -1.183174, -1.525468, 0.091626, 1.759403, -0.894384, 1.708647, 17.310020, -3.481026, -5.126164, -0.462365, 3.699655, -4.189553, 0.381989, 1.898774, -0.363996, 0.458372, 1.472605, 2.367271, 0.744145, 1.278426, -1.720926, 0.309616, -0.775783, -1.905312, -0.546735, -1.651521, -1.946627, 0.678068, -0.779717, -0.981420, 2.827802, -2.157915, 0.100543, 2.626758, -0.105437, -3.918166, 0.863819, -3.190261, -1.860274, -3.934666, 6.241655, 0.965956, -2.913852, 0.003365, 1.201703, -2.250198, 0.700441, -0.116006, -2.632267, 0.289158, -1.789185, 0.158896, -2.259541, -2.592633, -1.587397, -3.143592, 0.232405, -0.945033, -2.211038, 2.132655, -12.288932, 3.241026, -1.518620, 1.077391, 1.059972, -1.065658, -0.746511, -1.032448, -0.192777, 0.732772, -1.637340, -0.360361, -1.155054, -0.207818, 1.369705, 2.599172, 2.482543, -1.180554, -0.888696, 2.530247, -0.034697, 0.959176, 1.193800, 0.326233, -1.076948, 3.587785, -0.672961, -4.438885, -0.397180, -3.531070, -6.830092, -0.541381, 0.651230, 0.792656, 1.801674, 0.653098, 0.020145, -0.930110, 1.829328, 0.417755, -1.133758, -0.575342, -0.281619, -2.819133, 0.293893, 1.141983, -1.487329, -0.180050, -0.480632, 0.239821, 1.104678, -0.611782, -2.685412, 2.376577, 1.076170, 1.586790, -0.723516, 0.346962, 2.835624, -0.571163, -0.018835, -1.004567, 1.764784, -1.652679, 0.181482, 1.126542, -1.868064, 0.287467, -2.103253, 2.532168, 4.365986, 0.254799, -0.283266, -2.060601, -1.498800, -2.974444, -1.227938, 2.405658, 0.629228, 2.409974, 0.530177, 1.364315, 4.653522, -0.811898, -1.583435, -1.069651, -2.269577, -1.162140, 3.601405, -0.490714, -1.997232, -3.724590, 3.772781, -2.163326, -0.496667, -4.260191, -1.616646, -1.990680, 3.304270, -2.352931, -1.146297, -6.734396, 1.459528, -0.291793, -0.521500, 13.051759, -2.432004, -2.658777, -1.149259, 1.436345, -0.751880, -2.189231, 0.615586, -2.498409, -3.291502, -2.095250, 1.040146, 0.714031, 0.970416, -0.616222, -2.000639, 1.994921, 2.395315, -0.561833, 0.121436, -1.408910, -2.248885, -1.112001, 1.088048, -0.414269, -0.435527, 3.829697, 0.528129, 2.375221, -0.519734, -0.971582, 1.406152, 2.238408, 2.720003, -0.267679, -0.791799, -0.090492, 1.500209, -1.946325, -1.748135, 6.836010, -1.218125, 0.738863, -1.162376, 1.206608, -0.377882, 1.355038, 0.775188, -0.811568, 1.407529, -0.604065, 4.010166, -1.165298, 0.672806, -5.079141, 0.690641, 0.379462, -1.214549, -1.517621, 1.051511, 4.106417, 0.001753, 1.941271, -0.016778, -0.369193, -0.438197, -3.510782, -0.501950, 3.121536, 0.598490, 3.841994, 0.205602, -1.246792, -1.616723, -1.143543, -2.465981, -2.489971, 2.632675, -1.313524, 0.345787, 0.992921, 0.425921, -2.188962, 0.680292, 0.194884, 1.859877, -0.628118, 0.726368, -2.322854, -1.007814, 0.255692, 0.737325, 2.278359, 0.366714, -1.926799, -1.290199, -0.082554, 0.797631, 3.216966, 0.944166, -3.854357, 1.882313, -0.757823, -2.803020, -1.485754, 2.979990, 0.822588, -0.639839, -0.333204, 1.326457, 0.428489, 2.403607, -0.028208, 1.258138, -1.414546, 2.595982, -1.312079, 1.871267, 3.542688, -1.192679, -0.747623, 0.372664, 0.500303, -2.334963, -5.068396, -1.425156, -2.123881, -1.776343, -0.095469, 1.043374, 1.254840, -0.709997, -0.800484, 0.942643, -1.944042, -0.205155, 0.038740, 2.047468, -1.089243, 2.242003, -1.200643, 5.502473, -0.559836, 0.693430, -0.248119, 3.774752, -1.164453, 0.814244, 2.204838, 3.310038, -1.259518, -1.539988, 0.631875, 1.811592, 1.660107, 1.233070, 1.175259, -0.912320, -3.419650, 3.499059, -2.343933, 2.245733, 0.961249, -1.360731, -2.247202, -1.547391, 0.428645, 3.232787, 0.094637, -0.361900, 0.301173, -0.484404, 0.424156, -1.495115, 2.174474, -4.887064, -3.342853, 2.512707, -5.732465, 2.330478, 0.607593, 2.459414, 2.199535, -0.540271, -0.431682, -0.428440, -0.331239, 7.473131, -0.676626, -1.463685, 0.149552, 0.061583, 1.886732, 1.978416, -1.099899, -2.279263, 1.284642, -2.398491, 1.477496, -0.910583, 1.879131, -0.457095, -0.015456, 0.256275, -0.456831, -3.490171, -0.004046, -0.551340, -0.684976, 0.764194, 0.847659, -3.879746, -3.626539, -1.568061, 0.991960, -0.737746, 2.614909, 1.940545, -3.691171, -3.245214, -2.179925, 2.383390, -1.783122, -0.506950, 1.127885, -0.890014, 1.435303, 3.048434, -1.386346, -1.076663, -2.896395, -1.343953, -2.565495, -1.381629, -1.268511, 1.228591, 0.993910, -2.575418, 0.648286, 1.532965, -0.935785, -1.605901, -1.246196, -2.201324, -0.857666, 3.537265, -3.682135, 0.672675, 0.951825, -0.827279, -3.059412, 4.575532, -3.718941, -0.977951, -1.166684, -1.237993, 3.090737, 1.565580, 1.461544, -1.929372, 1.394221, -3.346243, -0.226639, 1.119265, -1.345803, -0.899012, -2.456377, 1.593067, 0.910089, 1.503470, -1.812514, -1.400435, 1.177375, 0.096689, -0.294497, 0.204516, 0.525131, 1.091468, 0.787260, 0.021002, -1.185784, -0.505473, -0.187676, -1.750899, -3.509349, 2.650220, 1.842912, -1.387142, -2.165509, -0.041363, 0.090037, 3.887540, -2.342398, 0.609889, -0.786447, 1.820952, 1.236354, -1.621533, 0.662158, 2.572353, 0.277775, -0.603314, -1.811826, -0.732945, 1.049378, -2.206546, 1.959944, -1.303532, 0.861363, -0.612073, 0.364254, 1.629579, 0.628865, -0.018568, -3.558367, -0.744735, -1.004377, -0.180408, 0.018036, 0.942299, 2.755231, -3.410764, -1.202904, 1.628659, 0.269436, 1.631451, 2.589946, -0.040498, 1.630141, 0.496223, 1.869666, -0.404186, 1.478820, -0.379239, -3.533465, 2.040182, -0.441357, 0.470875, -0.983391, 1.173682, 1.085656, -0.516549, -0.635032, -0.766274, -2.671732, -1.281379, 1.456411, -0.915409, -3.183290, -0.204772, 2.401025, 2.152182, -0.233675, 0.305950, 2.888514, 1.930429, -1.184927, 1.662104, -0.271050, 1.988250, -0.984551, 1.636173, -0.936373, -0.207487, -3.862692, -0.202516, 1.630704, -0.943317, 0.691296, -1.044768, 0.531007, 0.471725, 0.821349, -1.234496, -1.380861, -4.413808, 0.437657, -2.809109, 0.977942, 0.700759, -0.938202, 2.930912, 0.906982, 0.537006, -0.933399, -0.012241, -0.700445, 1.411654, 1.602838, 0.170198, 1.034546, -0.294352, -0.277783, -3.215640, -0.310807, 1.199179, 0.741775, -2.316380, -0.154027, -2.767129, -2.038565, 1.119279, -2.533041, 0.076981, -0.187397, 1.210819, 1.391541, -2.467751, -0.143722, -0.905126, 2.260527, -0.993221, -1.232143, 1.834083, -0.804748, -1.330043, -0.310126, -2.712278, -2.764476, -1.708571, 1.639717, 9.467136, 1.056757, 2.668218, 0.011055, 0.992315, -2.199364, 2.513154, 2.435208, -2.443968, 0.233445, -2.318432, 0.602261, 0.931487, -2.197106, 2.200380, -1.881728, 3.087734, -0.060384, 1.490917, -3.820469, -2.404525, 0.082236, -1.416263, 6.533288, -1.271866, 0.874339, -0.064727, -0.294185, 0.155135, -0.258834, -3.481010, 2.522374, 1.355910, -1.698231, -2.581586, 1.589780, -1.474311, 3.495509, 0.679703, -4.357119, -0.388887, 2.020856, -1.231388, 1.933426, -3.399568, -2.166835, 3.597775, -2.225888, -2.851705, -4.492297, 2.081487, 1.752849, 0.931443, 1.775176, -1.427293, 3.582803, 0.461577, 0.847037, 0.082594, -1.585400, -1.084166, -1.576557, -0.957542, -0.661302, 0.782861, -0.174181, 0.403607, -0.843751, 0.629765, -0.536972, -0.854492, 3.225008, -3.574810, -2.818572, 4.156142, 1.869532, 1.431887, -0.001005, 0.652833, -0.933383, 0.058137, -1.479301, -2.311369, -3.555691, 0.354053, 1.014045, -2.245627, -3.368932, 0.047818, -3.167487, -1.033651, 0.841527, -1.111248, 3.357744, 1.951621, 0.393261, -3.040123, -3.178878, 1.615363, -0.041975, -2.667777, 0.146986, -1.417583, -1.627755, -0.685473, -3.537882, 0.029217, -0.811283, -0.627015, -0.022089, -1.536894, -1.870402, 1.232272, -0.325790, 1.623947, 0.759523, 1.878842, 2.110918, -0.386001, 0.917781, 0.609635, -0.115149, 3.891633, -0.426539, -0.111032, -0.530549, 1.704758, -2.846103, 1.041818, 0.211156, -1.941607, 0.620646, -2.002606, 2.378968, 1.088245, 4.909698, 2.291356, -1.610609, -2.184232, -1.090413, -0.585811, 1.194546, -1.638153, -0.257659, -0.407240, 0.275248, 1.816893, -0.549051, -4.227658, -0.838440, 0.708863, 3.270518, -3.819626, 1.362298, 3.398038, 0.340970, -0.061836, -0.640408, 2.750417, 0.624174, 1.774026, -2.950428, -0.631979, -3.549893, 0.611664, -3.074150, 2.049044, -0.319013, 0.005977, -0.195755, -0.366373, -1.888581, -3.412463, 2.255754, 1.286729, -0.145309, 1.804818, 2.039304, -1.039636, 1.837572, 3.214562, -3.287841, -0.662308, 3.181302, -3.620912, -0.062886, -0.094632, 0.150107, 0.930212, -0.841085, -2.030969, 1.515888, 0.988359, 1.924312, -1.151969, 0.427673, -0.768772, -3.342158, 1.639557, 0.227930, -0.515050, 1.370292, 0.266383, 1.212152, -0.133667, 0.940615, -1.409638, 1.942192, 2.937795, -0.097905, -1.931062, 0.930637, 1.869293, 1.406385, -1.296549, 0.331738, -2.143225, 3.932092, 2.715863, 2.412196, 0.995437, 3.460501, -1.302808, 2.907689, 1.655309, 2.478021, -0.200031, 3.047060, -0.410674, -2.622407, 0.302213, -2.410561, -0.752223, -2.774459, 0.095784, -0.415811, -1.300381, -0.542583, -2.131463, 2.296709, 1.587769, -0.620450, -1.230258, -1.553957, 1.986467, 2.022719, 0.715672, 2.198387, -1.387220, -1.690441, -4.472634, 0.718367, -1.587503, 1.961340, 2.033993, -2.388419, -0.369963, 0.497210, -3.200239, -1.881070, -0.903583, -2.291441, -1.807256, 0.693820, -1.195831, -0.751228, -1.074330, 0.841028, 1.857826, 3.043664, -0.462683, -1.359275, -3.413938, -1.533384, 0.597736, -1.686284, -4.281181, 2.230808, -1.951093, 0.725956, 2.111386, -1.241247, -0.219312, -0.911483, 1.301086, 2.288175, 2.471712, 1.183948, 1.253952, -1.300000, -3.339571, 1.425936, -3.129603, 0.875244, -1.445471, 1.345114, -3.602144, -0.701855, 1.367753, 0.410516, -3.014481, 0.615465, -0.351461, -1.487927, -2.222125, -3.888845, 2.094670, -1.978802, -0.517767, 0.058138, -0.444669, -2.294036, 1.998730, 0.297695, 0.018321, 0.268706, -0.879354, 0.274088, -0.445150, 2.828982, -0.520435, -1.824163, -3.204499, 0.636438, 1.238371, -0.863154, 4.537728, 2.181285, 2.711806, -0.882168, -2.096454, 1.998263, 3.602588, 2.114964, -1.150518, -0.076333, 1.211806, -2.149293, -0.410827, 0.744497, -1.594841, 2.430060, -3.093566, -2.016082, -2.308710, 2.795972, -2.289312, -0.833563, 0.530762, 2.017911, 1.745501, 3.915513, -3.470293, -6.470500, -0.332230, 0.011448, -0.394733, 2.631524, -1.241708, -0.602433, -2.349052, 1.143003, -1.018770, -0.317715, 1.176016, -0.424689, 0.303274, -2.129854, -0.518942, -1.190350, -3.030563, 2.986264, 0.322974, -1.171382, -2.080170, -1.518298, 1.755744, 2.481019, -1.135633, 1.196923, -2.293517, -0.416247, -0.419748, 2.791564, 2.139885, 3.192137, -0.693876, 0.322415, -3.658330, 1.089203, 1.927066, -3.449436, 0.115316, -1.598851, -2.275302, -2.185232, -1.625353, -0.050324, 0.472127, 1.869982, 2.535981, -4.294853, -2.277530, 1.433655, 1.217977, -0.048806, 2.771290, -1.207959, -2.310196, 2.591723, -2.335873, -3.282704, 1.380378, 0.943002, -3.358639, -0.853271, -2.042148, 0.580582, 0.059815, 2.079012, -2.376285, -0.196453, -3.680857, 0.270507, -1.599318, -0.332833, 1.543115, -0.472989, -0.988075, 1.807841, -1.925379, -0.623305, 6.014649, 0.391908, -1.238512, 4.130530, -3.661225, 4.244806, 1.479349, 0.690953, 2.257133, -0.744723, 6.228973, -1.201092, -0.397441, -1.781538, 6.628207, 4.155041, -0.528522, 0.690781, -1.653167, 2.763517, 1.429862, 0.232714, 2.562456, -1.089225, -1.969519, -2.903962, -1.258486, -1.445670, -0.693330, 2.552694, -0.960994, -2.646074, 1.737455, -0.539961, 2.642196, -0.004014, -1.921011, 0.863813, -0.337553, -0.190130, -3.074884, -0.279881, -2.351591, 0.643860, -1.237673, -2.046865, 0.629775, 0.710130, -11.075646, 0.213202, -0.315120, -1.107641, -3.423430, -0.593594, -2.706415, -1.796721, 1.453879, -2.855584, -4.217345, -0.273067, 0.940133, -1.611402, 1.074368, -0.007585, 0.811808, -0.326986, -1.716057, -1.484485, 0.644776, 1.441688, -1.257548, -0.016782, -8.116377, -1.092561, 2.480797, 2.979291, -0.185106, -1.670235, 2.845420, 0.410733, -1.724248, -1.519051, 1.600042, -3.040879, 0.611754, 2.122114, -1.111939, -1.519439, -0.995059, -1.118172, 1.793956, -0.745843, 1.341350, 1.434149, 2.423525, 1.703777, 0.244239, 4.135608, 0.343924, 0.921309, 1.121611, 0.995589, 4.442313, -0.761273, 3.066107, -2.289390, 1.500767, -3.724264, 1.227392, -1.030693, 0.063110, 2.629669, 1.274395, 0.414828, -1.593909, -1.180849, 1.649198, 0.442606, 0.564936, 1.496706, 1.280906, -0.746933, 1.430545, -1.602895, -1.896697, 0.114705, -3.046932, -2.504452, -0.288728, -0.457942, 2.345242, -0.296523, 2.460886, 1.491791, 1.460493, 2.949695, 0.646835, 1.938821, 1.221826, -3.029753, 1.027855, -1.815510, -0.346786, 0.590048, -2.604692, 2.580613, -5.464456, -2.245464, 2.986708, -1.448367, -2.161349, 3.038980, -0.680506, -0.707855, -0.048277, 2.409130, -3.707219, -1.432573, -1.219632, -0.352755, -3.242210, -2.011038, -1.698111, -1.534498, -1.411064, -1.235086, -3.260119, 0.751513, 2.037949, 6.087719, -0.354985, 2.296287, -0.823595, -2.583332, 2.302197, 2.243156, 0.054603, -0.797066, 3.940995, -0.038713, 1.843858, -3.207862, 0.474149, -1.900639, -0.749204, 1.216251, -1.687184, 0.714031, -0.671786, -4.782870, 0.280564, -4.122239, 0.000554, -1.740705, 0.414890, -0.243921, -0.324114, 0.220153, 0.294286, -0.331063, 1.128937, -0.042819, -2.156856, -2.752139, 1.753495, -4.386309, 2.646769, 1.862037, -0.717543, -1.894679, -0.222581, -1.100053, 2.620292, -2.685768, 3.412238, -3.131670, 2.225328, -3.709669, -1.838988, 1.399511, -0.762291, 1.787900, 2.572352, 3.131334, -1.663082, -1.124640, 1.764323, 3.182696, -2.977489, 1.514403, -0.245818, -2.238850, 4.151056, 0.036864, -0.186692, -2.001691, 1.246032, 1.557085, 0.019165, 0.828518, 3.140849, -0.298136, 3.447491, 2.868086, 2.243513, -1.050800, -3.414328, 3.056563, -3.797553, 0.326561, -2.106754, -1.037465, -0.276275, -2.951207, 2.119463, -3.765703, 2.635464, 1.421755, -1.351970, -3.440002, -1.215871, 2.068440, 0.206381, 2.305964, -1.791120, -1.052097, 0.080710, 1.619676, 2.618592, -1.497641, -4.078834, 0.682429, -0.353784, -0.943733, -1.674624, 0.674289, 1.971855, -1.487211, 0.966145, 3.574993, -1.054608, 0.972266, -1.674012, 0.498910, 2.235878, -2.991302, -0.194584, -0.462182, -2.855854, 2.037934, -1.383906, 0.133169, -1.456793, -1.011050, -1.552601, 0.229914, -0.716652, -0.209454, 3.012683, -4.222848, 0.078118, -2.066082, -3.165344, 0.016393, -2.448389, -0.584697, 2.741099, 2.008625, 0.237100, 3.490952, -2.384674, -0.600877, 4.660543, 0.360961, 0.290765, -0.756962, -2.478991, 2.993396, 0.240600, 2.667300, 1.342028, 0.278699, 2.355097, 0.520877, -0.399971, -0.270937, 0.079413, -0.333582, 0.991009, -0.277407, -0.802320, 0.877358, -0.158195, -2.252553, -0.421490, -1.147035, 1.896074, 0.303360, 0.284368, 0.645959, -1.861567, -0.189420, 2.494082, 1.622385, -0.301365, -3.662149, 4.510878, 1.651112, 0.298648, 2.755453, 0.300312, 0.577870, -2.157631, 0.022099, 4.050114, -1.113486, 1.166550, 1.433385, 0.636592, 0.365205, -1.710035, -5.819573, -0.750998, -0.700895, -0.252428, -2.303637, 0.122997, -2.406902, 0.796248, 0.074350, -0.191512, -1.336334, -0.727378, -0.217502, 0.768734, 0.026973, 1.666830, 0.837306, 2.636509, -4.413403, -3.378228, -1.265510, -0.660484, -0.527256, 0.652431, 0.810927, -0.955311, -2.194607, 3.390256, -0.676141, -2.678200, -2.246846, 4.665495, -0.300063, -1.505307, 0.372656, -0.314319, -1.683735, 1.244436, 7.202695, 0.347428, -3.931497, -0.384359, 0.070719, 1.927199, 2.370592, 1.543958, 1.292192, -1.171941, 0.515360, -1.998340, -2.632734, 1.160288, 0.932155, -2.313407, 1.186146, -1.297318, 0.411109, 2.077339, 0.377120, 3.878205, 1.169885, -0.514475, 1.380647, -0.220764, -6.186738, -0.987336, -1.161258, 0.022470, 2.944921, -3.944496, -2.454340, -1.350916, -3.333500, -3.976480, -0.813268, -0.936630, 4.470922, -1.197749, 0.999399, -0.848027, -0.556465, 0.876649, 4.624232, 1.716997, -1.316871, 2.161342, 2.106090, -0.884313, -0.900795, -1.254095, -0.401763, -2.136899, -1.426937, -1.633984, -1.857804, 4.080161, 2.620926, 3.743167, 1.912780, 1.157939, -0.232982, 1.485714, 3.776951, -0.728675, 0.096456, 2.419091, -2.536920, 1.749187, 3.919883, 1.045194, 0.124967, 5.034767, -0.020524, -5.442020, -0.340481, 0.847831, -0.570193, -3.708969, -1.694553, -1.775758, -1.774149, 1.016557, 2.744298, -0.981895, -3.203657, 2.381986, -0.130370, -0.575154, 0.981555, -2.114384, -0.703772, -1.544474, -4.912024, 1.436929, -3.612131, -0.513180, 1.124501, 1.500321, 1.228162, 2.164581, -1.008544, 3.386724, 5.138061, 2.143028, -0.163646, 3.132955, 0.232388, 0.140801, 2.374613, -1.336730, -3.779842, -0.695596, -1.955346, -5.124892, -0.852984, 1.089569, 1.047508, -0.899278, -2.671301, 0.775844, -0.359940, 2.370226, -1.953356, 1.204961, 0.575502, 4.395379, -1.004487, -2.043077, -0.088922, 2.446170, 1.487957, -0.670360, 0.311429, -0.970048, 4.440516, 0.804933, 1.320401, -1.760813, -1.227641, -0.585840, -1.419913, 2.561329, -1.393864, -0.514215, -1.485557, 2.117904, 2.916862, -0.557396, 3.088374, -0.601427, -2.839745, -0.335838, 1.241137, -0.483900, 0.794831, 1.195930, 0.649772, 4.032490, -0.526397, 0.963251, -0.133096, -0.534765, -0.111903, -1.648276, -1.044494, 0.361883, -0.918383, -0.419664, 0.178241, -0.791128, -1.730827, 0.185987, -2.588084, -0.286288, -1.370608, -4.314114, 1.166656, -0.317811, 0.544867, -1.790729, 1.485108, 3.057710, -0.402884, -3.525095, 1.046638, 0.469837, -10.401346, -0.187911, 0.602185, 0.784814, 0.686189, -0.628216, -2.192061, -0.726722, -0.583705, -1.816725, 2.630092, 2.962842, -0.098243, -1.397867, 4.522958, -2.306017, 0.158243, 0.021326, 0.669872, 1.986373, -0.030303, -0.977807, 4.633324, 0.498535, 3.087742, 3.187632, -1.225902, -1.390914, 0.624044, 1.865394, -3.805171, -0.420694, 2.681516, -4.424584, 0.019501, 1.461479, 0.945931, -0.460550, -0.178802, 1.539152, 1.010580, 1.188666, -3.036549, -0.074547, -0.933213, 1.277559, -0.826474, -1.801009, 1.385077, 0.442225, 3.276604, -0.478880, 1.084784, 0.558545, -2.902088, 1.052059, -1.853553, 0.754707, 1.477189, 0.535306, 1.659573, 0.157024, -2.818528, 4.611130, -0.069930, 1.186805, 3.071634, -1.309555, 2.216107, 1.327526, 5.102565, -1.661079, -0.484750, 2.699060, 2.793719, 6.261371, 0.641963, -0.288440, -0.679968, -2.719732, -1.884756, 4.892308, 1.928781, 0.405360, -0.971816, 2.584059, -0.217052, -0.968458, 1.336929, 3.495005, 1.708722, 0.110665, 0.281588, -0.007339, 0.161777, 2.841527, 2.780258, 0.515291, -3.523671, 0.947459, -0.148462, -1.500179, -3.493957, -2.649297, 1.730525, 1.296232, 0.803082, -1.144793, 2.993541, 0.136460, 2.948369, 4.233619, -0.316875, -1.924663, 2.759971, 1.683363, 2.597641, 0.497635, 1.238039, 0.183853, 0.467309, 2.465760, 2.984601, -0.765347, -1.204559, 1.072209, -3.060474, -0.113188, -1.049060, -1.903181, 2.465544, 2.311688, 0.107047, -0.454180, 1.233230, -0.411675, -1.004517, -1.172773, -1.903938, 3.785788, 2.654215, -0.098252, 2.272377, -2.599964, 1.309094, 4.073755, 5.101512, 0.368129, -2.872931, 0.665823, -0.812287, -1.531622, 2.246672, 1.337333, 0.032180, -1.559512, -1.714184, -0.499598, -0.752196, -0.304698, 0.928405, 2.025278, 1.975079, -1.431077, -0.394958, 0.576032, -1.239387, -2.800098, -3.549074, -2.340450, -3.898644, -2.728451, 3.891414, 0.831638, 2.061581, 1.196748, 1.816497, 0.156793, 1.248291, 1.444935, 2.782322, 0.900815, 3.566337, 1.390690, -4.955343, 0.791208, -1.397928, 1.828276, -0.509986, -1.943480, -4.032582, -3.864328, -1.491560, 1.075016, 4.650043, 0.521032, -0.454500, 0.902760, 3.573976, -1.945750, 1.468956, 0.404240, 0.927548, 2.688647, -1.748163, 2.081175, 2.447916, -0.936185, 0.596772, -2.287498, 0.668699, -0.389346, -4.987719, -1.310262, -0.750813, -0.004687, -1.002931, -2.260232, 2.612141, -0.452705, 3.334452, 4.471656, 0.040492, 0.043492, 1.906444, 0.762864, 0.013549, -1.428000, 0.704618, 4.283254, -0.447447, 0.242987, 1.894843, -1.150813, 1.425953, 0.571005, -1.205558, 2.250842, 1.010226, 0.975433, -0.190685, 2.520607, -0.260550, -1.565487, 2.237750, 5.694161, 1.458744, 1.499833, -0.964584, 1.126270, -4.017033, 3.835517, 3.099368, 2.741929, 3.745870, 1.924112, -2.769214, -8.153638, -1.366967, -0.222473, 3.104762, 0.408815, -3.334017, -2.308936, -2.294003, -2.051878, 0.342104, 1.393579, 0.115670, 0.384816, 2.063236, -0.901094, 0.488691, 0.358792, 0.578016, 3.229413, 0.266376, -0.348818, -4.958932, -0.798056, 0.753551, -4.482573, -2.943725, -4.519968, -3.022888, 1.796830, 0.265710, 1.506794, 0.902000, 1.777172, -1.468453, 0.287163, 4.323891, 0.919122, -0.175923, -1.098132, 0.103859, 0.678686, -1.698002, 0.703255, 0.417760, 0.247697, 2.633260, -4.531164, -1.101134, -2.511764, 0.845692, 0.613309, -1.745634, -3.350141, 1.451234, -0.067550, 2.045863, 0.066918, -0.112667, -1.870865, -1.044443, -2.118799, 1.548138, -3.268719, 3.594374, -2.292007, -0.731400, 0.638149, 1.171789, -0.541452, 2.279704, 0.189897, 2.355322, 0.354103, 0.437733, 2.829152, 1.429048, 0.933853, -0.411465, -0.497428, -1.422664, 1.682440, -0.538568, -0.112830, 1.735603, -0.933443, 2.339074, -3.122605, 1.465831, 0.569450, 1.968063, -1.178774, 1.301933, -1.622244, -4.631080, 2.460362, -1.732073, -3.284402, 2.536322, -3.206595, -0.230824, 2.381750, -0.767115, -7.129733, 0.176164, 0.331321, -0.749577, -0.663327, 0.478035, -1.901979, 1.249618, 5.354519, -3.093740, -0.885269, 0.886041, 1.691122, -1.000635, -1.291685, -0.424720, -0.564069, -4.108914, 0.154632, 0.835152, -1.706249, -0.724605, 1.825291, -2.242805, 0.980393, -0.696546, 0.399662, 2.416517, -2.199752, -0.551790, 2.116515, 0.207906, 1.171264, 0.569540, 0.002868, -1.336563, 0.920318, -1.586634, -1.307527, -1.973062, -4.813841, -0.904314, -0.464118, -1.834300, 0.694039, 2.206343, 0.832556, -2.252901, 1.668644, 0.868798, -3.234017, -0.940149, 2.856216, 23.049131, -1.368276, -1.221370, -0.546990, -0.740844, -0.585601, 3.795197, 0.829124, -1.421780, -2.747960, 3.710470, -1.213560, -1.911591, 5.013275, 0.673200, 2.373624, 5.593239, -0.657087, 0.307902, -0.746474, 1.093833, -2.048117, 1.662055, 7.558481, 5.067765, -3.247648, 0.402622, 1.452313, -2.936994, 1.637158, -0.491472, 2.239504, 1.444507, 2.413685, 0.213852, -2.163628, 3.432185, 6.881503, 1.099702, -0.305388, -0.558462, 1.285792, 0.311286, -0.607929, 1.466767, 0.773958, -0.885835, 0.165974, -3.087383, 3.033762, -0.593204, 2.427591, -1.382962, 0.657607, 1.489019, -0.537272, -0.025472, -2.250768, 2.848840, 4.054736, -0.610624, 2.775694, 0.896121, -1.135586, 3.143772, 0.262527, 2.628227, -0.285485, 1.432166, -0.431331, -2.357857, -3.904627, 1.781239, 0.233672, 0.943503, 3.086826, -11.265800, -1.675461, 1.690181, -1.013600, -1.180526, 3.663422, -0.594121, 2.752018, 0.648223, -1.827538, -0.465675, 2.111429, -1.462047, -2.965230, -0.563627, 0.869006, -0.651719, 1.894362, -3.315145, 3.054309, -0.267607, -0.312651, -0.644268, -0.030751, -0.662597, 1.378941, -2.936244, 1.098406, 0.021220, -0.233837, 0.533106, 0.207417, 0.242259, -1.860722, 2.138731, -0.163239, 0.123441, 3.316773, 3.244506, -0.519945, 0.607942, -1.150754, -2.454610, -1.068099, 1.211501, -0.863424, 1.330000, 0.356083, 2.411207, -0.378593, -0.009602, -9.444021, -1.246032, 1.298337, -5.255346, -3.966677, -0.270956, 0.393291, 0.164479, 0.425593, 2.018335, -0.624935, -0.044252, 2.483628, 0.823812, -1.851756, -1.228499, -1.269864, -1.755970, 0.393510, -6.143491, -0.245994, -0.033852, -0.529157, 0.950466, 2.724706, 1.496825, -1.469126, 1.589233, -3.745199, -0.860516, -0.340473, 0.733098, 0.764605, -3.821471, 0.853751, 0.148301, 2.213405, 4.123124, 1.283300, -0.087948, 1.402946], "qwen2.5-coder:latest": [-0.436007, -1.314809, 0.139666, 5.148615, 1.838311, -1.521601, -3.773632, 0.581152, -0.971788, 7.219580, -0.066787, 1.198955, 1.659349, 3.621571, 1.779006, -0.560876, 0.355461, -1.897615, 2.303185, 0.109255, 6.044055, 1.446214, 0.674157, -0.692821, -1.987091, -2.048842, -1.936660, -2.349168, -3.152107, -3.172126, 0.873809, -1.490791, 0.970821, 1.441998, 2.091025, -0.651286, -2.027265, -1.386296, -0.700867, 2.765764, -1.228565, -0.294553, 0.431368, 1.284794, 0.631266, -0.846027, 16.100187, -0.651704, -0.213284, -1.980950, 0.748919, -1.524519, -3.034139, 1.428813, -0.906473, -0.359399, 0.568063, -0.806468, -0.336932, -0.204364, -2.249646, -1.886204, -2.921830, -2.873020, -2.915986, -0.558776, -5.761330, 0.710662, -14.035521, -2.583575, -0.747539, 0.809441, -0.951276, 0.723559, 1.804405, 0.222171, -1.118399, -0.074350, -11.002135, -1.436381, -1.212464, 0.143454, -2.309623, -2.390476, -0.117248, 1.324861, -1.891298, -0.915926, -0.236022, 2.277486, -1.947304, -1.112398, -0.341477, -1.563656, -1.073132, 0.275794, 2.828944, 0.914620, 2.530140, -2.008723, 1.740619, 1.433238, -0.137113, 1.737560, -2.091690, -0.144877, -1.682601, -0.273133, 0.891697, 3.031501, -0.963722, 2.925961, -0.128959, 5.033491, -2.237801, -0.329063, -0.569064, 1.594402, 0.993208, -5.571981, 3.424447, -0.005736, -1.073074, -0.499183, -1.037315, -0.981740, -0.731634, -2.787812, -2.274329, 0.616070, 2.383073, 0.642813, -0.994558, 0.252382, -4.711788, -0.271676, 3.495092, 1.964713, 1.607625, 1.494871, 1.336313, -1.587251, -1.514070, -5.126953, 0.097945, 0.919678, 2.441731, -2.928982, -0.321582, 0.855205, 1.654471, -1.153838, 4.138180, -0.955069, 1.078292, -1.350932, -4.036607, 1.604253, -0.872791, -1.828661, 2.580978, -0.709960, 6.295347, 1.293245, -1.309786, -2.217097, 0.007649, -2.884886, 1.301318, -0.435597, -0.482649, 3.068917, -1.476866, 2.451068, -1.917819, -1.498407, 0.184997, 1.833188, 0.816717, -0.623813, -2.193322, -1.512282, -1.615667, -1.072993, -4.262381, 0.010529, 1.257217, -2.236494, -4.156431, -0.603306, 2.949536, -0.846157, -0.592448, 1.623096, 0.024929, -0.389215, -1.058699, 0.131432, -1.387448, -3.724334, -0.617063, -1.108744, 0.636820, -1.469211, 2.240824, 0.869600, 4.049538, 3.993536, -0.610457, -0.671499, -3.111218, -2.991330, 0.696155, 2.608358, 1.001497, 1.474154, -3.532830, 2.660499, -1.066343, 2.081527, -1.972203, -0.570178, 1.080361, 0.286836, 0.940870, 1.073439, 0.588628, -2.507718, 2.726036, 0.290909, 1.218498, -1.227438, 0.014571, -0.629023, -0.212335, 0.783704, 4.464511, -1.721464, 8.572509, -4.255687, -0.337520, 0.154504, -0.440107, 1.332823, 6.227478, 1.853769, 0.825418, -0.090258, -0.175270, -0.294410, 2.580056, 0.677362, 2.338051, -0.415923, -1.507566, -3.802147, 0.664998, 1.294685, 0.533289, -4.037803, 1.228877, 1.029611, -2.024710, -2.125656, -0.549981, 2.533221, -2.395383, 0.313111, 0.601733, -2.608969, 0.216491, 0.406232, 0.609647, 0.711135, 2.503493, -1.242730, 0.346222, 0.542468, -3.621418, 0.442082, 0.813194, 2.384302, 0.784717, 0.717723, -1.476249, -1.676991, 0.117224, 0.378211, -0.194512, -0.284983, -3.506894, -1.025167, -0.378918, -1.602937, 1.909185, 2.551661, 2.117800, -0.885118, 0.708412, -1.859785, -0.479130, -0.738324, -2.670425, 2.509346, -1.601109, -2.061625, -1.747532, 2.314368, -2.034420, -2.830469, 0.645515, -0.335051, -1.528542, 1.276098, 1.399137, -2.062893, 3.157044, -0.355756, 0.008239, 0.529171, -3.172691, -1.092223, 4.341345, -4.409624, -0.345799, -1.003428, -4.002488, 1.766916, 1.763788, 1.652610, 0.630642, 0.222056, -1.701876, 2.214486, 2.379781, 0.916326, 2.501186, -0.940371, 0.257316, 0.774674, 2.868283, 0.917253, 0.978641, 0.544919, 7.170478, 1.259203, -1.779769, -1.019169, 3.777537, 0.795997, 1.963864, -0.853626, -0.343095, -1.443662, 0.394236, 0.305709, -0.351705, 0.524120, -1.773943, 2.920990, -0.804305, -3.644200, 2.719176, 2.336826, 2.026261, 0.880051, 0.758393, 0.228448, -2.986163, 5.670512, -0.611930, -0.002579, -0.251571, 0.224900, -1.935357, -0.920759, 8.231796, 1.971320, 1.576411, 1.943412, -0.953862, -0.375397, 3.036218, -2.298655, 0.889425, 1.108079, 0.258971, 2.647190, -1.539036, 0.500190, -0.727908, 1.263729, -3.033226, 1.961906, 0.810480, -2.089427, 0.708266, -4.047872, -0.527808, -14.221658, 2.361469, -1.320704, 0.836442, -0.386591, -2.236613, 0.486746, 1.005684, -0.821714, -0.599291, 3.795557, -4.883873, 3.555826, 0.689591, -0.508781, 1.518971, -2.128448, 0.553367, -0.200060, 1.611065, -1.504008, -3.721884, -3.394505, 3.972555, -0.445324, -1.183890, 3.563913, 1.988339, -0.476866, 0.289509, 0.277188, -0.175615, -1.041347, 1.197368, -0.477900, -3.759550, -5.051634, -1.572159, -0.226668, 0.180172, -0.086704, 2.489282, -3.111502, -2.003207, 1.001518, -0.583694, 1.663272, -1.637953, 0.545772, 2.723981, 0.770260, 0.925456, 0.989657, 2.473933, 2.747635, 2.175160, 1.760652, -0.013310, -4.443879, 37.422813, 1.288954, 2.467471, 2.146290, 1.013962, -2.084025, 2.127673, -0.888463, -1.521253, -0.895446, 4.524404, -0.104275, -1.633331, -9.016782, 1.046254, -1.463881, 3.555183, -1.504391, 0.277646, -1.592795, 0.272081, -0.698069, 0.696722, -1.410604, 1.937863, -0.376410, -1.499335, 1.668633, 1.260768, -2.925495, -1.584171, 1.157681, -3.337044, 3.435848, 0.386049, 1.061394, 1.006677, 0.669047, -1.470088, 1.235580, -1.276175, -0.555348, -1.114300, 0.408720, 0.459978, -2.024472, 1.675984, 1.306379, -0.254271, -2.635481, 1.731658, 2.153453, 0.928214, 0.306720, -3.400702, 2.915029, 0.031397, 1.996717, -1.877975, -4.729939, -0.609253, 0.541097, 1.393276, -0.035789, -1.044343, 0.889185, -0.336026, 2.131327, -1.250257, -1.924997, -0.287107, 1.208007, -1.054186, 1.337112, 1.274661, -2.434684, 3.016158, 1.469676, -0.695139, 1.505548, 2.100273, 1.219837, -0.772411, 3.593307, 5.785521, 2.612813, -3.146140, -0.555663, 0.243268, -3.966340, 0.486868, -1.174454, 1.676140, 0.317372, 0.213547, 0.103339, -1.636872, -1.078531, 3.002460, 0.008469, -2.326029, 0.867810, 0.871880, -1.080869, -0.768073, -4.836300, 0.383782, 0.691735, -0.713520, 1.354015, 2.611808, 6.183017, -1.084267, 0.292993, 0.816155, 1.273668, -0.560137, -1.290062, 1.882704, 1.010641, -1.717645, 1.066451, 0.092508, 0.932700, 0.062565, -0.448221, -1.144008, 1.872820, -1.933916, -4.155032, 1.226316, -0.842341, -1.021217, 1.994580, -0.036678, -0.072384, 0.812870, 1.374167, -2.501688, -0.321656, 3.544239, 2.224328, 2.181247, -1.169514, -2.582330, -1.573699, -2.363550, -2.202179, 1.673287, 0.711271, 3.818364, -1.154218, -0.725751, -1.511678, 0.080292, -8.115035, -3.127221, -3.172623, -1.133258, 5.428519, -0.574502, -2.239282, 3.485080, -2.038507, 2.057469, -0.269610, 1.192541, -1.718417, 1.374550, -1.552978, -2.797242, -2.503451, -4.544235, 0.636139, 1.108707, 3.459209, 2.595264, -0.305967, -0.003920, 5.865139, -2.725132, -1.437748, -1.930311, 1.015183, 0.153521, -0.693563, -1.841589, 1.740456, -0.131857, -1.259349, 1.107379, 0.814856, 2.672195, 1.490787, 0.823584, -0.483895, -0.674276, 3.958681, -4.697596, 0.824732, -0.919115, -4.634318, -0.438301, -0.317027, -0.150343, 1.811067, 1.930733, 2.765487, -3.315078, -1.065359, 1.905147, 1.231166, -0.812233, -0.079013, 3.098343, 1.337479, 0.038435, -1.502123, -0.722991, -0.738154, 1.651970, 3.549967, -2.832844, 2.526853, -4.530358, -1.405437, -0.020393, 1.433086, -0.355914, -1.094193, 0.618523, -0.507370, 1.855727, -0.352494, -1.034884, -0.052144, -1.694263, -1.334652, -0.578301, 1.324574, -1.522585, -0.899184, 3.180330, -2.992112, -0.966538, 1.624621, -2.214644, -0.544830, 0.516665, -2.317800, -0.701623, 0.938447, 0.935423, 7.276221, 2.074192, -0.549290, -0.723959, -1.024809, -2.093009, 0.378594, 1.273304, -1.574900, -2.294054, 1.627634, 2.471876, 0.272425, 0.226814, 0.288442, 0.485079, 0.329602, 0.418512, -1.030336, 0.618441, -0.205079, -0.751919, -0.533747, 0.987899, -0.886300, -1.857596, 0.970968, 0.782056, 1.377219, -3.131898, -2.278232, 0.560626, -5.161348, -0.380426, 0.495337, 0.181925, -0.484390, 2.722185, 1.441381, -3.070866, -1.380223, -2.827929, -0.428279, -2.406868, -1.231688, -1.393025, -2.365047, 1.180624, 4.354129, 0.116944, 1.813338, 0.807496, 5.356595, -0.893292, -2.298326, 0.608894, -2.458173, -3.912065, 1.234876, -0.789027, 0.989614, 1.082788, -0.383455, 0.617175, 3.955452, -2.602732, 0.340088, -2.055584, -1.332299, -2.340340, -2.757793, 1.896885, 1.322887, 0.021037, 2.907897, -0.321884, -2.944226, -1.640788, -1.364418, 2.449870, -4.300892, 4.101160, 1.867850, 0.166516, -0.710645, -1.172884, -1.457727, 2.929085, -0.218713, 2.206084, -0.883459, -1.040819, -2.344896, -1.173216, -0.345644, 1.559119, -1.632681, -2.044267, -2.017350, -0.654807, -0.716796, -0.901010, -1.560858, -2.691737, 2.685868, 1.782925, 3.339304, 0.565915, 0.391605, 3.610030, -3.298944, -0.705625, -1.543735, -1.870102, 1.338298, 1.612513, 0.385103, 1.112341, -0.001102, 0.658675, -0.503109, 1.369964, 2.857534, -0.630183, -0.912625, -1.217065, -0.550542, 2.285744, -2.127434, -3.157195, -2.480932, -0.733192, -1.617258, -1.182549, -1.050172, -0.540184, -1.505584, -1.097327, 1.250695, 0.358068, 1.122536, 0.025612, -0.006341, 1.774774, -2.286098, -2.659644, 3.301547, 0.470509, 1.290470, -1.164262, 0.626534, -3.474944, -1.804067, -1.576454, 3.310174, -0.299973, -1.118997, 0.285524, -0.434010, -0.235016, 2.567713, 2.659858, 0.159478, 0.923787, 0.271707, 0.721747, 1.390933, 1.695482, -35.562675, -0.141364, 3.048325, -1.559244, 0.738925, -1.291286, -2.524163, -0.560224, -2.540498, -0.174527, 7.668949, -0.468271, 2.184141, -0.495266, 4.118879, -0.306242, 1.102313, -4.138959, -3.319368, 1.504961, -0.296103, -0.015277, -0.136177, -2.483171, -0.736024, 2.367176, 2.261982, -0.849145, -0.117148, 0.933736, 1.702317, -0.301388, -0.906758, -1.500765, -3.250679, 3.582183, -0.331119, -0.037043, -1.458076, 1.892284, -0.766386, 0.352996, -0.095548, -2.071965, -0.491389, 2.410537, 2.485983, 0.411316, 1.657690, 2.453979, 0.544419, 1.520648, 0.534818, 5.743890, -1.727980, 1.763634, -1.548289, 4.157279, 2.225596, -0.064131, 1.865851, -2.642000, 0.274984, -0.866992, -0.558188, -6.135097, 1.808120, 1.006938, -0.790857, 0.461167, -0.860196, 2.084737, 5.253679, -0.978117, -0.453410, -0.819495, 0.236075, -2.136300, 0.021472, 0.437208, -2.464820, -0.033580, 0.188308, -3.331381, 3.856862, 3.718570, 1.367965, -0.220747, -2.310231, -1.149265, -1.234862, -1.002847, -1.974621, 1.130993, 4.297782, 0.930710, -1.492884, -0.713347, -0.377052, 2.645738, -0.489495, 4.203647, -0.183929, 2.602280, -0.300018, 0.319702, 0.484708, 1.726325, -2.608667, 0.224787, 0.628384, -0.105292, -0.063091, 3.132477, -1.491148, 0.832812, 2.437400, -1.012723, 0.005830, -1.205948, -3.225458, 3.295642, 0.629336, 1.091727, -3.268245, -4.394265, 1.717201, 0.162753, 2.099977, -0.146153, 0.175116, 2.906495, -3.290455, -1.285598, 7.942502, 0.160152, 0.179591, 4.543355, 3.116941, 3.622699, 3.588044, 0.600032, 1.724521, -1.439306, 0.903875, 1.313146, -0.755100, 8.836950, -0.952139, -0.333382, 0.589961, -1.261379, -1.519937, 0.594025, 1.764304, 4.243717, -1.108132, 1.925209, -3.976291, 0.235073, 1.686457, -2.469280, -8.696437, -3.778089, 0.477731, 2.229237, -1.451623, -0.649171, -0.769915, 4.691844, -0.204187, -1.182377, 1.737289, 0.001245, 0.825427, 0.922635, 0.588298, 1.344133, -0.989039, 4.205564, -0.115000, -0.715023, 0.977006, -0.833281, -0.086541, 7.658555, 1.852724, -1.467018, -4.778613, -5.723665, 11.013165, -0.535247, 1.304788, -0.278120, -3.020401, 1.007590, -1.811615, -2.275402, 0.979134, -1.302686, -1.351974, -1.836746, 0.542104, 0.011732, 0.467072, 1.436075, -1.912158, -0.329361, 0.906835, 1.310337, -1.657316, -1.454618, -0.057573, 1.054785, 2.734224, 0.864511, -2.898540, 0.721743, 3.449452, 2.660782, -3.281970, 0.793531, -0.939987, 2.136473, 2.435774, -0.931134, 2.375545, -1.911816, 1.508261, 1.438431, -1.554907, 10.636431, 10.181509, 2.162733, 0.092647, -1.967986, -0.263800, -4.868186, -6.291202, 2.514089, -1.563930, 2.339781, 1.775288, 0.459960, -0.443059, 1.163096, 0.124251, -6.209301, -1.731759, 0.933085, -2.717155, -2.114019, 0.764707, -3.598078, -3.171012, -3.224734, -2.070172, -1.446331, 2.061223, -1.382807, 2.091001, -0.697143, -1.502571, -0.759867, 0.295158, 0.411237, -4.074205, -0.500290, -1.297131, 0.634780, -0.869266, 4.393310, 0.298832, -2.684585, 2.340415, -2.036564, 0.622905, -2.296228, 1.926786, 3.597500, -1.945964, 0.066769, 0.744542, 0.415474, -0.575763, 1.847393, -0.139588, -4.165627, 4.270953, 2.814140, 3.188056, -0.450949, 0.978394, -0.130957, -0.081690, 2.020355, 1.264999, 2.465885, -1.305273, -1.484652, 1.913352, 1.274480, 1.071047, 32.553612, -3.937133, 1.768175, 3.947704, -3.099431, 1.153636, 2.373486, 2.360721, -2.020087, 2.675651, 2.172042, 1.795803, 0.559099, -1.527675, 1.322240, 2.126581, 1.912614, -4.063071, 1.399081, 4.218424, 5.575891, -0.873354, -0.366251, 0.625111, -3.856376, 3.276441, -1.155635, -2.581620, -3.619178, -2.831986, 1.644085, 0.133040, 1.446548, -1.878526, 1.286804, 2.888686, -2.470215, -2.362903, -0.971186, 1.676210, 0.187425, 1.839463, -1.896287, -2.081707, 1.814273, 0.938150, 1.786313, -3.683124, 3.041575, 0.135497, -0.442793, -1.932262, 3.042321, -1.016880, 2.041277, -2.040796, -0.421785, -1.340821, 0.105589, 2.341746, 0.292477, 0.475112, -0.495902, -0.475660, -1.491468, -2.629827, 3.888282, 2.804010, 0.150179, -0.908230, -1.783017, 0.495803, -0.610453, 0.289567, -0.078798, -1.560782, -1.805168, -2.729794, 1.172817, 2.256393, -0.442398, 0.522677, -1.631465, 0.356985, 3.064646, 2.438519, -4.161843, -2.152719, -3.548280, -1.114138, -0.859323, -1.193447, 1.367581, 0.220056, 0.163054, 1.273387, -0.971761, -2.162730, 0.737844, -0.055890, -0.216024, 0.862837, 0.151634, -0.601276, 0.358210, 1.970998, -0.229344, -0.017123, -4.781201, -1.324280, -2.443600, 1.611152, 0.149101, 3.477759, -0.751891, -0.717900, -1.417735, 1.407221, -1.185286, 2.538493, 2.014459, 1.288729, 0.121079, -0.775372, 1.095880, 2.870790, 2.828023, -0.678468, 0.029732, 0.468892, 0.173412, -0.554108, 0.868722, -0.347043, 0.670441, -0.115965, -1.428901, -1.543623, 1.486332, -0.581373, 2.728561, 0.982189, -2.209894, 3.659011, -0.832777, -2.435964, -1.310557, -1.055155, -1.834021, -1.562546, -3.038025, 3.090860, -2.067988, -3.092913, 0.118994, -1.485082, 2.321303, -1.523272, -1.430369, -0.029205, -1.739594, 0.019325, 0.569824, -1.613913, 1.451680, 0.781437, 2.524419, 1.962079, 0.689057, 0.560689, -1.565743, 0.606738, -0.366487, 0.825362, 0.567166, -2.249720, 1.356489, 1.744161, 1.633906, -1.390061, -0.507055, 0.629742, 1.448011, 1.131591, 1.812740, 1.272440, -4.645047, -4.487141, 1.566564, -1.905134, -2.191869, 0.572741, -0.455768, -2.307575, 1.607099, 2.300626, -0.653849, 0.221068, -1.337414, 3.450238, 1.781016, 1.688878, -1.666610, -1.287050, 1.609571, -1.283102, 1.244545, -1.443785, 1.087902, 1.376294, 0.741840, 3.563230, 2.121260, 1.941692, -1.210015, -1.075073, -1.537348, -3.691505, 1.077912, 1.348834, 0.498320, -1.474098, 2.513743, 0.034244, -0.481899, 1.553230, 2.030468, -0.253737, -0.082557, -1.063913, -2.305557, 0.269592, -2.979993, 0.824435, -2.439241, 0.592147, -1.105869, 0.421400, 1.004838, -1.749238, 0.952104, 0.919140, 1.684337, 2.167483, -0.795320, 3.446412, -0.198248, -3.708080, 1.209328, -2.476055, -2.325776, -3.323361, -0.395597, -2.130917, 3.682760, 0.225562, -0.801695, -0.575638, 0.496647, -1.356318, 0.600407, -3.152001, 0.148890, -0.074623, 0.417407, 2.860341, -1.896055, -2.015702, -0.401882, -9.507044, -1.447032, -0.036139, -0.848994, -0.379429, -0.110780, -0.272023, -0.108900, 3.433757, 0.450129, 0.396789, 1.146661, -3.451382, 2.723109, 2.754865, 0.767603, 1.128464, 1.264460, -1.380370, 9.841789, 1.019593, -0.664542, 0.108049, -4.518657, 3.028623, 0.955966, 2.159739, -0.992145, -3.089396, -2.380461, 3.170674, 3.399100, -1.193787, -1.100064, -1.222812, 1.331865, -4.171266, -0.224741, -0.748337, 0.600880, -4.090579, 1.785532, 0.843806, -0.897074, 1.120391, 0.226116, 1.339815, -0.558596, -0.229787, 0.421276, -1.492851, 2.736720, -1.181484, 1.033056, -2.783668, 0.868677, 1.473027, -0.409419, -0.926960, -1.578116, -1.657176, -4.058239, -2.139534, -4.226902, 0.042924, -4.044942, 0.975387, 3.117355, -2.011942, 1.897419, -0.394071, -2.194358, 0.765811, -1.147434, -0.198820, 4.750773, 1.958392, 0.566013, 0.746863, 0.205276, 6.244852, 0.292197, -0.264540, -1.483035, -0.521862, -0.821273, 0.241537, -1.867573, 0.887766, 0.970351, 0.943935, 0.907276, 0.863435, 0.609040, 0.758961, -2.844856, -4.021582, 2.337291, -0.033980, -1.227764, -2.930726, -3.163921, -0.012551, 2.236899, 0.202573, 4.800804, -3.107728, -2.295274, -2.600798, 0.614929, 5.242928, 1.385072, -0.441010, 0.496632, 3.172925, 1.638507, -0.125987, 5.617575, 0.405079, -2.154732, -2.598893, 3.025715, 0.283551, -1.228085, -3.021509, -1.903265, -1.605352, -2.429972, 0.022392, -1.591423, 1.237664, -0.669618, 2.921721, -3.849840, 0.507070, 2.304445, 1.411633, 0.925293, -1.438281, 4.524236, 3.233037, -0.987758, 4.162241, 1.087106, -3.555248, -1.064314, -1.531470, -0.240191, -2.561610, -1.132247, 1.349483, 2.722615, 0.037841, 1.215080, -1.428522, -2.192810, 1.110667, -2.803056, -0.672909, -1.977815, -0.052312, -2.126122, 1.366835, -3.752233, 1.900298, -0.700606, 0.833546, -0.918877, -1.297674, -1.865373, 2.671888, -0.841652, 0.527628, -3.107635, 1.778736, 6.087658, 10.168410, 1.901435, 0.008429, 1.880625, 0.691697, 1.251415, 2.020111, 0.165611, -1.962046, -2.567108, -1.646126, 1.804067, -4.159413, -2.876052, 0.582987, -2.315321, -2.834074, -0.143538, 0.333040, 2.520455, -1.053464, -0.936291, 0.497383, 2.377161, 0.688116, 0.252176, 1.265777, 1.036058, -1.827014, 0.232887, -1.180766, 1.928753, 0.833420, -5.280427, -3.046776, -0.869522, 2.473118, -4.426275, -1.009449, -0.393237, -0.908902, -0.088011, 0.554225, 0.937001, 0.837067, -3.216558, -0.756160, 1.128917, 0.964452, -0.273247, 0.137980, -0.334299, 0.644827, 1.049165, 1.687600, -0.582325, 0.678831, 3.991718, 3.445086, -2.419575, -1.215780, 0.836874, -1.800161, -4.176029, 1.932638, 4.352362, -0.640140, -0.239435, -0.297160, 0.884801, -5.469331, -0.966645, -2.994428, -1.457712, -1.884226, -1.562829, -0.063225, -1.810441, 1.137481, 1.578315, 4.733838, -0.413119, -2.760687, -4.963616, 2.259133, -1.462614, -0.125429, 1.826625, 0.243741, -1.940729, 4.980882, -1.852582, -2.769213, 3.123670, -3.025451, 1.775786, 0.248842, -0.728544, -1.077011, -1.391178, 0.466446, -1.327900, 0.010156, -1.119019, 0.406476, -3.023509, -2.991761, -2.242916, -2.700393, -1.149276, -0.574640, -0.623269, 4.592867, 0.085469, 2.845264, 0.500940, 1.761536, -0.758278, 5.217238, 0.938784, -0.517966, 0.385037, -1.285301, -2.375467, -3.277109, 5.389163, -1.802665, -1.425073, 2.843237, -1.299174, -2.772480, 1.326686, -0.461601, 2.181789, -1.470306, 0.363675, -1.825438, -1.595971, -2.633875, -1.263443, 11.719139, 0.751982, -5.437104, -0.402077, -1.769803, -0.488290, -3.136333, -0.206896, -2.774237, -6.601932, 3.922718, -4.739939, 2.086078, -3.384333, -0.272934, -1.534686, -0.467060, 1.184582, -0.470652, 12.171447, -2.855459, -8.415219, 1.164989, -1.923474, -0.980914, -0.601838, 1.957440, 0.195562, 1.815339, 0.264663, 0.158703, -1.772841, 1.136608, -0.290822, 1.927877, 0.562484, 2.267569, 0.140409, 2.506416, -0.439031, 0.860347, -1.896669, 1.397222, 1.579570, 0.485520, 0.957377, -1.102430, 0.442603, -0.920362, -1.657130, 4.251542, 0.022865, 1.276556, 0.728996, -1.456246, 1.469049, 17.869566, -3.002699, -2.320802, -0.407301, 0.102661, 1.159996, 3.357108, -0.603254, -1.308291, -3.801401, 0.142082, 0.596891, 3.836693, 1.812102, -2.562486, 2.357852, 7.259953, 0.651906, 1.206739, 2.925079, 0.821211, -3.675958, 4.616876, -0.251318, 1.483648, -0.104531, 3.231146, 1.392708, 0.213286, -4.196676, -1.593157, -3.684262, -0.936107, -1.851555, 1.740610, 0.759534, -0.725802, -32.275948, 0.821350, -2.229753, -3.098523, 0.335518, -3.705818, -2.480252, -2.382177, 2.335834, 0.559883, -0.078840, 0.265768, 0.672536, -0.617955, -2.180997, 0.050136, 1.086448, 2.409880, -1.411031, 0.431328, -0.727582, 1.202057, 0.676885, 2.333050, 0.667056, 0.704034, -3.499928, 0.377022, -3.475610, -0.461822, -0.347930, 0.258168, 1.602084, 0.519829, -0.155424, 1.457234, 0.126837, 0.867928, 4.091778, 1.680606, 4.067370, -1.098706, 1.745742, -1.184344, 2.251384, -0.088541, 0.205009, 0.219193, 0.667914, 3.437428, -1.572693, 0.404962, -1.385043, 2.847858, -0.488513, -0.725883, 0.752501, -0.153214, 1.029474, 1.962504, 3.080513, 2.361320, -0.633588, -1.436562, 0.260860, 26.884304, 0.354994, 0.023750, 0.750965, 1.353084, 0.426553, -1.632285, 0.965408, 4.016201, 2.177340, 1.253002, -1.670665, 1.720569, -2.180720, 0.322581, 0.279140, 3.985285, -3.724750, -3.344991, 0.349461, 1.836719, -0.266715, -0.634795, 1.627708, -2.453827, 2.661302, -3.227677, 0.053888, -2.480497, -0.681098, -0.049629, 2.152981, -0.581374, 0.205937, -1.006680, -0.501193, -2.547718, -0.441294, -0.789560, 1.551607, 1.530261, 3.002646, -3.386208, 0.647514, -1.249280, 0.674235, -1.530674, -3.838077, 3.722666, 2.963236, 2.088184, 3.554217, -1.200083, 1.763361, -1.686896, -1.194727, -2.250880, 0.875894, -0.186709, -25.303444, -0.505925, -1.991530, -1.867339, 1.825814, -0.766634, -0.213235, 4.148894, -0.046314, 2.148987, -1.871356, 0.335448, -0.485913, -0.093247, -3.264498, 0.528065, 0.522954, -1.395236, -2.970262, -0.581401, -1.861303, 0.173270, 1.324822, -0.018929, 1.717697, 1.011000, 2.334809, 0.878979, -0.626252, 0.987577, 2.981012, 2.250051, 2.358181, 1.915170, -1.237683, -2.741322, 2.350392, -2.240174, -2.580384, -0.634321, 0.317450, 1.725892, 0.162523, -0.422389, -0.178484, -2.114431, -4.840956, -0.001833, 0.139491, 4.319082, -1.384395, 0.323176, -1.353467, -1.339914, -1.007622, 8.807545, 1.286002, -2.342683, -1.185613, -1.472908, -1.924402, 4.893974, 0.222800, -0.740518, 2.156360, 2.747664, -0.312752, 2.395748, -0.615992, -8.395935, -0.730813, -0.073218, 1.852324, 0.291219, 1.468868, -1.620990, -0.756095, 1.401205, -1.078258, 0.483770, -0.073835, 0.852245, -2.719850, 1.460309, 1.374558, 8.152399, 2.677590, -1.674278, 0.375696, -0.073122, 0.925756, 0.923439, -0.069477, 2.279387, 2.252507, 1.538246, 1.814035, 1.929125, -1.111000, 0.341246, -1.168864, 2.812251, -3.087615, -0.544177, -0.092432, -3.101908, 2.925842, -1.453098, -0.303561, -1.021411, -0.136682, -5.494426, 0.610433, -2.182517, 3.119542, -1.883906, -0.745869, 2.522353, -1.648916, 0.309569, -0.577917, 1.033753, 2.487350, 7.941953, -0.071100, 0.527296, 0.779083, 6.350358, -3.325049, 1.075220, 0.105128, 0.276124, 0.347600, 0.306576, 3.274002, 1.036085, -2.652688, -2.018159, -0.050256, 0.416610, 0.418685, -1.532887, 0.523052, 3.134366, 0.387958, 1.859184, 2.089394, 1.539085, 1.657541, 1.209471, 1.878232, -3.116538, 0.800451, -3.243739, 2.656432, -0.071025, -0.537716, 2.768167, -0.689752, -4.555896, -2.193995, 1.733658, -1.575865, 0.762150, 2.772280, 0.646617, -2.128708, 2.234904, -2.126125, 0.459754, 1.413508, 0.166073, -2.205717, -0.741679, 0.306937, -1.672999, 0.225218, 0.055466, -1.127936, 7.234384, 0.063245, 1.979858, -2.608743, -2.807106, -2.397985, 0.106909, -3.808125, 1.513469, -1.036234, 0.811388, -2.431126, 2.732509, 0.589472, -1.097014, 0.551920, -1.516936, -2.533307, 0.797511, -2.425043, -3.588099, -1.160765, -0.590146, 3.004026, 0.810287, -2.107759, -0.651307, -5.370883, 2.917656, 0.991122, 0.444265, 2.693725, 0.286769, -1.541028, -3.018210, -2.283916, -0.207154, 1.837811, -1.245008, -5.110644, 1.981874, 1.236664, 1.905553, -0.207308, -1.105416, 4.194646, 0.440569, 1.659566, -0.880465, -4.547189, -2.118434, 1.557453, 2.841252, -5.531410, -2.707672, -0.314554, 0.362334, -2.255540, 0.359243, -0.935208, -0.932006, -1.131725, 1.080974, -1.834844, -0.971595, 2.742725, 3.401529, -1.261523, 0.220123, 2.862516, -0.858204, -0.541691, 1.122402, 3.115028, 0.782756, -1.248350, 0.043220, -0.986485, -0.844554, 2.879936, -0.039591, 0.383270, -0.683167, 1.174577, -0.593699, -1.137541, 1.152888, 1.766484, -7.748785, -0.404558, -0.467036, -0.500680, 1.611474, -0.383380, -3.673885, 1.017248, -0.770349, 0.237903, 2.069979, 1.630354, -2.739210, 3.819390, -1.732684, 2.563601, 2.213180, -2.380856, -2.852253, 1.170615, 0.828863, 1.211223, 2.499185, -0.747931, 0.118144, -1.666906, 0.565364, -1.657013, 0.774405, 0.693492, 7.705479, -1.408086, -0.155176, 1.412633, -5.123066, -0.853785, 0.116899, 2.256178, -2.218943, 1.518238, -0.196591, 0.143943, 0.285298, -2.482928, 0.615830, -2.046060, -0.975004, -2.833664, 3.254058, -1.370345, 0.944582, -0.014555, 1.834112, 2.484418, 0.814740, -0.132238, 1.918314, -1.774436, 0.460359, -0.580844, -1.645435, 1.954685, 2.618668, 0.267995, -2.560254, -0.594974, -2.286581, 4.714139, -0.405705, 1.295618, 0.840673, -2.838477, 0.366501, -1.036157, -0.400613, -1.738227, 1.300794, -1.672292, 3.829979, -3.479953, 3.891050, 0.341126, -0.399483, 1.662642, 3.562313, -1.184650, 2.612743, -3.805794, -2.371483, 1.948942, 2.188965, -0.459029, -0.178023, -1.700147, 3.120115, 5.629900, 1.224204, -1.879678, 1.268884, -2.494606, 0.783073, 0.035119, -0.997701, 0.350684, -1.231518, 1.909020, -2.098758, -0.661888, 0.994528, -1.374923, 2.376079, 0.031113, -1.458987, 0.113488, 0.266556, -2.248442, -0.549227, -1.980329, -0.843419, 0.893711, -1.482391, -0.773147, 1.418742, -0.100246, -2.365190, 1.173016, -0.702880, 0.275237, -1.607930, 0.260333, -2.186061, -0.709117, -1.991098, 3.764254, -3.194555, -1.260547, 1.071692, 3.033343, -0.641039, -0.255529, 3.194899, 1.048945, -0.669115, 4.109255, -2.829791, -0.658942, -0.670331, -3.036335, 1.109055, 0.316420, 9.496139, -1.423056, 4.316561, -3.777977, -0.611273, 4.206706, -3.108094, 2.093183, -0.592171, -2.694616, -2.441919, 0.851335, 0.160457, -2.691463, -0.074577, 0.744847, 1.683688, -0.708158, -1.727233, -1.502183, 0.525043, 0.983311, -0.089715, 0.532941, -0.811899, 0.976230, -2.550968, -0.344250, -0.052567, -0.187679, -0.121404, 2.285113, 1.596615, -1.304425, -2.099068, 1.705707, 0.282685, -1.729276, 1.459553, 0.320318, -4.074892, 1.004075, -3.545393, 1.803405, 0.490817, -4.327343, 1.060507, 2.926682, 1.222938, 3.830746, -0.337875, 1.288773, 0.835695, 1.570575, 3.477370, 0.256512, -1.542086, 0.159785, 0.399716, -0.581507, 2.310691, -0.635375, 0.457800, 2.778989, 4.189120, -1.164997, -2.390188, -1.172381, -2.444343, 0.283850, 2.672297, -0.091714, 0.491060, 3.292467, 0.893021, -0.021946, -4.014757, -2.941429, -3.471601, 0.691645, 0.227577, -0.660446, -0.993292, 1.157482, 0.663260, -0.723520, -2.946966, 0.588723, 0.340043, -0.358396, 1.354849, 1.527313, 0.792265, 0.354935, -3.001401, 2.202762, -0.826216, 1.678419, -0.252368, 2.635601, 3.578719, 3.746924, -0.271551, -2.890261, 2.253434, 0.673195, 1.846275, 0.377794, 1.797091, -0.269892, -2.769761, 0.315330, 3.191075, 1.445889, 1.961443, 0.250564, 1.217402, 1.685299, -0.773906, 3.594047, -0.128995, 0.446818, -2.274753, 1.524932, -5.203915, 1.088118, 0.079942, 3.207416, -18.327145, 1.500175, -0.097854, 2.531832, -1.950670, -0.083165, 2.955576, 0.681629, -0.329289, -3.521765, 2.633065, -0.085364, 5.152068, -0.129515, 2.816390, -3.628945, 0.186241, -1.867233, 1.983360, -0.515092, 1.410643, 0.049975, -0.440827, -0.180698, -0.876025, -1.237767, 2.805179, 1.542505, -0.951790, 0.443897, -2.973520, -3.115988, -3.136214, -1.392136, -4.330337, -2.264652, 0.625757, -4.091477, 5.732088, 0.250295, -0.520872, -0.999017, -5.399399, 1.408189, -1.518100, 0.076899, 0.173335, -0.000062, -0.360869, 1.392707, -0.592256, -0.071057, 1.330815, 1.857339, 1.405391, -1.056357, 3.606451, 0.844912, 0.430513, 0.705153, -0.095922, -3.005373, -6.676332, -1.697640, -2.368549, 0.382992, -0.944774, 1.198584, -1.091452, -1.749086, -3.316520, -2.743624, -0.783803, 0.405243, 0.186438, -2.742365, 2.742394, -0.300057, 1.709888, -0.119355, -1.055460, -0.554546, 1.549400, 3.316050, 1.876061, -0.066178, 0.378583, 2.503380, -0.843418, -2.350612, 1.370083, 4.067267, -0.592636, -1.435324, -0.224314, 3.440698, -3.128674, -1.354647, 0.871077, 0.835582, -2.731527, 3.152105, -13.977840, -3.494694, -1.888668, -0.369791, -2.067304, -1.631454, 1.369503, 1.902894, -0.525999, 0.394126, -0.526112, -1.452425, 0.266547, 0.910412, 2.690451, 0.452772, -0.904265, -0.274242, -1.941391, -0.781456, 1.339111, -0.650924, -2.150021, 0.411827, 0.306410, -1.720717, -0.847376, -0.433327, 8.572508, 5.576869, -1.604961, -3.355456, 0.578090, 3.117163, 0.740736, -1.816481, -1.542356, 1.402068, -3.337177, -3.460914, 0.022969, 0.263521, 2.447577, 2.692053, -2.677152, -1.154691, -0.166149, 2.455483, -1.960949, 0.644558, 0.451422, -2.235790, 0.731520, -0.340948, -1.902708, 0.762279, 0.125709, -1.536236, -0.245376, 5.871339, -1.315382, 1.152163, 0.313443, 2.684904, 1.522253, -2.293488, 1.985134, -1.717681, 1.696683, -2.022664, 2.500051, 0.610928, 2.378994, 0.165070, 2.061111, -0.533743, 2.667622, 3.499480, 2.485019, 1.861316, -1.150906, 1.217355, -2.907215, 1.377991, 2.626987, 2.557068, -2.540422, -6.635588, 0.997638, 3.295728, 0.550338, 1.183280, -0.518508, 2.825861, 3.199516, -0.683444, 2.162984, 2.370837, 4.094376, -0.561323, 0.784378, 0.182726, 1.457809, -1.172417, 2.387145, 0.324454, 1.190073, 4.428410, 1.672801, 2.330206, 3.335100, -0.557584, -1.865616, 0.313762, 2.122341, 1.508437, 2.887865, 3.216261, 3.003069, 4.676777, 3.199836, -1.936886, -1.224461, -0.051272, -3.227903, 1.606121, 1.955314, 0.546526, 0.661211, -1.968910, 1.787858, 1.048485, -0.461875, -0.509854, -1.147480, 1.632553, 1.572888, 1.340511, 2.246652, -0.744147, 1.609240, 0.752408, -1.323570, 1.025877, 0.814479, -1.259313, 2.315588, -1.214857, 0.447328, 1.474373, 1.504266, -0.497109, 0.577923, -0.198901, 0.571817, 1.572346, -0.229665, -0.482301, 0.700038, 1.471916, 0.754951, -4.222556, -0.885760, 2.337687, 3.346586, 0.457164, 1.315627, 1.394005, -1.703673, 0.883069, 1.166987, -0.816095, 1.200109, -2.375040, 0.826257, 0.166096, -1.786063, -3.522976, 3.031999, -3.488905, -0.379346, -1.583687, 1.950091, -0.198649, -1.979665, 0.650037, -1.176644, 2.323772, -1.853436, -1.898267, 0.950301, 2.705135, -1.034136, 1.889340, -1.965656, -1.262732, -1.637557, 0.371705, 0.813203, -2.314916, -2.003534, -0.387944, -1.389285, 0.283866, 3.240730, -5.791040, 2.754301, -1.986605, 5.498390, -0.273563, 2.022370, -0.106619, 5.943636, 2.984904, 0.613079, -0.669535, -0.343581, -2.248723, 0.348882, 0.099603, 0.015564, -0.590038, -1.586601, 2.235744, -1.424522, -0.791221, 0.587769, -1.771873, 0.902260, -0.221643, -1.720520, 0.883625, 2.038339, 1.595812, -0.467120, 1.634040, 1.504307, -0.748396, -1.020328, -4.903410, 1.616802, 1.099766, 0.745407, -0.021960, -0.519794, 1.157142, -0.792646, -3.190317, 2.819183, -1.442791, -0.101592, 0.509061, -1.473928, -2.708030, 0.791176, 0.473671, -1.907742, 2.050101, 0.003960, -1.289529, -2.341501, -0.373291, -0.925099, -0.040393, -0.479414, -0.349596, 1.877697, -0.776685, 1.993453, 0.372975, 0.333932, -2.029835, -1.045179, 2.716208, 1.865170, 0.747971, 4.798818, 1.579798, 1.555352, -1.155734, 3.145859, 0.887819, -3.669680, 0.728159, -0.804880, 1.060545, 0.031082, -2.118540, 1.153332, 3.899940, -1.163471, 1.295803, -2.362670, -2.000762, 0.001854, 0.792970, -2.817198, 1.936855, -3.864910, 2.709139, 1.039248, 0.088502, 0.231339, 0.591631, 0.030345, 1.299663, 3.123892, -0.961180, 0.043213, 5.733878, -4.098113, 3.308362, 0.510799, -4.140917, 0.843890, -1.273466, -2.071602, 1.445319, -1.818631, 0.162340, 1.681147, 0.362149, 5.492742, 0.825089, 1.722855, -0.476926, 0.432826, -2.043187, -2.016047, -2.091608, 0.597438, 0.042428, 3.973989, 0.883174, -0.129641, 2.237979, 0.567980, 0.839164, 0.976711, -2.264026, 1.649265, 1.344430, 1.434795, 5.406022, 3.077146, 2.661374, 3.106198, -2.270719, 0.887519, -0.145979, 3.447535, 1.555766, -0.138652, 1.195158, 3.220088, 0.879781, -2.362654, -6.360222, 0.395982, -2.533304, 4.291794, 3.155651, 1.251960, -2.407456, -0.218786, -2.583162, -0.899505, -0.780827, -0.983244, -3.478741, 1.778161, -0.658125, 0.597271, 1.897745, 1.777378, -0.100952, 1.271212, -0.681034, 2.460438, 1.110631, 1.252983, -0.114646, 0.479300, 1.725348, 4.230619, 1.187448, -2.427088, -1.236009, 2.918612, -1.148390, -1.583798, -1.435729, -1.783818, -2.767493, -1.541569, 0.203426, -1.242959, 0.281229, -0.392559, 0.520429, -3.234192, -5.539012, -2.151313, -2.462922, 0.635300, 1.190563, -0.151796, -0.622898, -0.042642, -0.083337, -0.730799, 3.834823, -0.641445, 1.817954, 3.308098, 2.103870, -1.456245, 1.461707, 1.371928, 0.733169, -2.590485, 0.514485, 3.581903, -1.606874, 0.716330, 1.596098, -0.624766, 0.570248, 0.343349, 0.814959, 2.110700, 3.385014, -0.654017, 0.103144, 1.129105, 0.854295, -1.497854, -0.833896, 0.883463, 2.740201, 1.252455, 0.383430, 4.764862, 0.244701, 0.397635, 2.114073, 3.711519, -1.009765, -2.999290, -0.075239, 1.702090, 4.073386, -2.221128, -0.173297, -1.598970, 1.764099, 1.255920, -3.153074, 0.940827, -0.085776, -0.153299, 3.531004, -2.893436, 2.231733, -2.386345, -1.483328, -3.745522, -5.938141, -1.462745, -1.746019, -2.920442, 0.672212, 0.022831, -2.624349, -2.061772, 0.922139, -2.596588, 2.745512, 1.169222, 0.959605, 3.072976, -0.254427, 5.414840, 0.527497, -1.430076, -1.501413, 2.057433, -1.026962, 5.627439, -2.945267, 0.809274, 2.203258, -0.872946, 4.923897, -1.509921, 0.499868, -1.122034, 0.442121, 4.887639, 2.859808, 0.291162, -0.565278, 0.837086, -5.220106, -0.498070, 1.504637, -0.144360, -0.114923, 0.522100, -2.818848, 1.758005, -0.885172, 1.853490, -2.924307, 2.890154, 1.534062, 1.918770, -0.117491, -0.226110, 0.317553, 0.141368, 1.936240, 2.881628, 0.301003, -2.461584, 0.041076, 0.719569, -0.117806, -2.094253, -2.824019, -1.391719, 1.183342, 1.017699, 0.376079, -0.095749, 0.889862, 1.062274, -0.524244, 2.349395, 1.186792, 2.231821, -1.102010, -1.332299, 2.127689, 0.351990, 1.511401, 0.622317, -3.630643, 0.548977, -3.208701, -2.996888, 2.649668, 0.304146, 1.341268, -2.146057, -1.861894, 4.507286, 0.904826, 1.395878, 0.450848, 0.070200, 3.053755, -2.140768, -2.565916, -0.972909, 1.667059, 1.065236, 0.645459, 0.884246, 0.220186, -3.282768, 1.298033, 4.002408, 0.833334, -3.617452, 0.368626, -1.903755, -0.626035, 1.850133, -3.082982, -2.774525, 1.769177, 0.522702, 0.263647, 1.313907, 3.750313, 2.017340, 0.823528, 2.732456, -3.571347, 2.283923, -0.022426, -0.600996, 1.081142, 0.014277, 2.022802, 0.375120, 1.400345, -2.918789, -1.019804, -1.212387, -4.197097, 0.703760, -1.529950, 2.411883, -0.584985, 1.346535, -0.723685, -2.590233, 3.926456, 2.828357, -1.109949, 1.414086, -0.135390, 0.382299, 2.244257, -0.295509, 4.518786, -3.660137, -2.117182, -2.384901, 3.393075, 0.985744, -0.516841, -0.374062, -4.132094, 2.249375, 2.650640, 0.171410, 3.396479, 0.069148, -0.270228, 0.905647, 3.294580, -3.019566, -3.331932, 2.340150, 2.807488, 2.162702, -3.298753, -1.766656, 1.310992, 0.260271, -0.273308, 4.299369, 3.715046, 1.070362, -1.441203, 0.129578, 0.900721, 5.234731, 2.976517, -0.659467, 0.284069, -0.954808, -0.256831, 0.583463, 1.284158, 1.351618, -0.708611, 2.189399, 0.599912, 1.235909, -1.296227, -0.026225, -0.115726, -1.447470, 1.321840, -0.550714, -0.555223, 1.529182, -1.148362, -1.568398, 3.317645, -0.735568, 0.437838, -0.237062, -1.593969, -1.894876, -1.147040, 6.875043, 11.417304, -1.270446, -0.386883, 0.448277, 0.306192, 0.444976, 0.047800, 0.323642, 0.030898, -0.246496, 1.848722, -0.699640, -2.337017, 3.233449, 1.195664, -1.263332, 2.178815, 0.288599, -1.581660, 3.966269, -0.985900, -0.975200, -0.793356, -2.546527, 0.314768, -1.653216, 0.076431, 0.527626, -4.124629, -0.522641, 2.863382, -0.581339, -0.483502, -1.371282, -2.072694, 2.361188, 1.388546, 0.461689, 0.054946, 2.079017, -0.905662, -1.667842, 1.560647, -2.268772, 2.163057, -1.461485, -9.571809, 2.473650, 27.804079, -0.824428, 1.293797, -3.527821, 0.322644, -0.829934, -1.641960, 1.559845, 0.651142, -2.018788, -0.652320, -0.534693, 2.995028, 0.591905, -0.210980, 1.812501, -2.794134, -0.021838, -1.462607, -4.406371, 0.911137, -1.410916, 1.293619, 0.665017, -0.952757, -1.092961, 3.622439, 4.556889, -0.322486, 3.173908, 2.919918, 1.061108, 1.607022, 0.961167, 1.944426, -2.335304, -0.616037, 0.400255, 3.586429, 0.725041, 0.298737, -1.890886, -0.024518, 0.091631, -1.731019, -1.398399, 1.815781, 4.581513, 2.576745, 1.146513, -0.740690, 0.073969, 3.108618, 0.535761, -0.192938, 0.454423, 2.053000, -0.904300, 2.072179, 0.649410, -0.662264, -0.673923, -1.733026, 2.902329, -1.831697, 3.206669, -1.994342, 1.821010, -1.573532, -0.257321, 5.394223, -0.162241, -3.188801, 0.847548, -2.007671, 3.947463, 1.168257, 2.246848, 1.873971, -0.494318, 0.724031, 2.609424, -1.787059, -0.378604, -1.377902, -3.019529, -1.382988, 0.023967, -0.011094, 0.253847, -1.189952, 0.510796, 1.386401, -2.568677, 1.666898, 2.934337, -1.311064, 0.238594, 0.870015, -0.393368, 2.161388, 3.384138, -2.131927, -2.342558, 0.287042, 3.170932, -4.181534, 1.297255, -2.535244, 0.296611, 0.657501, 2.486350, 0.579065, 0.901807, -1.826978, 3.008395, 0.015162, -0.812392, -5.237899, -0.064044, -0.666688, 1.184950, 1.343734, -1.661986, 0.091535, 2.635372, -0.006127, 9.887297, 4.918875, 0.266848, -2.460409, 0.854952, -1.334359, 0.044901, -0.075265, 0.200052, -2.027168, 0.649583, -0.283191, -0.853923, -2.430825, 2.559544, -0.007875, -0.466313, 0.781030, -0.850323, -1.138282, 1.897385, -0.009445, -0.401657, -2.866511, -2.329610, -1.208160, -1.773448, 0.369010, 0.188112, -0.367383, -0.806061, -0.871504, -1.204538, 0.251074, 0.966655, -0.356678, 2.718896, 3.092988, -2.580250, -0.622785, -0.762455, 0.026455, -1.173746, -3.874841, 1.271729, 1.204147, -1.713592, -0.083661, -2.208164, 1.115615, 0.098561, -0.295923, -1.642114, 3.539276, -1.431937, -0.611073, -2.333883, -0.713340, -1.436290, 0.124211, 0.463780, -1.330325, -2.417066, -1.452549, -0.621094, -0.331997, -0.248763, -1.276592, -0.835736, -0.789976, 0.210436, 2.124712, -1.141259, 0.878823, 3.689657, -0.996731, -0.788607, -1.662502, 0.702889, 0.455417, 0.196082, -1.905845, 1.417846, 0.184089, -0.428133, -1.289425, 1.620126, -0.999889, 0.915726, 2.355841, 8.382513, -2.161212, 15.398536, 2.307024, 1.641678, -1.617959, 1.106972], "qwen:latest": [6.398593, -9.147916, 3.167985, -0.016698, 5.146078, 5.260313, -0.588165, 6.041094, -0.448700, 1.538580, 4.409100, 1.965835, -2.027954, 3.322869, -0.209723, -1.591358, -0.332919, 0.490199, -1.801314, -1.394502, 6.208906, 2.582299, 8.829220, -1.215299, -10.441714, 6.180038, 2.357235, -0.251530, -1.217379, 6.260386, -1.005010, 3.707588, 2.900968, -2.015647, 1.153357, 2.200070, 4.747961, -2.433741, 4.030592, -9.292274, -3.870678, 2.177808, -0.545548, -2.237058, 2.649582, -0.528622, 1.250894, -0.112586, -0.969227, -0.571061, -0.680663, 1.271584, -5.168596, 4.699327, 5.181798, 1.421188, 6.487319, 3.149934, 2.146110, 0.893968, 4.691792, -2.652787, 0.780389, -7.469417, -1.913996, -4.114168, -1.824573, 1.825537, -7.670074, 0.083751, 1.497249, -1.842984, -0.207182, -1.217132, 1.720826, -0.462654, -0.142980, 6.752104, 0.513007, -9.219392, -6.861326, -1.046578, -3.621952, 8.216535, -1.929723, -0.226388, -0.364569, 0.592417, 0.661270, -2.502738, 0.655540, -2.301271, -2.658660, 6.579635, -2.761786, 3.214799, 1.964015, 0.085705, 0.268774, 1.773046, 6.180820, -4.607996, -0.740156, 2.677974, -0.014020, -5.367133, -0.792135, 1.014724, -2.928968, -2.636251, -0.764111, -2.006562, 0.120694, -4.609838, 1.088676, -11.941098, 5.737967, -3.500586, 1.158638, 5.277419, -0.824252, 3.719745, 1.067214, 6.999010, -3.490391, 3.380606, -3.941285, 3.037470, -3.074491, -5.476095, 0.703748, 0.590704, 1.712015, 2.907949, -3.779428, 4.336009, -3.208945, 6.012998, -0.633373, -0.858766, 0.509552, -2.374973, 2.670326, -3.898677, 3.515145, -1.504274, -2.528835, -4.196266, 1.497983, -0.251315, 1.430840, -6.129501, -0.432433, -2.067062, 0.683940, -0.651003, 2.075870, 0.301573, 0.456413, 1.512750, -2.011982, -2.651834, -3.443306, -1.217246, 2.249681, 4.056913, -0.001803, 1.856310, -1.120802, 5.100811, -4.818334, -1.832536, 0.575663, 2.821508, -2.484176, 1.090846, 3.982687, -3.890761, 1.816324, -1.156088, -1.480178, -0.946538, -0.579305, -0.901073, -0.626901, 0.268467, -3.557676, -1.586237, 1.791939, -0.206871, -0.901410, 1.202474, 2.744930, -0.571335, 3.200516, 0.411471, -0.096053, 1.948472, 1.908952, 2.884496, 1.760468, -0.835073, 3.515426, 4.381332, -0.223358, -1.809401, 0.086503, 2.059222, -1.261783, -0.836733, 0.856923, -4.898800, 1.888360, -2.207174, 0.376906, 3.295334, 5.689600, -2.268684, 1.449032, -1.591826, 1.197192, -0.958380, -1.880106, 2.903637, -1.158304, 0.286425, 1.411350, -1.044144, 4.642242, 5.629546, -1.222408, -0.653535, -0.909632, 1.304384, 0.949307, 2.316329, 2.287262, -0.217201, -0.789907, -0.414622, -3.416742, 1.308199, -4.681273, 5.369122, -4.293337, 0.764565, -2.343703, -3.794211, 1.932296, 2.057181, -1.281097, -5.943415, 1.662839, 0.180384, -4.695090, 0.702479, -3.153823, -3.411024, 0.180862, -3.886384, -1.007891, -1.704118, 0.862702, -0.511424, 3.460708, -1.867585, -7.966740, 2.017166, 7.204095, -0.777631, -2.125801, 1.848246, 2.023623, 2.431535, -0.120389, -0.336207, 2.110109, -0.700987, 4.741506, 2.285265, 2.249003, -3.817211, 3.017461, 1.867846, 4.302247, 0.304751, -0.310234, -2.985993, 4.513823, -1.711970, 0.424881, 1.872621, 8.673007, -2.507097, 0.834635, 3.987681, 0.865946, -2.122818, 2.411204, 0.166906, 4.962452, 2.623420, -0.873380, 1.751310, 3.454344, -4.624747, 0.288112, -1.057867, -7.632612, 3.383047, 2.529626, 0.388937, 0.680255, 0.010386, -0.285309, 2.821573, 3.907364, 0.626498, 6.285956, 2.837718, -4.136279, 2.172159, -0.740020, 2.603785, 0.488508, -2.008139, 1.194486, 2.840358, 1.642457, 1.124606, -2.682470, 2.333769, 0.111492, 1.035283, 1.694300, 2.123012, -2.884374, -7.135779, 4.317023, -0.086166, -1.382437, 0.327938, 0.949700, 6.212725, -1.359681, 0.690782, -1.888311, -3.168814, -4.167558, -6.230211, -4.251054, 1.301581, -2.542017, -27.238344, 0.095754, 4.638476, 3.855103, 0.669284, 0.051903, -0.231133, 5.462236, 1.029191, -1.093251, -2.784316, 2.413422, 4.149455, -6.006116, 1.867116, -1.621314, -2.286732, 1.713983, 1.969203, 2.893601, -1.096513, 2.859676, 5.299241, -1.564409, -4.103854, 3.312290, 7.535967, -10.267817, 0.146586, 0.198330, -0.458738, 5.533854, -0.592334, -1.642502, -0.867962, 0.357863, 1.499991, 0.317374, 1.710249, -3.821599, -4.164293, 0.626525, 2.896627, 4.243000, 2.967313, 1.367663, 6.382295, -6.377450, -4.389627, -0.473153, 2.991648, -1.669134, 1.115779, -0.026947, 0.960267, -5.584852, -1.439730, 0.244562, 4.521901, 2.087858, -0.596442, 1.345291, -1.198195, 1.134727, 7.460937, 3.136116, -1.059861, -3.781231, -3.531691, 1.444958, -3.786484, 4.863374, -4.276469, 5.663068, 0.108844, -4.433540, 3.566478, -0.737263, -0.857777, 3.881980, -2.940099, -0.232970, 1.024321, 0.370978, 1.810182, -1.729423, -0.953740, 0.363640, 0.101128, -0.457497, -5.657916, 2.896876, -2.673734, -0.363334, 2.990057, 1.737989, 1.253162, -2.239347, -0.736923, -1.326416, 1.898398, 0.172377, 3.468918, -1.249191, -2.190415, 4.681655, 4.550338, 8.617194, 1.506140, -4.036048, -5.532407, 3.284861, 0.496790, 0.007859, 1.394254, 0.463913, -0.189794, -3.210449, -5.304472, 1.114539, -1.199341, 0.368409, -3.468653, 0.319924, -7.521216, -0.515546, -0.846355, 3.365871, -7.414912, -1.118039, 0.057726, 0.471262, -0.671277, 5.010963, -1.456864, 0.795316, 5.710810, 5.483665, -3.680223, 3.203779, -7.778675, 1.581216, 0.493372, 1.308644, -0.168083, -1.437484, -4.021763, 3.567986, -1.012697, -1.095128, 4.802341, -1.501106, -3.554114, 1.340577, -2.658166, 0.273735, -2.363988, -3.568919, 2.061281, 1.113543, 0.817031, -1.828942, -3.359740, 5.046678, 4.811258, 1.904614, 1.466768, -1.282211, -2.547651, 4.968533, -4.411788, 4.293465, 0.164253, -1.659536, 0.885092, -0.371292, -1.131706, 5.356121, -4.582285, -6.341745, 2.657330, -1.279451, -3.364145, -4.528135, 6.154132, 0.255915, 0.995843, -6.160520, -4.956539, 0.385687, 1.730230, -4.108953, 1.728342, 1.137076, 0.444596, -3.889635, -4.341253, -2.400633, 0.832284, -3.842432, -2.179685, 1.179994, -3.570244, -3.793373, -0.992903, -6.269731, -2.026199, -2.740402, 3.908176, -0.131522, -2.611959, 5.417125, 4.601712, -0.944981, 1.923583, 3.380219, 0.557484, -4.402332, -1.821094, -1.393727, -4.922822, -0.122208, 1.627198, -2.698848, 0.543942, 7.662592, 6.865628, -3.118904, -2.938929, -1.204334, 0.880950, -2.432030, 0.659579, -5.937887, -3.082295, -2.329674, -0.929039, -6.670197, 4.624016, 1.847857, -6.151865, -1.295520, 4.561952, -4.189340, -1.271205, -3.114521, -5.400505, 2.505269, -1.440362, -0.543130, 1.583606, -3.754081, 1.425768, 4.600868, 2.031716, 4.381232, 0.756865, 1.269595, 2.452235, -3.499653, 0.843460, 5.095802, -3.224932, -1.973590, -0.918793, -2.948713, -1.726048, 0.723157, -1.867589, -3.809328, -1.288017, -1.415320, 9.147604, 4.031230, -2.515405, 6.017965, -0.808787, 6.938612, -0.940846, 0.514639, 4.288442, 0.824478, 0.728683, 1.763265, 0.545236, 1.356232, 5.527868, 4.475016, -3.437302, 3.826543, 5.086852, 0.299217, 4.161529, -0.210087, -2.249442, -2.974098, -6.259721, -1.801933, -1.427602, 3.724947, 3.003747, 0.540840, 0.380978, -0.046774, -0.128234, -6.908992, 0.317432, -0.887537, 3.312163, -0.027360, 2.139082, 2.805890, -0.598460, 2.746891, 3.195262, -0.988383, -0.554773, -1.027135, 0.100964, -3.613074, 5.187125, 0.852667, -0.236328, 6.884120, 2.533068, 6.731904, -0.778514, 0.375824, -1.817388, -5.946012, -2.352830, 5.102455, -1.938184, -1.387404, -4.757504, -2.173933, -0.201531, -3.642277, -5.362231, 3.637059, 2.751554, -1.022320, 0.236827, -3.948135, 0.148865, -1.120026, 7.616945, -0.963430, -0.756691, -0.919667, 4.724895, 0.976052, 2.488966, -3.964747, -3.973845, -0.865214, 6.001307, -2.119917, 4.599735, -0.121487, 4.304985, -1.779739, -1.172909, -0.084037, -4.013283, 2.303892, 3.205657, 3.503476, 1.411669, 1.280174, 8.759250, 0.880316, -2.128790, -4.705835, -3.441471, 1.247740, 1.331217, -2.363749, -0.643617, 1.385127, -5.501627, -7.879881, 5.820022, -1.216314, -2.178036, -1.108843, 3.346485, -3.363919, 3.609971, 2.944907, -1.553591, 0.286604, -1.090472, 3.108707, -0.752903, 3.523269, 4.811034, 5.104235, -0.589819, -1.991645, 0.047370, 0.897678, 1.814540, -2.876569, 0.355313, -0.919469, 0.483877, 4.114913, -4.182666, -1.667883, -0.516528, 3.272009, 1.569783, 1.219359, -2.658862, 1.495385, -6.089864, -2.128447, 2.838350, 0.987129, 1.348928, -2.442407, 3.521696, -0.835038, -0.587476, -0.638110, 0.922019, 2.216001, 1.777817, 1.454185, 4.067125, 1.120436, 0.289011, -10.992048, 1.672040, -0.465922, 0.592983, 0.989681, 1.752660, -0.118764, -1.185995, 4.017984, -0.896778, 1.672302, -2.030242, -4.911530, -5.827189, 1.958246, -7.221211, -2.034618, 1.940285, -1.227662, 1.219494, 4.101542, -2.650769, -2.734393, -6.333424, 2.812163, 2.962675, -4.904971, 0.171724, 1.579389, 3.322502, -0.514203, 1.357068, 5.346297, 0.754811, 2.778355, 1.658731, 1.977090, -1.235332, 5.064486, 1.987611, 6.833618, 1.541649, 2.747869, -2.104572, -1.799155, 0.295171, -4.860865, -0.765238, -1.785131, 2.577568, 0.511845, 0.550628, -2.406874, -2.713742, -2.492991, -0.175184, -0.126996, -1.339363, 0.823035, 2.378861, -2.361350, 1.065711, 2.798916, -0.116275, -5.790211, 4.252931, -0.497029, 1.009890, -1.029344, 1.774927, -6.555468, 2.879664, 0.601462, -1.747875, 1.505472, 1.982710, 2.030839, -2.763254, -1.348869, -0.266905, 4.583949, 3.799467, 3.299871, -6.750926, -5.246260, 0.655915, 5.968640, 0.782474, -1.063692, 1.318094, 3.220077, -3.783729, -0.677141, 0.595987, -2.890465, -1.360495, -0.124686, -0.284035, -2.925553, 0.262872, 2.786888, 4.160244, 0.014939, 0.569116, 1.181584, -2.572151, -1.554686, 1.284756, -4.822996, 0.997169, -3.813555, 0.617060, 2.565912, 1.037117, -0.167606, 7.223352, -2.058073, -3.086306, 4.202200, 0.180718, 3.078883, -1.246702, -2.439780, -1.529070, 2.548267, 3.340835, -6.063392, -1.775246, -0.201369, 1.530203, -0.590155, -4.917980, -2.405702, -1.290209, -0.190081, 4.806739, 4.704314, 3.928058, 2.274630, -0.550331, 0.822520, 1.818900, 1.580151, -3.903281, -0.857364, -3.976201, -0.785557, 1.069647, -5.818671, 2.766473, 1.259937, 2.320919, -1.525085, 2.384599, 4.469339, 0.205931, -0.547983, 2.354457, -0.327334, -6.448855, -0.907820, 5.117530, -0.255594, -0.240747, -1.073850, -1.768409, -5.428776, 0.481095, -3.812433, -1.717587, -3.733663, 2.083388, 10.582418, -4.710001, 0.563532, -7.536781, -6.452072, 2.862794, -1.463039, 0.230586, -1.815489, 1.481611, 1.544786, 1.551767, -0.059472, 0.789418, -0.825706, 1.940825, 5.768491, -0.494084, -2.538201, -1.392608, -1.908880, -1.700553, 5.082959, 10.069242, 0.678830, -0.559667, 2.036356, 1.434546, -2.558431, 2.288113, 0.170179, 2.484860, -2.987775, -0.382073, -0.226642, -2.446935, -3.155464, -3.223151, 8.751858, 0.507641, -2.614630, 1.343406, -0.892121, -2.539207, -4.111343, 1.182877, -4.679571, -2.524526, -5.096138, -1.301956, 2.281354, -2.029088, 0.427091, 3.804364, 5.483494, -40.825272, -1.143677, -2.260099, -0.089059, -1.493274, -1.944442, -2.385689, 1.698596, -1.873040, -0.443924, 2.621414, 1.287797, -0.057905, -2.352337, 1.238259, 1.387846, 1.417577, 0.688409, 4.645106, 2.341688, 0.465816, -7.549288, -5.974134, 6.261028, -5.144201, -2.522077, -0.662837, 4.664794, -0.539391, -3.439147, 4.087629, -1.963182, 2.466938, 2.107603, -1.249598, -6.342202, -3.447833, -3.704789, -1.074862, 0.765216, -1.964713, -5.504405, 1.663442, -2.911618, -0.616493, -2.083917, 2.209042, 0.487668, 1.796368, -1.596250, 4.042785, 2.681681, 2.703725, -1.306239, 2.601149, -2.099759, -5.318709, -1.065740, 0.589329, -3.567276, 0.062211, -0.425451, -3.580077, -1.296509, -5.680976, -0.678171, -0.762381, 0.299684, 0.240645, -1.079323, -2.957309, 1.849129, 0.099766, 4.726409, -2.363460, -4.875949, 1.315865, 1.462351, -4.889824, 1.677129, -0.667212, -5.138101, 5.330096, -2.835877, -3.902706, 3.045518, -0.134772, 6.075926, 4.500954, 5.112908, -0.905090, -1.909402, 0.904969, 0.690467, -0.619620, 2.666605, 0.904155, -4.029288, 0.402307, -3.036969, -1.037772, -1.147319, -1.899678, 6.248024, 1.974826, 5.616739, 8.500142, 2.245967, -3.625181, -2.428448, -1.810710, 0.331268, -3.135995, 1.302269, 1.225240, 3.909755, -0.403967, -3.433037, -1.311207, -2.177593, -18.017111, 1.118036, -0.756810, -6.615274, 1.566789, -2.418086, 2.953141, -2.826252, -0.735017, 1.528646, -3.723853, -1.885919, -4.900423, -2.478693, -3.619526, -1.294044, 1.890493, -1.871122, 3.827642, 2.343641, -2.662672, 3.404760, -0.482313, 5.578103, -2.134901, 5.369441, 2.492493, 2.104894, -5.205935, -1.008856, 3.011516, -0.641482, 2.548511, -8.027479, 0.661283, -7.434981, 1.621635, 2.033537, -1.930799, -1.076036, -0.960546, -0.117069, -3.372064, -2.063029, 1.654161, 1.716305, 4.944497, -0.019142, -4.051437, -3.766835, 5.842964, -3.690862, -3.004082, -4.533800, -1.790580, -2.050976, 0.561251, 1.564617, -2.369565, 2.445515, 0.568967, 0.080670, 1.160943, -1.780555, -0.186662, 0.478309, -4.549850, -17.848110, -4.087846, -0.273795, 5.222282, 5.777647, 0.494441, -1.833347, 0.257310, 1.124015, -1.849150, -1.647193, -1.132576, -2.164145, 4.289577, 0.453893, 1.341683, 0.748200, -2.471226, 5.312476, 0.836187, 1.723641, 0.498636, 1.428005, 1.828248, -3.398380, -2.883598, -5.426562, 2.745332, 1.425643, 0.006252, -3.581231, -0.262443, -0.622419, -1.944646, -3.869738, 4.476512, -1.191560, 4.310348, -1.112125, 3.697780, -1.439761, -4.078893, 2.179358, 2.530555, -0.148791, 2.168117, 0.114731, 1.831150, 1.157180, 2.166352, 2.055449, -2.871636, 4.420805, -1.390404, -6.722301, -1.769241, 1.519223, -2.692869, 1.015409, 7.691347, 0.583279, -4.058763, -1.453730, 2.806040, 0.556832, 2.730164, -2.572670, 2.370497, 1.912015, 3.515903, 1.919521, 0.369309, -2.022592, -0.218010, -6.205793, -5.902825, 4.063861, -2.432916, 33.217953, -0.551205, 3.915136, -3.102407, 3.210474, -1.160345, -2.887495, 1.523478, -1.018772, -2.422345, 1.810487, -5.200218, 1.671701, -0.721633, 4.423970, -1.442638, -5.779491, -0.397599, -3.750891, 2.698724, -5.718669, -2.086298, -7.792905, 3.388355, -3.770853, -0.215906, 2.560900, -2.680165, -2.986103, -6.097780, 4.229813, -2.723863, -3.065816, 2.248178, -0.651940, -1.214029, 2.257283, 2.486542, 2.257068, 4.830973, -2.277972, -1.946075, 3.999813, 3.456074, 0.536131, 1.805786, 0.709500, -4.097561, -5.412663, 4.285580, -4.488030, 1.119286, 2.302969, 5.947579, -0.113177, -1.724484, -0.104090, 1.840069, 4.592496, 1.569632, -2.483522, -0.311809, -0.194584, -2.869655, -2.849751, 4.997175, 2.512164, 1.758786, 0.490910, 4.247337, 0.569464, -0.659693, -0.454139, 4.440271, 1.410414, -0.773389, -2.376852, -6.240281, -4.227293, -1.671386, 0.603003, 2.479946, -2.180943, 3.127448, 1.668763, 0.149739, -0.781542, 7.098460, 2.111330, -4.436161, -2.082523, -2.478790, 49.008598, 1.723189, 5.866392, 3.153973, -0.181730, -7.328019, -1.992281, -2.027254, 2.432771, 0.979790, -0.092340, 1.070737, 0.099132, 6.923749, -4.058023, -8.293395, -1.186165, -3.612353, 0.829185, -0.439513, -1.879160, -1.854795, -5.092451, -1.970279, -4.055690, -7.000295, -4.616829, -0.039388, 4.131409, 1.653316, -7.131061, -5.091387, 3.990695, 3.182390, -1.955781, 0.335848, 1.516590, -0.282827, 2.731199, -1.598010, 5.144428, 2.064570, 2.552294, 1.812606, -2.513990, 2.353807, 7.561320, -0.036158, -0.385612, -4.383836, -3.971509, 0.043287, 4.296186, 0.464179, -1.848343, -5.514888, -8.349360, -8.976414, 0.569201, -4.740505, -1.152213, 5.905676, -5.700530, 3.814966, 2.234695, -0.222299, 4.771212, -3.930703, 2.960777, 0.165897, -1.333587, 2.028787, -0.778738, -1.235554, -1.246747, -0.041303, 3.006766, 3.820721, -5.493714, -0.189577, -3.010185, -2.883236, 0.024274, -1.494312, -2.523447, -4.104508, -2.666597, 3.281853, 3.544635, 3.610449, -0.587052, -0.098227, 0.073750, 0.127269, -0.344085, 0.770601, -2.514192, -4.431854, -1.949836, -1.391094, -2.886564, 0.702709, -0.731393, -2.069504, 1.040801, 3.144605, -0.115405, 2.797428, -1.210172, 3.943987, 2.697510, 1.059327, 0.742219, 4.559521, 1.323614, -0.211233, -0.215014, 1.990037, 1.956536, -2.729569, -3.183045, 2.447714, 2.988651, 5.736022, -3.185564, -2.579111, 3.323712, 4.193132, 0.727872, 5.551432, -0.744345, -2.344970, -3.441572, 5.568137, -4.830486, 3.880134, -5.891489, -2.210011, 0.171487, -1.457955, -2.245640, 2.203776, 0.875658, 1.959566, 8.602451, 4.474270, 2.726350, -0.354296, -2.258420, -2.513044, 4.931829, 2.946697, 0.520752, 4.333879, -0.821970, 2.029501, -2.277242, -4.952377, -1.538025, -1.064874, 3.308603, 1.247338, 2.504045, -3.224561, -2.891461, 3.316285, 7.715738, -1.605185, 3.243022, -5.836881, 2.714357, -0.388968, -1.375407, -3.556258, -2.474644, 1.088091, -3.216538, -2.776881, -8.227687, 5.125639, 5.283110, 6.973928, 37.955891, 3.139960, -5.763666, 1.350248, -1.406692, -3.986167, 2.335608, 3.157670, 4.455172, 2.308789, 2.309107, -3.130976, -4.247937, -2.270962, 1.574682, 5.501863, 4.485770, -10.403132, 0.773148, -1.224575, 5.244150, -3.847297, -1.800287, 6.947053, -2.622383, 4.851179, 3.233861, 4.995902, -4.162736, -1.983544, 3.995517, 6.687618, -3.040680, -6.543100, 5.237826, -4.066863, -1.226237, 1.639323, -4.240799, -0.423647, -2.178972, 1.593127, -3.968011, 5.461793, 2.277310, -2.744098, -3.358952, 4.035235, -3.137929, 2.964127, 7.214087, -1.364202, -4.651634, 6.503850, 2.740077, 0.886651, 2.558752, 0.043657, -5.242559, -0.792984, -2.882527, -5.403186, 3.155617, 3.991355, -0.568835, 1.141021, -3.702596, -0.279778, -0.912082, -3.422060, -1.879990, -0.733728, 2.278115, -2.372427, -2.681229, -1.556143, -0.770521, 0.899023, 4.508717, -3.010523, -1.908418, -1.475849, 3.509949, 3.582316, -0.681324, -7.056856, 1.278887, 0.770643, -0.584454, 2.825036, -0.821392, 1.240007, 7.577941, 0.096160, -3.572768, 1.793741, 3.848424, 0.995781, 0.054073, -0.235984, -0.301966, 2.916689, 1.391444, 1.480205, -3.852633, -1.223310, -3.153660, 3.194536, 1.030388, -4.751256, 1.885638, -2.114385, -6.170693, 1.415366, 4.334911, -4.597432, -0.846934, -3.220494, 7.086485, 3.530614, -2.059356, 1.625898, -2.086513, -1.870273, 3.097336, -2.899379, 4.410741, -2.872575, 0.711264, 5.131933, 1.355852, -2.781525, 3.417987, -2.958153, -0.075279, 1.298355, -0.051176, 2.940977, 0.224740, 4.669080, -3.098835, -1.002103, -2.227010, -0.529945, 1.581029, -1.380316, -0.444712, 1.033249, -2.976829, 1.184253, -3.129241, -4.516237, 1.861109, 0.749785, -0.883805, 2.467990, -1.366948, -2.021918, 9.678317, 0.780995, 5.006135, 2.355187, 4.709684, 0.273654, 0.848736, -1.546677, 0.190088, -1.301066, -1.275526, -0.592527, 2.147533, 4.496069, 2.915365, -2.232816, -3.967589, 2.198257, 3.257962, -1.514701, -1.426605, -0.670297, 0.013288, -1.436101, -3.523503, 1.392464, -7.692491, 4.621131, 4.215765, -6.433372, 8.560532, -5.166466, 1.641891, 0.939057, -1.161749, -0.620703, 0.268485, 0.818933, -1.119831, -2.502043, 1.727721, -2.059391, 2.365984, -3.621562, 0.293849, 2.106696, 2.435274, -0.056334, 1.744176, -0.872306, -3.566818, -3.476950, -13.363617, -2.873412, -0.464867, 0.363577, 2.328689, -3.345537, -8.831505, -0.320157, 0.871519, -4.472153, -0.401718, -0.096133, 0.386455, -1.806300, 1.170147, 1.817253, -3.103240, -0.165836, 3.598591, 0.253634, 2.874893, 3.903717, 1.369801, 5.628721, 2.572816, 2.252870, -1.170226, -1.062752, -0.456143, -3.078095, -6.341308, 4.122602, 2.681516, 0.093924, 3.844733, 3.950950, -3.356711, -0.854817, 3.384428, 13.341409, -4.234371, 2.496033, -1.629354, -1.507377, 0.173900, 2.698258, 0.267310, 1.372167, 0.510639, 2.101537, 1.261789, 2.597848, 2.449376, -0.840756, -1.046823, -0.002541, -2.707355, 0.755944, 1.257627, 1.465595, 0.194379, 7.094273, 0.264183, 5.946443, -6.767539, 3.796284, -2.374620, -2.461099, -1.391973, -3.764230, -2.892979, 1.555705, -1.332523, -1.799763, -2.815821, -4.461638, 0.016501, -0.694294, -0.483337, 1.215167, 0.758542, 4.294414, -2.490471, -2.232953, 0.058542, -0.601252, 4.924240, 3.834749, 2.027948, 1.471411, 0.901828, 0.966709, 0.390534, 7.367773, 1.259584, -0.460943, -6.608699, -4.643585, 2.040176, -4.100492, -1.690598, 3.704851, -2.805748, 1.226204, 6.179536, -1.176640, 3.004384, 3.111051, 3.486775, -4.034345, 4.785528, 7.299822, -0.907930, -5.061491, -3.685342, -1.961922, -3.682397, -0.804982, -1.361447, -0.922343, -4.764962, -0.093592, 6.292956, 9.184029, -1.652289, -0.349144, 2.227105, 2.586640, -1.344222, -0.799372, -1.067394, 1.337438, -4.368445, -3.637234, 0.453892, -2.348681, 3.619864, 0.782791, -0.751199, -2.257583, -0.637428, -2.538015, -4.910626, 0.804029, 1.904546, 2.071048, -4.913878, 7.321102, 2.329751, -4.918932, 9.630699, 4.401609, 4.836075, -37.672218, -0.989470, -3.400167, -1.611549, -4.914345, -2.178263, -0.507991, 7.461368, 1.499078, 9.014656, 0.385251, -5.354575, -0.235111, 1.188299, 2.999619, 4.158133, -2.458529, 0.836758, -3.781934, 2.581318, -0.546479, -0.767487, -1.108148, 1.394199, -6.728708, -0.049908, 2.439957, 2.281411, -0.256135, 2.625314, 0.526615, 5.245579, -0.693234, -4.652315, -6.053971, 0.761856, 1.380219, 2.186297, 1.574326, 3.023049, -3.008269, -5.497355, 1.208180, 1.866580, 0.383009, -5.021918, -1.859053, -0.577350, 0.791435, 1.862842, -2.977753, -2.982435, -3.968694, -2.911249, -0.671418, -0.664998, -4.328650, 0.777280, -3.322049, -1.996393, -10.946344, 1.487512, 3.325927, 2.512698, -4.560525, 2.487353, -3.221335, 2.498028, 1.558956, -7.027573, -4.410831, 1.153316, -2.383291, 2.752918, -2.445125, 1.669130, 2.055993, 3.435565, -2.200916, 7.625603, -0.429790, -7.823750, 0.715926, 1.946298, 0.931786, -0.959709, -2.558171, 1.349378, 2.164468, -1.039176, -0.840470, -1.273818, -1.788402, -0.181308, 4.731328, -1.308624, -4.392232, -3.329735, 3.710122, -0.471847, 0.321802, -2.903600, 2.763454, 1.606396, 5.018901, -4.307723, -5.778127, 1.896904, -1.911034, -0.877533, -0.970224, -6.048330, -1.055960, -0.004360, 3.615595, 0.670895, -3.405262, -6.979682, 1.391201, -2.248485, -0.812938, -3.361127, -0.245261, -2.004265, 2.180632, -2.492358, 0.106926, -2.312480, -0.849407, -3.182122, -3.868700, 1.483516, -2.800287, 1.606942, 5.240799, -4.588236, 3.825581, 1.475691, 5.782595, 0.572563, 0.615392, -3.263512, 2.357608, 2.504305, -0.131105, -4.494020, -3.431349, 1.323660, -1.233401, 5.227014, 1.833779, 4.529938, 0.172702, 2.495649, 1.844747, -0.571699, 6.221216, 3.510139, 2.034399, -2.708287, 2.504865, 0.827564, -0.389984, 0.973059, -1.210372, 3.136084, -0.528568, -2.895905, -2.945923, -3.354907, -6.284786, 2.405454, -3.611116, -1.510917, -0.275045, -2.606582, 0.023821, -0.977702, 4.496226, 3.893765, -5.102979, 4.505174, -2.151086, -2.598680, 3.437677, -1.525504, -3.606384, 2.925199, -1.819598, -1.297506, -3.938237, 0.758268, -6.536557, 2.823950, -1.268384, -4.748070, -1.569543, -4.040869, -4.199319, -1.537972, -2.470041, -2.424089, 4.282330, 2.027144, -2.279008, 1.956247, 0.243131, -1.307445, 7.631332, 3.666424, 3.244223, 0.806356, 1.559336, -6.454810, -2.210939, 6.478909, 8.370586, -4.877525, 1.632561, 0.528654, 7.175992, 3.294758, -2.347389, -0.656673, 2.440072, 0.033162, -3.974950, 3.992282, 1.041370, -0.452447, 1.250809, 5.725498, 0.952154, -1.769804, -6.182436, -1.491944, 0.978641, -1.848162, -3.526021, 2.130781, -1.247035, 0.430292, 2.094477, 1.015652, -0.982822, -3.514047, 1.362636, -1.385784, 7.721636, 0.452686, -0.514129, -0.185515, -4.155198, -2.091428, -0.342165, 3.609978, 2.325354, -2.549308, -1.935902, -4.843817, 4.849944, 0.559908, 0.317363, -6.979578, 4.078461, -1.781615, -0.168588, -3.608682, 0.540197, -3.224210, -2.160827, -2.483818, -1.395126, 2.122100, 0.045637, -5.244937, 1.460025, -1.313215, -1.831590, -3.608512, 2.137432, 1.063356, -4.977770, -4.077838, -0.227993, -2.521503, -2.196034, 2.424290, 3.695994, -0.664042, 3.369231, -0.868323, -1.362390, 2.049519, 1.623333, -3.785452, -0.366721, 0.929944, 1.189942, 2.357296, -2.012867, 4.039164, 5.018045, 1.173269, 1.037018, 1.239052, -1.506088, 1.858684, -8.143816, -8.377998, -4.266102, 2.238616, 0.990164, -5.564924, 4.017815, 0.013090, -2.655519, -2.138789, 0.309147, 0.815738, -3.671095, -1.768397, 0.642272, -4.239707, -1.738047, -3.186784, 0.928931, -3.288824, 2.778164, 2.127247, 1.010160, -0.380883, -1.794696, -0.380974, 2.568671, 2.952252, -0.378777, 3.388682, 1.112046, 3.843087, -4.014625, -2.586132, -1.373591, 0.875761, -2.153665, -4.098034, 4.174786, -3.971316, 3.831267, 2.632764, -2.082423, 2.323014, 0.126692, -1.658272, 3.813766, 0.731241, 5.020086, -1.314192, -0.455286, 2.529340, 1.737364, -2.307101, 3.249065, 3.359480, 2.342399, 2.964352, 0.450493, 6.821503, -0.876081, 0.104224, -4.359997, 6.337316, 1.961707, 1.584909, 5.782849, -7.702838, 0.486187, -2.076824, 0.556629, -0.377218, 1.867574, -2.546370, 3.420962, 3.743823, 0.830919, -3.154605, 2.442055, -0.077978, -2.174074, 7.835155, 0.042525, 5.061534, -1.297285, 0.468504, -0.896670, -2.810953, -2.835888, -4.904580, 1.410659, 6.832096, -1.246026, -2.643165, 2.461378, 1.465698, -2.603248, 3.614096, 0.604979, -0.203376, 0.620558, 2.010493, -3.164634, 2.355133, 0.910522, -9.153761, -6.733269, 0.191007, -2.222224, -3.317362, 1.621990, -3.271222, -3.361245, -2.236486, -3.204127, -0.102103, -6.418847, 0.475259, 0.960290, 0.809332, 2.043769, 0.233913, 1.443959, 3.518958, 0.381731, 3.579408, -1.736145, 0.935460, 0.490841, -1.445789, -2.606658, -0.390318, 1.241003, 1.852338, 3.745817, -3.404975, -2.927765, -1.375834, 4.025413, -2.450278, 0.526100, 0.871327, -8.299149, -3.752483, 8.046568, -3.801634, -2.328504, -2.873852, 12.037412, 5.806139, 0.151948, 1.294600, 0.820967, 1.635987, 1.551868, -16.845692, -0.875519, -2.038996, 4.211426, -4.056794, 5.402084, -3.281292, 7.430756, 1.138992, -3.348103, -0.064485, 2.009815, 3.773692, 3.913712, 4.532695, 1.372892, -1.063987, 1.025051, 2.728509, 9.491262, 4.673415, 2.517928, 0.824100, 1.517722, -2.984458, 0.270283, -1.110453, -2.754004, -3.045094, 1.386551, -6.637009, -6.467078, 5.209693, 0.221588, 1.345757, -1.233974, 1.406563, -11.624565, -2.558307, 3.355196, -1.574055, 11.278919, -4.175966, -3.528599, -5.955521, 5.521868, -6.089912, 1.871637, 0.704803, -0.803571, -3.412199, 6.814842, -2.581755, 1.461685, -1.340520, 1.030475, 0.972731, -2.055774, 1.705598, 3.048807, -1.037460, 2.906647, -0.434317, -0.054622, -3.738979, -4.256308, -4.723948, -0.522000, 4.060871, -4.338176, -24.310600, 3.466682, -0.756065, -4.103305, 1.243985, -2.868507, 0.153364, -0.084377, 3.205434, -1.317540, -0.319762, -0.802727, 2.456871, 3.382993, 4.963143, -0.354739, -2.737841, 2.374343, -1.887784, 1.198460, -7.294523, -2.117028, 46.393829, 2.054618, -1.339482, 2.647988, 1.026573, -0.775336, -1.399183, -6.095242, 2.313640, -2.377140, -2.874966, 1.462420, 1.763429, -4.175523, 0.322729, 0.076063, -0.492210, 5.528355, 2.494064, 3.575900, -4.731384, 1.914505, 0.780838, -0.513604, 0.834317, 4.637135, 0.748915, -2.242534, 5.284515, 0.941967, -4.771373, 12.168940, -2.509539, 2.931109, 6.130014, 2.304723, -0.401287, 1.846620, 2.191286, 0.522786, -2.212207, -2.330939, 3.671052, -3.602032, -8.045081, 1.841934, -15.802244, -4.091495, 2.758163, 3.801088, -0.116542, 4.200107, 3.809458, 2.594901, 8.304158, 2.272768, -3.162868, -4.738754, -1.238095, -2.981887, -0.739677, -8.554259, -1.512607, -3.465696, -0.641592, -5.622955, 0.712421, -0.786531, -3.055792, -1.708393, -1.229299, -2.440192, -2.387127, -1.278995, 0.990231, 0.675825, 7.402359, 2.521222, 0.413226, -1.471801, -0.305300, -2.968605, -0.631904, 2.190862, 1.293332, 0.988720, 1.396958, 1.053343, 0.096911, 5.328904] } From a45231af47035e89031625d47210663881a4e2cf Mon Sep 17 00:00:00 2001 From: Min Yoo Date: Sun, 25 May 2025 05:18:32 +0900 Subject: [PATCH 14/54] readme: Add macLlama to community integrations (#10790) This commit updates the README to include macLlama within the community integrations section. macLlama is a native macOS application built for lightweight and efficient LLM interaction. Key features include: * **Lightweight & Native:** Designed to be resource-friendly and perform optimally on macOS. * **Chat-like Interface:** Provides a user-friendly, conversational interface. * **Multiple Window Support:** Allows users to manage multiple conversations simultaneously. The primary goal of macLlama is to offer a simple and easy-to-run LLM experience on macOS. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 6a4815c1e..00de95a76 100644 --- a/README.md +++ b/README.md @@ -406,6 +406,7 @@ See the [API documentation](./docs/api.md) for all endpoints. - [AppFlowy](https://github.com/AppFlowy-IO/AppFlowy) (AI collaborative workspace with Ollama, cross-platform and self-hostable) - [Lumina](https://github.com/cushydigit/lumina.git) (A lightweight, minimal React.js frontend for interacting with Ollama servers) - [Tiny Notepad](https://pypi.org/project/tiny-notepad) (A lightweight, notepad-like interface to chat with ollama available on PyPI) +- [macLlama (macOS native)](https://github.com/hellotunamayo/macLlama) (A native macOS GUI application for interacting with Ollama models, featuring a chat interface.) ### Cloud From 012cf65340ccd44de76017c56eb553b0803fde95 Mon Sep 17 00:00:00 2001 From: RAPID ARCHITECT <126218667+rapidarchitect@users.noreply.github.com> Date: Mon, 26 May 2025 14:05:03 -0500 Subject: [PATCH 15/54] readme: add AWS Strands Agents SDK example to community integrations (#10865) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 00de95a76..d7cf5bfec 100644 --- a/README.md +++ b/README.md @@ -450,6 +450,7 @@ See the [API documentation](./docs/api.md) for all endpoints. - [orbiton](https://github.com/xyproto/orbiton) Configuration-free text editor and IDE with support for tab completion with Ollama. - [orca-cli](https://github.com/molbal/orca-cli) Ollama Registry CLI Application - Browse, pull, and download models from Ollama Registry in your terminal. - [GGUF-to-Ollama](https://github.com/jonathanhecl/gguf-to-ollama) - Importing GGUF to Ollama made easy (multiplatform) +- [AWS-Strands-With-Ollama](https://github.com/rapidarchitect/ollama_strands) - AWS Strands Agents with Ollama Examples ### Apple Vision Pro From aea6fb9b5862932ea4622e854be9ddae8c320658 Mon Sep 17 00:00:00 2001 From: Parth Sareen Date: Mon, 26 May 2025 17:16:00 -0700 Subject: [PATCH 16/54] tools: remove newline stripping (#10869) --- tools/tools.go | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/tools/tools.go b/tools/tools.go index 509ca90ac..529bd3be3 100644 --- a/tools/tools.go +++ b/tools/tools.go @@ -137,11 +137,6 @@ func parseJSONToolCalls(s string, name, arguments string, prefix string) ([]api. // - The processed string with prefix removed if found // - error: ErrAccumulateMore if prefix is incomplete, or nil if successful func (p *Parser) checkPrefix(s string) (string, error) { - original := s - if strings.ContainsRune(s, '\n') { - s = strings.ReplaceAll(s, "\n", " ") - } - if s == "" || p.prefix == "" { return s, nil } @@ -158,7 +153,7 @@ func (p *Parser) checkPrefix(s string) (string, error) { // Return everything except overlapping portion p.sb.Reset() p.sb.WriteString(s[idx:]) - return original[:idx], errAccumulateMore + return s[:idx], errAccumulateMore } // Check if prefix appears in middle of string @@ -167,7 +162,7 @@ func (p *Parser) checkPrefix(s string) (string, error) { p.sb.Reset() p.sb.WriteString(strings.TrimSpace(s[idx:])) // Return everything before prefix - return original[:idx], errAccumulateMore + return s[:idx], errAccumulateMore } // No partial prefix found @@ -181,9 +176,6 @@ func (p *Parser) checkPrefix(s string) (string, error) { // - tools: Any parsed tool calls // - content: Non-tool call content func (p *Parser) Add(s string) (tools []api.ToolCall, content string) { - if strings.TrimSpace(s) == "" { - return nil, s - } if p.done { if p.index == 0 { // Return original string if no tool calls found at start From 066d0f474671fd38532f4a245533158312a68e75 Mon Sep 17 00:00:00 2001 From: Parth Sareen Date: Mon, 26 May 2025 18:59:06 -0700 Subject: [PATCH 17/54] tools: relax JSON parse constraints for tool calling (#10872) --- tools/tools.go | 46 ++++++++++++++++++--------------------------- tools/tools_test.go | 45 ++++++++++++++++++++++++++++++++++++-------- 2 files changed, 55 insertions(+), 36 deletions(-) diff --git a/tools/tools.go b/tools/tools.go index 529bd3be3..914a5eaf0 100644 --- a/tools/tools.go +++ b/tools/tools.go @@ -17,15 +17,14 @@ var ( ) type Parser struct { - parseLeadingJSON bool - prefix string - prefixFound bool - tmpl gotmpl.Template - sb strings.Builder - index int - name string - arguments string - done bool + greedyParseJSON bool + prefix string + prefixFound bool + tmpl gotmpl.Template + sb strings.Builder + index int + name string + arguments string } // parseJSONToolCalls attempts to parse a JSON string into a slice of ToolCalls. @@ -176,14 +175,6 @@ func (p *Parser) checkPrefix(s string) (string, error) { // - tools: Any parsed tool calls // - content: Non-tool call content func (p *Parser) Add(s string) (tools []api.ToolCall, content string) { - if p.done { - if p.index == 0 { - // Return original string if no tool calls found at start - return nil, s - } - // Return empty if no tool calls found after start - return nil, "" - } p.sb.WriteString(s) s = p.sb.String() @@ -195,7 +186,7 @@ func (p *Parser) Add(s string) (tools []api.ToolCall, content string) { } // Exit if prefix exists in template, greedy parsing is off, and prefix not found - if !p.parseLeadingJSON && !p.prefixFound { + if !p.greedyParseJSON && !p.prefixFound { p.sb.Reset() return nil, s } @@ -206,10 +197,9 @@ func (p *Parser) Add(s string) (tools []api.ToolCall, content string) { return nil, "" } p.sb.Reset() - // Do not try parsing leading JSON if JSON not found - p.parseLeadingJSON = false - if p.prefix == "" { - p.done = true + // Only do greedy JSON parsing if there is no prefix from template + if p.prefix != "" { + p.greedyParseJSON = false } if p.index != 0 && p.prefix == "" { return nil, "" @@ -253,11 +243,11 @@ func NewParser(templateToProcess *gotmpl.Template) (*Parser, error) { } return &Parser{ - tmpl: *tt, - sb: strings.Builder{}, - prefix: tp, - parseLeadingJSON: true, - name: name, - arguments: arguments, + tmpl: *tt, + sb: strings.Builder{}, + prefix: tp, + greedyParseJSON: true, + name: name, + arguments: arguments, }, nil } diff --git a/tools/tools_test.go b/tools/tools_test.go index 1ae3bff89..5fee8f57d 100644 --- a/tools/tools_test.go +++ b/tools/tools_test.go @@ -536,11 +536,18 @@ func TestParseToolCalls(t *testing.T) { expectedTokens: "", }, { - name: "model without prefix in template, prefix in output", + name: "model without prefix in template, prefix in output, multiple tool calls in list", model: "llama3.2", output: ` [{"name": "get_current_weather", "parameters": {"format":"fahrenheit","location":"San Francisco, CA"}} {"name": "get_current_weather", "parameters": {"format":"celsius","location":"Toronto, Canada"}}]`, - expectedToolCall: []api.ToolCall{}, - expectedTokens: ` [{"name": "get_current_weather", "parameters": {"format":"fahrenheit","location":"San Francisco, CA"}} {"name": "get_current_weather", "parameters": {"format":"celsius","location":"Toronto, Canada"}}]`, + expectedToolCall: []api.ToolCall{t1, t2}, + expectedTokens: ``, + }, + { + name: "model without prefix in template, prefix in output, individual tool calls", + model: "llama3.2", + output: ` {"name": "get_current_weather", "parameters": {"format":"fahrenheit","location":"San Francisco, CA"}} {"name": "get_current_weather", "parameters": {"format":"celsius","location":"Toronto, Canada"}}`, + expectedToolCall: []api.ToolCall{t1, t2}, + expectedTokens: ``, }, { name: "model with prefix in template, no prefix in output, tokens before", @@ -567,15 +574,37 @@ func TestParseToolCalls(t *testing.T) { name: "model without prefix in template, no prefix in output, tokens before", model: "llama3.2", output: `some tokens before [{"name": "get_current_weather", "parameters": {"format":"fahrenheit","location":"San Francisco, CA"}} {"name": "get_current_weather", "parameters": {"format":"celsius","location":"Toronto, Canada"}}]`, - expectedToolCall: []api.ToolCall{}, - expectedTokens: `some tokens before [{"name": "get_current_weather", "parameters": {"format":"fahrenheit","location":"San Francisco, CA"}} {"name": "get_current_weather", "parameters": {"format":"celsius","location":"Toronto, Canada"}}]`, + expectedToolCall: []api.ToolCall{t1, t2}, + expectedTokens: `some tokens before`, }, { - name: "model without prefix in template, prefix in output, tokens after", + name: "model without prefix in template, prefix in output, tokens after", + model: "llama3.2", + output: ` + [{"name": "get_current_weather", "parameters": {"format":"fahrenheit","location":"San Francisco, CA"}} {"name": "get_current_weather", "parameters": {"format":"celsius","location":"Toronto, Canada"}}] some tokens after`, + expectedToolCall: []api.ToolCall{t1, t2}, + expectedTokens: ``, + }, + { + name: "model without without prefix, match all jsons", model: "llama3.2", - output: ` [{"name": "get_current_weather", "parameters": {"format":"fahrenheit","location":"San Francisco, CA"}} {"name": "get_current_weather", "parameters": {"format":"celsius","location":"Toronto, Canada"}}] some tokens after`, + output: `model outputs some text [{"name": "get_current_weather", "parameters": {"format":"fahrenheit","location":"San Francisco, CA"}} {"name": "get_current_weather", "parameters": {"format":"celsius","location":"Toronto, Canada"}}] some tokens after`, + expectedToolCall: []api.ToolCall{t1, t2}, + expectedTokens: "model outputs some text", + }, + { + name: "model flushes tokens if tool call doesn't match", + model: "llama3.2", + output: `{ "user": {"id": 12345, "name": "Alice", "preferences": {"theme": "dark", "notifications": true}, "stats": {"points": 987, "level": 42}}}`, expectedToolCall: []api.ToolCall{}, - expectedTokens: ` [{"name": "get_current_weather", "parameters": {"format":"fahrenheit","location":"San Francisco, CA"}} {"name": "get_current_weather", "parameters": {"format":"celsius","location":"Toronto, Canada"}}] some tokens after`, + expectedTokens: `{ "user": {"id": 12345, "name": "Alice", "preferences": {"theme": "dark", "notifications": true}, "stats": {"points": 987, "level": 42}}}`, + }, + { + name: "model flushes tokens if tool call doesn't match array", + model: "llama3.2", + output: `[ { "user": {"id": 12345, "name": "Alice", "preferences": {"theme": "dark", "notifications": true}, "stats": {"points": 987, "level": 42}}}]`, + expectedToolCall: []api.ToolCall{}, + expectedTokens: `[ { "user": {"id": 12345, "name": "Alice", "preferences": {"theme": "dark", "notifications": true}, "stats": {"points": 987, "level": 42}}}]`, }, } From 9239a254e054d24b0de3358ba8c4bd9b50730bfd Mon Sep 17 00:00:00 2001 From: Kyle Steere Date: Tue, 27 May 2025 18:28:48 +0000 Subject: [PATCH 18/54] server: abort download on empty digest Signed-off-by: Kyle Steere --- server/download.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/download.go b/server/download.go index 6f79fd2d3..784ba2d5e 100644 --- a/server/download.go +++ b/server/download.go @@ -464,6 +464,10 @@ type downloadOpts struct { // downloadBlob downloads a blob from the registry and stores it in the blobs directory func downloadBlob(ctx context.Context, opts downloadOpts) (cacheHit bool, _ error) { + if opts.digest == "" { + return false, fmt.Errorf(("%s: %s"), opts.mp.GetNamespaceRepository(), "digest is is empty") + } + fp, err := GetBlobsPath(opts.digest) if err != nil { return false, err From ea79003180205680000bacf97466fc9d78d71f5e Mon Sep 17 00:00:00 2001 From: Jesse Gross Date: Tue, 27 May 2025 13:33:57 -0700 Subject: [PATCH 19/54] kvcache: Skip computing causal mask for worst case graph reservation Computing an attention mask for a large context and max batch is expensive - over 100ms. Models like Gemma3 that have multiple types of caches and custom attention masks need to do this 4 times, so this adds approximately 500ms to startup time when using 128k context When we are reserving the worst case graph, we don't need the mask, only its shape, so we can skip this. --- kvcache/causal.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/kvcache/causal.go b/kvcache/causal.go index f6bacaaf8..b594d0b41 100644 --- a/kvcache/causal.go +++ b/kvcache/causal.go @@ -30,6 +30,11 @@ type Causal struct { // ** current forward pass ** + // curReserve indicates that this forward pass is only for + // memory reservation and we should not update our metadata + // based on it. + curReserve bool + // the active layer for Get and Put curLayer int @@ -159,12 +164,13 @@ func (c *Causal) Close() { } func (c *Causal) StartForward(ctx ml.Context, batch input.Batch, reserve bool) error { + c.curReserve = reserve c.curBatchSize = len(batch.Positions) c.curSequences = batch.Sequences c.curPositions = batch.Positions c.opts.Except = nil - if !reserve { + if !c.curReserve { c.updateSlidingWindow() var err error @@ -304,6 +310,11 @@ func (c *Causal) buildMask(ctx ml.Context) ml.Tensor { c.curCellRange.max = roundUp(c.curCellRange.max+1, c.config.CachePadding) - 1 length := c.curCellRange.max - c.curCellRange.min + 1 + + if c.curReserve { + return ctx.Input().Empty(c.config.MaskDType, length, batchSize) + } + mask := make([]float32, batchSize*length) for i := range c.curBatchSize { From aa25aff10d1ccc6dd4e85952678d63946bdf89dc Mon Sep 17 00:00:00 2001 From: Patrick Devine Date: Tue, 27 May 2025 16:50:57 -0700 Subject: [PATCH 20/54] client: add request signing to the client (#10881) If OLLAMA_AUTH is set, sign each request w/ a timestamp and pass the signature in the token header --- api/client.go | 50 +++++++++++++++++++++++++++++++++++++++++++++ envconfig/config.go | 2 ++ 2 files changed, 52 insertions(+) diff --git a/api/client.go b/api/client.go index 3dffce600..9f0dba8dc 100644 --- a/api/client.go +++ b/api/client.go @@ -24,7 +24,10 @@ import ( "net/http" "net/url" "runtime" + "strconv" + "time" + "github.com/ollama/ollama/auth" "github.com/ollama/ollama/envconfig" "github.com/ollama/ollama/format" "github.com/ollama/ollama/version" @@ -76,6 +79,14 @@ func NewClient(base *url.URL, http *http.Client) *Client { } } +func getAuthorizationToken(ctx context.Context, challenge string) (string, error) { + token, err := auth.Sign(ctx, []byte(challenge)) + if err != nil { + return "", err + } + return token, nil +} + func (c *Client) do(ctx context.Context, method, path string, reqData, respData any) error { var reqBody io.Reader var data []byte @@ -97,6 +108,21 @@ func (c *Client) do(ctx context.Context, method, path string, reqData, respData } requestURL := c.base.JoinPath(path) + + var token string + if envconfig.UseAuth() || c.base.Hostname() == "ollama.com" { + now := strconv.FormatInt(time.Now().Unix(), 10) + chal := fmt.Sprintf("%s,%s?ts=%s", method, path, now) + token, err = getAuthorizationToken(ctx, chal) + if err != nil { + return err + } + + q := requestURL.Query() + q.Set("ts", now) + requestURL.RawQuery = q.Encode() + } + request, err := http.NewRequestWithContext(ctx, method, requestURL.String(), reqBody) if err != nil { return err @@ -106,6 +132,10 @@ func (c *Client) do(ctx context.Context, method, path string, reqData, respData request.Header.Set("Accept", "application/json") request.Header.Set("User-Agent", fmt.Sprintf("ollama/%s (%s %s) Go/%s", version.Version, runtime.GOARCH, runtime.GOOS, runtime.Version())) + if token != "" { + request.Header.Set("Authorization", token) + } + respObj, err := c.http.Do(request) if err != nil { return err @@ -143,6 +173,22 @@ func (c *Client) stream(ctx context.Context, method, path string, data any, fn f } requestURL := c.base.JoinPath(path) + + var token string + if envconfig.UseAuth() || c.base.Hostname() == "ollama.com" { + var err error + now := strconv.FormatInt(time.Now().Unix(), 10) + chal := fmt.Sprintf("%s,%s?ts=%s", method, path, now) + token, err = getAuthorizationToken(ctx, chal) + if err != nil { + return err + } + + q := requestURL.Query() + q.Set("ts", now) + requestURL.RawQuery = q.Encode() + } + request, err := http.NewRequestWithContext(ctx, method, requestURL.String(), buf) if err != nil { return err @@ -152,6 +198,10 @@ func (c *Client) stream(ctx context.Context, method, path string, data any, fn f request.Header.Set("Accept", "application/x-ndjson") request.Header.Set("User-Agent", fmt.Sprintf("ollama/%s (%s %s) Go/%s", version.Version, runtime.GOARCH, runtime.GOOS, runtime.Version())) + if token != "" { + request.Header.Set("Authorization", token) + } + response, err := c.http.Do(request) if err != nil { return err diff --git a/envconfig/config.go b/envconfig/config.go index 9d7c2e218..763f04646 100644 --- a/envconfig/config.go +++ b/envconfig/config.go @@ -183,6 +183,8 @@ var ( NewEngine = Bool("OLLAMA_NEW_ENGINE") // ContextLength sets the default context length ContextLength = Uint("OLLAMA_CONTEXT_LENGTH", 4096) + // Auth enables authentication between the Ollama client and server + UseAuth = Bool("OLLAMA_AUTH") ) func String(s string) func() string { From 5f57b0ef4268a6bd9e8043d54c351a608a7e1bca Mon Sep 17 00:00:00 2001 From: Devon Rifkin Date: Wed, 28 May 2025 19:38:52 -0700 Subject: [PATCH 21/54] add thinking support to the api and cli (#10584) - Both `/api/generate` and `/api/chat` now accept a `"think"` option that allows specifying whether thinking mode should be on or not - Templates get passed this new option so, e.g., qwen3's template can put `/think` or `/no_think` in the system prompt depending on the value of the setting - Models' thinking support is inferred by inspecting model templates. The prefix and suffix the parser uses to identify thinking support is also automatically inferred from templates - Thinking control & parsing is opt-in via the API to prevent breaking existing API consumers. If the `"think"` option is not specified, the behavior is unchanged from previous versions of ollama - Add parsing for thinking blocks in both streaming/non-streaming mode in both `/generate` and `/chat` - Update the CLI to make use of these changes. Users can pass `--think` or `--think=false` to control thinking, or during an interactive session they can use the commands `/set think` or `/set nothink` - A `--hidethinking` option has also been added to the CLI. This makes it easy to use thinking in scripting scenarios like `ollama run qwen3 --think --hidethinking "my question here"` where you just want to see the answer but still want the benefits of thinking models --- api/types.go | 21 +- api/types_test.go | 47 ++++ cmd/cmd.go | 178 +++++++++++++-- cmd/interactive.go | 32 +++ cmd/warn_thinking_test.go | 63 ++++++ docs/api.md | 3 + model/bytepairencoding.go | 11 +- readline/types.go | 2 + server/images.go | 20 +- server/prompt.go | 14 +- server/prompt_test.go | 3 +- server/routes.go | 89 +++++++- server/routes_generate_test.go | 19 ++ server/thinking.go | 300 ++++++++++++++++++++++++ server/thinking_test.go | 403 +++++++++++++++++++++++++++++++++ template/template.go | 38 ++-- types/model/capability.go | 1 + 17 files changed, 1195 insertions(+), 49 deletions(-) create mode 100644 cmd/warn_thinking_test.go create mode 100644 server/thinking.go create mode 100644 server/thinking_test.go diff --git a/api/types.go b/api/types.go index 602f93da8..94d492006 100644 --- a/api/types.go +++ b/api/types.go @@ -83,6 +83,12 @@ type GenerateRequest struct { // Options lists model-specific options. For example, temperature can be // set through this field, if the model supports it. Options map[string]any `json:"options"` + + // Think controls whether thinking/reasoning models will think before + // responding. Needs to be a pointer so we can distinguish between false + // (request that thinking _not_ be used) and unset (use the old behavior + // before this option was introduced) + Think *bool `json:"think,omitempty"` } // ChatRequest describes a request sent by [Client.Chat]. @@ -108,6 +114,10 @@ type ChatRequest struct { // Options lists model-specific options. Options map[string]any `json:"options"` + + // Think controls whether thinking/reasoning models will think before + // responding + Think *bool `json:"think,omitempty"` } type Tools []Tool @@ -126,8 +136,11 @@ func (t Tool) String() string { // role ("system", "user", or "assistant"), the content and an optional list // of images. type Message struct { - Role string `json:"role"` - Content string `json:"content"` + Role string `json:"role"` + Content string `json:"content"` + // Thinking contains the text that was inside thinking tags in the + // original model output when ChatRequest.Think is enabled. + Thinking string `json:"thinking,omitempty"` Images []ImageData `json:"images,omitempty"` ToolCalls []ToolCall `json:"tool_calls,omitempty"` } @@ -478,6 +491,10 @@ type GenerateResponse struct { // Response is the textual response itself. Response string `json:"response"` + // Thinking contains the text that was inside thinking tags in the + // original model output when ChatRequest.Think is enabled. + Thinking string `json:"thinking,omitempty"` + // Done specifies if the response is complete. Done bool `json:"done"` diff --git a/api/types_test.go b/api/types_test.go index 1a6fc811c..9c2fb1f11 100644 --- a/api/types_test.go +++ b/api/types_test.go @@ -372,3 +372,50 @@ func TestPropertyType_MarshalJSON(t *testing.T) { }) } } + +func TestThinking_UnmarshalJSON(t *testing.T) { + trueVal := true + falseVal := false + + tests := []struct { + name string + input string + expectedThinking *bool + expectedError bool + }{ + { + name: "true", + input: `{ "think": true }`, + expectedThinking: &trueVal, + }, + { + name: "false", + input: `{ "think": false }`, + expectedThinking: &falseVal, + }, + { + name: "unset", + input: `{ }`, + expectedThinking: nil, + }, + { + name: "invalid", + input: `{ "think": "true" }`, + expectedThinking: nil, + expectedError: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + var req GenerateRequest + err := json.Unmarshal([]byte(test.input), &req) + if test.expectedError { + require.Error(t, err) + } else { + require.NoError(t, err) + assert.Equal(t, test.expectedThinking, req.Think) + } + }) + } +} diff --git a/cmd/cmd.go b/cmd/cmd.go index b9047529d..2d1653790 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -39,6 +39,7 @@ import ( "github.com/ollama/ollama/format" "github.com/ollama/ollama/parser" "github.com/ollama/ollama/progress" + "github.com/ollama/ollama/readline" "github.com/ollama/ollama/runner" "github.com/ollama/ollama/server" "github.com/ollama/ollama/types/model" @@ -46,6 +47,23 @@ import ( "github.com/ollama/ollama/version" ) +// ensureThinkingSupport emits a warning if the model does not advertise thinking support +func ensureThinkingSupport(ctx context.Context, client *api.Client, name string) { + if name == "" { + return + } + resp, err := client.Show(ctx, &api.ShowRequest{Model: name}) + if err != nil { + return + } + for _, cap := range resp.Capabilities { + if cap == model.CapabilityThinking { + return + } + } + fmt.Fprintf(os.Stderr, "warning: model %q does not support thinking output\n", name) +} + var errModelfileNotFound = errors.New("specified Modelfile wasn't found") func getModelfileName(cmd *cobra.Command) (string, error) { @@ -265,6 +283,9 @@ func loadOrUnloadModel(cmd *cobra.Command, opts *runOptions) error { req := &api.GenerateRequest{ Model: opts.Model, KeepAlive: opts.KeepAlive, + + // pass Think here so we fail before getting to the chat prompt if the model doesn't support it + Think: opts.Think, } return client.Generate(cmd.Context(), req, func(api.GenerateResponse) error { return nil }) @@ -299,6 +320,22 @@ func RunHandler(cmd *cobra.Command, args []string) error { } opts.Format = format + thinkFlag := cmd.Flags().Lookup("think") + if thinkFlag.Changed { + think, err := cmd.Flags().GetBool("think") + if err != nil { + return err + } + opts.Think = &think + } else { + opts.Think = nil + } + hidethinking, err := cmd.Flags().GetBool("hidethinking") + if err != nil { + return err + } + opts.HideThinking = hidethinking + keepAlive, err := cmd.Flags().GetString("keepalive") if err != nil { return err @@ -362,6 +399,11 @@ func RunHandler(cmd *cobra.Command, args []string) error { return err } + opts.Think, err = inferThinkingOption(&info.Capabilities, &opts, thinkFlag.Changed) + if err != nil { + return err + } + opts.MultiModal = slices.Contains(info.Capabilities, model.CapabilityVision) // TODO: remove the projector info and vision info checks below, @@ -923,17 +965,19 @@ func PullHandler(cmd *cobra.Command, args []string) error { type generateContextKey string type runOptions struct { - Model string - ParentModel string - Prompt string - Messages []api.Message - WordWrap bool - Format string - System string - Images []api.ImageData - Options map[string]any - MultiModal bool - KeepAlive *api.Duration + Model string + ParentModel string + Prompt string + Messages []api.Message + WordWrap bool + Format string + System string + Images []api.ImageData + Options map[string]any + MultiModal bool + KeepAlive *api.Duration + Think *bool + HideThinking bool } type displayResponseState struct { @@ -989,6 +1033,26 @@ func displayResponse(content string, wordWrap bool, state *displayResponseState) } } +func thinkingOutputOpeningText(plainText bool) string { + text := "Thinking...\n" + + if plainText { + return text + } + + return readline.ColorGrey + readline.ColorBold + text + readline.ColorDefault + readline.ColorGrey +} + +func thinkingOutputClosingText(plainText bool) string { + text := "...done thinking.\n\n" + + if plainText { + return text + } + + return readline.ColorGrey + readline.ColorBold + text + readline.ColorDefault +} + func chat(cmd *cobra.Command, opts runOptions) (*api.Message, error) { client, err := api.ClientFromEnvironment() if err != nil { @@ -1016,14 +1080,34 @@ func chat(cmd *cobra.Command, opts runOptions) (*api.Message, error) { var latest api.ChatResponse var fullResponse strings.Builder var role string + var thinkTagOpened bool = false + var thinkTagClosed bool = false fn := func(response api.ChatResponse) error { - p.StopAndClear() + if response.Message.Content != "" || !opts.HideThinking { + p.StopAndClear() + } latest = response role = response.Message.Role + if response.Message.Thinking != "" && !opts.HideThinking { + if !thinkTagOpened { + fmt.Print(thinkingOutputOpeningText(false)) + thinkTagOpened = true + } + displayResponse(response.Message.Thinking, opts.WordWrap, state) + } + content := response.Message.Content + if thinkTagOpened && !thinkTagClosed && content != "" { + fmt.Print(thinkingOutputClosingText(false)) + thinkTagClosed = true + } + // purposefully not putting thinking blocks in the response, which would + // only be needed if we later added tool calling to the cli (they get + // filtered out anyway since current models don't expect them unless you're + // about to finish some tool calls) fullResponse.WriteString(content) displayResponse(content, opts.WordWrap, state) @@ -1040,6 +1124,7 @@ func chat(cmd *cobra.Command, opts runOptions) (*api.Message, error) { Messages: opts.Messages, Format: json.RawMessage(opts.Format), Options: opts.Options, + Think: opts.Think, } if opts.KeepAlive != nil { @@ -1101,13 +1186,32 @@ func generate(cmd *cobra.Command, opts runOptions) error { }() var state *displayResponseState = &displayResponseState{} + var thinkTagOpened bool = false + var thinkTagClosed bool = false + + plainText := !term.IsTerminal(int(os.Stdout.Fd())) fn := func(response api.GenerateResponse) error { - p.StopAndClear() - latest = response content := response.Response + if response.Response != "" || !opts.HideThinking { + p.StopAndClear() + } + + if response.Thinking != "" && !opts.HideThinking { + if !thinkTagOpened { + fmt.Print(thinkingOutputOpeningText(plainText)) + thinkTagOpened = true + } + displayResponse(response.Thinking, opts.WordWrap, state) + } + + if thinkTagOpened && !thinkTagClosed && content != "" { + fmt.Print(thinkingOutputClosingText(plainText)) + thinkTagClosed = true + } + displayResponse(content, opts.WordWrap, state) return nil @@ -1133,6 +1237,7 @@ func generate(cmd *cobra.Command, opts runOptions) error { System: opts.System, Options: opts.Options, KeepAlive: opts.KeepAlive, + Think: opts.Think, } if err := client.Generate(ctx, &request, fn); err != nil { @@ -1348,6 +1453,8 @@ func NewCLI() *cobra.Command { runCmd.Flags().Bool("insecure", false, "Use an insecure registry") runCmd.Flags().Bool("nowordwrap", false, "Don't wrap words to the next line automatically") runCmd.Flags().String("format", "", "Response format (e.g. json)") + runCmd.Flags().Bool("think", false, "Whether to use thinking mode for supported models") + runCmd.Flags().Bool("hidethinking", false, "Hide thinking output (if provided)") stopCmd := &cobra.Command{ Use: "stop MODEL", @@ -1399,7 +1506,6 @@ func NewCLI() *cobra.Command { PreRunE: checkServerHeartbeat, RunE: ListRunningHandler, } - copyCmd := &cobra.Command{ Use: "cp SOURCE DESTINATION", Short: "Copy a model", @@ -1488,3 +1594,45 @@ func NewCLI() *cobra.Command { return rootCmd } + +// If the user has explicitly set thinking options, either through the CLI or +// through the `/set think` or `set nothink` interactive options, then we +// respect them. Otherwise, we check model capabilities to see if the model +// supports thinking. If the model does support thinking, we enable it. +// Otherwise, we unset the thinking option (which is different than setting it +// to false). +// +// If capabilities are not provided, we fetch them from the server. +func inferThinkingOption(caps *[]model.Capability, runOpts *runOptions, explicitlySetByUser bool) (*bool, error) { + if explicitlySetByUser { + return runOpts.Think, nil + } + + if caps == nil { + client, err := api.ClientFromEnvironment() + if err != nil { + return nil, err + } + ret, err := client.Show(context.Background(), &api.ShowRequest{ + Model: runOpts.Model, + }) + if err != nil { + return nil, err + } + caps = &ret.Capabilities + } + + thinkingSupported := false + for _, cap := range *caps { + if cap == model.CapabilityThinking { + thinkingSupported = true + } + } + + if thinkingSupported { + thinking := true + return &thinking, nil + } + + return nil, nil +} diff --git a/cmd/interactive.go b/cmd/interactive.go index d7e6fbcfb..a285b365c 100644 --- a/cmd/interactive.go +++ b/cmd/interactive.go @@ -62,6 +62,8 @@ func generateInteractive(cmd *cobra.Command, opts runOptions) error { fmt.Fprintln(os.Stderr, " /set noformat Disable formatting") fmt.Fprintln(os.Stderr, " /set verbose Show LLM stats") fmt.Fprintln(os.Stderr, " /set quiet Disable LLM stats") + fmt.Fprintln(os.Stderr, " /set think Enable thinking") + fmt.Fprintln(os.Stderr, " /set nothink Disable thinking") fmt.Fprintln(os.Stderr, "") } @@ -128,6 +130,7 @@ func generateInteractive(cmd *cobra.Command, opts runOptions) error { var sb strings.Builder var multiline MultilineState + var thinkExplicitlySet bool = opts.Think != nil for { line, err := scanner.Readline() @@ -195,11 +198,19 @@ func generateInteractive(cmd *cobra.Command, opts runOptions) error { opts.Model = args[1] opts.Messages = []api.Message{} fmt.Printf("Loading model '%s'\n", opts.Model) + opts.Think, err = inferThinkingOption(nil, &opts, thinkExplicitlySet) + if err != nil { + return err + } if err := loadOrUnloadModel(cmd, &opts); err != nil { if strings.Contains(err.Error(), "not found") { fmt.Printf("error: %v\n", err) continue } + if strings.Contains(err.Error(), "does not support thinking") { + fmt.Printf("error: %v\n", err) + continue + } return err } continue @@ -260,6 +271,22 @@ func generateInteractive(cmd *cobra.Command, opts runOptions) error { return err } fmt.Println("Set 'quiet' mode.") + case "think": + think := true + opts.Think = &think + thinkExplicitlySet = true + if client, err := api.ClientFromEnvironment(); err == nil { + ensureThinkingSupport(cmd.Context(), client, opts.Model) + } + fmt.Println("Set 'think' mode.") + case "nothink": + think := false + opts.Think = &think + thinkExplicitlySet = true + if client, err := api.ClientFromEnvironment(); err == nil { + ensureThinkingSupport(cmd.Context(), client, opts.Model) + } + fmt.Println("Set 'nothink' mode.") case "format": if len(args) < 3 || args[2] != "json" { fmt.Println("Invalid or missing format. For 'json' mode use '/set format json'") @@ -448,6 +475,11 @@ func generateInteractive(cmd *cobra.Command, opts runOptions) error { assistant, err := chat(cmd, opts) if err != nil { + if strings.Contains(err.Error(), "does not support thinking") { + fmt.Printf("error: %v\n", err) + sb.Reset() + continue + } return err } if assistant != nil { diff --git a/cmd/warn_thinking_test.go b/cmd/warn_thinking_test.go new file mode 100644 index 000000000..31dc4156b --- /dev/null +++ b/cmd/warn_thinking_test.go @@ -0,0 +1,63 @@ +package cmd + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + + "github.com/ollama/ollama/api" + "github.com/ollama/ollama/types/model" +) + +// Test that a warning is printed when thinking is requested but not supported. +func TestWarnMissingThinking(t *testing.T) { + cases := []struct { + capabilities []model.Capability + expectWarn bool + }{ + {capabilities: []model.Capability{model.CapabilityThinking}, expectWarn: false}, + {capabilities: []model.Capability{}, expectWarn: true}, + } + + for _, tc := range cases { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/show" || r.Method != http.MethodPost { + t.Fatalf("unexpected request to %s %s", r.URL.Path, r.Method) + } + var req api.ShowRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Fatalf("decode request: %v", err) + } + resp := api.ShowResponse{Capabilities: tc.capabilities} + if err := json.NewEncoder(w).Encode(resp); err != nil { + t.Fatalf("encode response: %v", err) + } + })) + defer srv.Close() + + t.Setenv("OLLAMA_HOST", srv.URL) + client, err := api.ClientFromEnvironment() + if err != nil { + t.Fatal(err) + } + oldStderr := os.Stderr + r, w, _ := os.Pipe() + os.Stderr = w + ensureThinkingSupport(t.Context(), client, "m") + w.Close() + os.Stderr = oldStderr + out, _ := io.ReadAll(r) + + warned := strings.Contains(string(out), "warning:") + if tc.expectWarn && !warned { + t.Errorf("expected warning, got none") + } + if !tc.expectWarn && warned { + t.Errorf("did not expect warning, got: %s", string(out)) + } + } +} diff --git a/docs/api.md b/docs/api.md index abd276150..11eaf73ab 100644 --- a/docs/api.md +++ b/docs/api.md @@ -43,6 +43,7 @@ Generate a response for a given prompt with a provided model. This is a streamin - `prompt`: the prompt to generate a response for - `suffix`: the text after the model response - `images`: (optional) a list of base64-encoded images (for multimodal models such as `llava`) +- `think`: (for thinking models) should the model think before responding? Advanced parameters (optional): @@ -490,11 +491,13 @@ Generate the next message in a chat with a provided model. This is a streaming e - `model`: (required) the [model name](#model-names) - `messages`: the messages of the chat, this can be used to keep a chat memory - `tools`: list of tools in JSON for the model to use if supported +- `think`: (for thinking models) should the model think before responding? The `message` object has the following fields: - `role`: the role of the message, either `system`, `user`, `assistant`, or `tool` - `content`: the content of the message +- `thinking`: (for thinking models) the model's thinking process - `images` (optional): a list of images to include in the message (for multimodal models such as `llava`) - `tool_calls` (optional): a list of tools in JSON that the model wants to use diff --git a/model/bytepairencoding.go b/model/bytepairencoding.go index 6bb9a003e..246d2ba3e 100644 --- a/model/bytepairencoding.go +++ b/model/bytepairencoding.go @@ -3,6 +3,7 @@ package model import ( "cmp" "context" + "fmt" "iter" "log/slog" "strings" @@ -210,6 +211,14 @@ func (bpe BytePairEncoding) Encode(s string, addSpecial bool) ([]int32, error) { return ids, nil } +type lazyIdsString struct { + ids []int32 +} + +func (l lazyIdsString) LogValue() slog.Value { + return slog.AnyValue(fmt.Sprint(l.ids)) +} + func (bpe BytePairEncoding) Decode(ids []int32) (string, error) { var sb strings.Builder for _, id := range ids { @@ -234,6 +243,6 @@ func (bpe BytePairEncoding) Decode(ids []int32) (string, error) { } } - slog.Log(context.TODO(), logutil.LevelTrace, "decoded", "ids", ids, "string", sb.String()) + slog.Log(context.TODO(), logutil.LevelTrace, "decoded", "string", sb.String(), "from", lazyIdsString{ids: ids}) return sb.String(), nil } diff --git a/readline/types.go b/readline/types.go index e136d9962..f4efa8d92 100644 --- a/readline/types.go +++ b/readline/types.go @@ -61,6 +61,8 @@ const ( ColorGrey = Esc + "[38;5;245m" ColorDefault = Esc + "[0m" + ColorBold = Esc + "[1m" + StartBracketedPaste = Esc + "[?2004h" EndBracketedPaste = Esc + "[?2004l" ) diff --git a/server/images.go b/server/images.go index a69e2a9f2..58fb87dcc 100644 --- a/server/images.go +++ b/server/images.go @@ -37,6 +37,7 @@ var ( errCapabilityInsert = errors.New("insert") errCapabilityVision = errors.New("vision") errCapabilityEmbedding = errors.New("embedding") + errCapabilityThinking = errors.New("thinking") errInsecureProtocol = errors.New("insecure protocol http") ) @@ -111,6 +112,12 @@ func (m *Model) Capabilities() []model.Capability { capabilities = append(capabilities, model.CapabilityVision) } + // Check for thinking capability + openingTag, closingTag := inferThinkingTags(m.Template.Template) + if openingTag != "" && closingTag != "" { + capabilities = append(capabilities, model.CapabilityThinking) + } + return capabilities } @@ -127,6 +134,7 @@ func (m *Model) CheckCapabilities(want ...model.Capability) error { model.CapabilityInsert: errCapabilityInsert, model.CapabilityVision: errCapabilityVision, model.CapabilityEmbedding: errCapabilityEmbedding, + model.CapabilityThinking: errCapabilityThinking, } for _, cap := range want { @@ -141,11 +149,19 @@ func (m *Model) CheckCapabilities(want ...model.Capability) error { } } + var err error if len(errs) > 0 { - return fmt.Errorf("%w %w", errCapabilities, errors.Join(errs...)) + err = fmt.Errorf("%w %w", errCapabilities, errors.Join(errs...)) } - return nil + if slices.Contains(errs, errCapabilityThinking) { + if m.Config.ModelFamily == "qwen3" || model.ParseName(m.Name).Model == "deepseek-r1" { + // append a message to the existing error + return fmt.Errorf("%w. Pull the model again to get the latest version with full thinking support", err) + } + } + + return err } func (m *Model) String() string { diff --git a/server/prompt.go b/server/prompt.go index 147a02b69..f8c895d71 100644 --- a/server/prompt.go +++ b/server/prompt.go @@ -19,7 +19,7 @@ type tokenizeFunc func(context.Context, string) ([]int, error) // chatPrompt accepts a list of messages and returns the prompt and images that should be used for the next chat turn. // chatPrompt truncates any messages that exceed the context window of the model, making sure to always include 1) the // latest message and 2) system messages -func chatPrompt(ctx context.Context, m *Model, tokenize tokenizeFunc, opts *api.Options, msgs []api.Message, tools []api.Tool) (prompt string, images []llm.ImageData, _ error) { +func chatPrompt(ctx context.Context, m *Model, tokenize tokenizeFunc, opts *api.Options, msgs []api.Message, tools []api.Tool, think *bool) (prompt string, images []llm.ImageData, _ error) { var system []api.Message // TODO: Ideally we would compute this from the projector metadata but some pieces are implementation dependent @@ -41,8 +41,12 @@ func chatPrompt(ctx context.Context, m *Model, tokenize tokenizeFunc, opts *api. } } + thinkVal := false + if think != nil { + thinkVal = *think + } var b bytes.Buffer - if err := m.Template.Execute(&b, template.Values{Messages: append(system, msgs[i:]...), Tools: tools}); err != nil { + if err := m.Template.Execute(&b, template.Values{Messages: append(system, msgs[i:]...), Tools: tools, Think: thinkVal, IsThinkSet: think != nil}); err != nil { return "", nil, err } @@ -96,7 +100,11 @@ func chatPrompt(ctx context.Context, m *Model, tokenize tokenizeFunc, opts *api. // truncate any messages that do not fit into the context window var b bytes.Buffer - if err := m.Template.Execute(&b, template.Values{Messages: append(system, msgs[currMsgIdx:]...), Tools: tools}); err != nil { + thinkVal := false + if think != nil { + thinkVal = *think + } + if err := m.Template.Execute(&b, template.Values{Messages: append(system, msgs[currMsgIdx:]...), Tools: tools, Think: thinkVal, IsThinkSet: think != nil}); err != nil { return "", nil, err } diff --git a/server/prompt_test.go b/server/prompt_test.go index fb6c96c0c..0043b9a47 100644 --- a/server/prompt_test.go +++ b/server/prompt_test.go @@ -208,7 +208,8 @@ func TestChatPrompt(t *testing.T) { t.Run(tt.name, func(t *testing.T) { model := tt.model opts := api.Options{Runner: api.Runner{NumCtx: tt.limit}} - prompt, images, err := chatPrompt(t.Context(), &model, mockRunner{}.Tokenize, &opts, tt.msgs, nil) + think := false + prompt, images, err := chatPrompt(t.Context(), &model, mockRunner{}.Tokenize, &opts, tt.msgs, nil, &think) if tt.error == nil && err != nil { t.Fatal(err) } else if tt.error != nil && err != tt.error { diff --git a/server/routes.go b/server/routes.go index 42e8cdd1d..236f92e22 100644 --- a/server/routes.go +++ b/server/routes.go @@ -17,7 +17,6 @@ import ( "net/netip" "os" "os/signal" - "regexp" "slices" "strings" "syscall" @@ -186,6 +185,13 @@ func (s *Server) GenerateHandler(c *gin.Context) { if req.Suffix != "" { caps = append(caps, model.CapabilityInsert) } + if req.Think != nil && *req.Think { + caps = append(caps, model.CapabilityThinking) + // TODO(drifkin): consider adding a warning if it's false and the model + // doesn't support thinking. It's not strictly required, but it can be a + // hint that the user is on an older qwen3/r1 model that doesn't have an + // updated template supporting thinking + } r, m, opts, err := s.scheduleRunner(c.Request.Context(), name.String(), caps, req.Options, req.KeepAlive) if errors.Is(err, errCapabilityCompletion) { @@ -254,6 +260,9 @@ func (s *Server) GenerateHandler(c *gin.Context) { values.Messages = append(msgs, api.Message{Role: "user", Content: req.Prompt}) } + values.Think = req.Think != nil && *req.Think + values.IsThinkSet = req.Think != nil + var b bytes.Buffer if req.Context != nil { slog.Warn("the context field is deprecated and will be removed in a future version of Ollama") @@ -273,6 +282,15 @@ func (s *Server) GenerateHandler(c *gin.Context) { prompt = b.String() } + var thinkingState *thinkingParser + openingTag, closingTag := inferThinkingTags(m.Template.Template) + if req.Think != nil && *req.Think && openingTag != "" && closingTag != "" { + thinkingState = &thinkingParser{ + openingTag: openingTag, + closingTag: closingTag, + } + } + ch := make(chan any) go func() { // TODO (jmorganca): avoid building the response twice both here and below @@ -297,6 +315,12 @@ func (s *Server) GenerateHandler(c *gin.Context) { }, } + if thinkingState != nil { + thinking, content := thinkingState.addContent(cr.Content) + res.Thinking = thinking + res.Response = content + } + if _, err := sb.WriteString(cr.Content); err != nil { ch <- gin.H{"error": err.Error()} } @@ -324,11 +348,13 @@ func (s *Server) GenerateHandler(c *gin.Context) { if req.Stream != nil && !*req.Stream { var r api.GenerateResponse - var sb strings.Builder + var sbThinking strings.Builder + var sbContent strings.Builder for rr := range ch { switch t := rr.(type) { case api.GenerateResponse: - sb.WriteString(t.Response) + sbThinking.WriteString(t.Thinking) + sbContent.WriteString(t.Response) r = t case gin.H: msg, ok := t["error"].(string) @@ -344,7 +370,9 @@ func (s *Server) GenerateHandler(c *gin.Context) { } } - r.Response = sb.String() + r.Thinking = sbThinking.String() + r.Response = sbContent.String() + c.JSON(http.StatusOK, r) return } @@ -1436,6 +1464,9 @@ func (s *Server) ChatHandler(c *gin.Context) { if len(req.Tools) > 0 { caps = append(caps, model.CapabilityTools) } + if req.Think != nil && *req.Think { + caps = append(caps, model.CapabilityThinking) + } name := model.ParseName(req.Model) if !name.IsValid() { @@ -1476,13 +1507,22 @@ func (s *Server) ChatHandler(c *gin.Context) { } msgs = filterThinkTags(msgs, m) - prompt, images, err := chatPrompt(c.Request.Context(), m, r.Tokenize, opts, msgs, req.Tools) + prompt, images, err := chatPrompt(c.Request.Context(), m, r.Tokenize, opts, msgs, req.Tools, req.Think) if err != nil { slog.Error("chat prompt error", "error", err) c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) return } + var thinkingState *thinkingParser + openingTag, closingTag := inferThinkingTags(m.Template.Template) + if req.Think != nil && *req.Think && openingTag != "" && closingTag != "" { + thinkingState = &thinkingParser{ + openingTag: openingTag, + closingTag: closingTag, + } + } + var toolParser *tools.Parser if len(req.Tools) > 0 { toolParser, err = tools.NewParser(m.Template.Template) @@ -1516,6 +1556,16 @@ func (s *Server) ChatHandler(c *gin.Context) { }, } + if thinkingState != nil { + thinkingContent, remainingContent := thinkingState.addContent(res.Message.Content) + if thinkingContent == "" && remainingContent == "" && !r.Done { + // need to accumulate more to decide what to send + return + } + res.Message.Content = remainingContent + res.Message.Thinking = thinkingContent + } + if r.Done { res.DoneReason = r.DoneReason.String() res.TotalDuration = time.Since(checkpointStart) @@ -1523,12 +1573,14 @@ func (s *Server) ChatHandler(c *gin.Context) { } if len(req.Tools) > 0 { - toolCalls, content := toolParser.Add(r.Content) + toolCalls, content := toolParser.Add(res.Message.Content) if len(content) > 0 { res.Message.Content = content } else if len(toolCalls) > 0 { res.Message.ToolCalls = toolCalls res.Message.Content = "" + } else if res.Message.Thinking != "" { + // don't return } else { if r.Done { ch <- res @@ -1536,6 +1588,7 @@ func (s *Server) ChatHandler(c *gin.Context) { return } } + ch <- res }); err != nil { ch <- gin.H{"error": err.Error()} @@ -1544,12 +1597,14 @@ func (s *Server) ChatHandler(c *gin.Context) { if req.Stream != nil && !*req.Stream { var resp api.ChatResponse - var sb strings.Builder var toolCalls []api.ToolCall + var sbThinking strings.Builder + var sbContent strings.Builder for rr := range ch { switch t := rr.(type) { case api.ChatResponse: - sb.WriteString(t.Message.Content) + sbThinking.WriteString(t.Message.Thinking) + sbContent.WriteString(t.Message.Content) resp = t if len(req.Tools) > 0 { toolCalls = append(toolCalls, t.Message.ToolCalls...) @@ -1568,7 +1623,9 @@ func (s *Server) ChatHandler(c *gin.Context) { } } - resp.Message.Content = sb.String() + resp.Message.Content = sbContent.String() + resp.Message.Thinking = sbThinking.String() + if len(toolCalls) > 0 { resp.Message.ToolCalls = toolCalls } @@ -1595,8 +1652,6 @@ func handleScheduleError(c *gin.Context, name string, err error) { } } -var thinkTagRegexp = regexp.MustCompile(`(?s).*?(\n)*`) - func filterThinkTags(msgs []api.Message, m *Model) []api.Message { if m.Config.ModelFamily == "qwen3" || model.ParseName(m.Name).Model == "deepseek-r1" { finalUserIndex := -1 @@ -1608,7 +1663,17 @@ func filterThinkTags(msgs []api.Message, m *Model) []api.Message { for i, msg := range msgs { if msg.Role == "assistant" && i < finalUserIndex { - msgs[i].Content = thinkTagRegexp.ReplaceAllString(msg.Content, "") + // TODO(drifkin): this is from before we added proper thinking support. + // However, even if thinking is not enabled (and therefore we shouldn't + // change the user output), we should probably perform this filtering + // for all thinking models (not just qwen3 & deepseek-r1) since it tends + // to save tokens and improve quality. + thinkingState := &thinkingParser{ + openingTag: "", + closingTag: "", + } + _, content := thinkingState.addContent(msg.Content) + msgs[i].Content = content } } } diff --git a/server/routes_generate_test.go b/server/routes_generate_test.go index 6bbf5b112..75a246fc6 100644 --- a/server/routes_generate_test.go +++ b/server/routes_generate_test.go @@ -143,6 +143,25 @@ func TestGenerateChat(t *testing.T) { } }) + t.Run("missing thinking capability", func(t *testing.T) { + think := true + w := createRequest(t, s.ChatHandler, api.ChatRequest{ + Model: "test", + Messages: []api.Message{ + {Role: "user", Content: "Hello!"}, + }, + Think: &think, + }) + + if w.Code != http.StatusBadRequest { + t.Errorf("expected status 400, got %d", w.Code) + } + + if diff := cmp.Diff(w.Body.String(), `{"error":"registry.ollama.ai/library/test:latest does not support thinking"}`); diff != "" { + t.Errorf("mismatch (-got +want):\n%s", diff) + } + }) + t.Run("missing model", func(t *testing.T) { w := createRequest(t, s.ChatHandler, api.ChatRequest{}) if w.Code != http.StatusBadRequest { diff --git a/server/thinking.go b/server/thinking.go new file mode 100644 index 000000000..2213b6b6e --- /dev/null +++ b/server/thinking.go @@ -0,0 +1,300 @@ +package server + +import ( + "strings" + "text/template" + "text/template/parse" + "unicode" +) + +type thinkingState int + +const ( + // We're looking for the opening tag, but we haven't seen any non-whitespace + // characters yet + thinkingState_LookingForOpening thinkingState = iota + // We've seen the opening tag, but we haven't seen any non-whitespace + // characters yet (we want to eat any whitespace between the opening tag and + // the thinking content) + thinkingState_ThinkingStartedEatingWhitespace + // We've seen non-whitespace characters after the opening tag, but we haven't + // seen the closing tag yet + thinkingState_Thinking + // We've seen the closing tag, but we haven't seen any non-whitespace + // characters after the closing tag yet (we want to eat any whitespace between + // the closing tag and the content) + thinkingState_ThinkingDoneEatingWhitespace + // We've seen the closing tag and seen at least one non-whitespace character + // after it + thinkingState_ThinkingDone +) + +func (s thinkingState) String() string { + switch s { + case thinkingState_LookingForOpening: + return "LookingForOpening" + case thinkingState_ThinkingStartedEatingWhitespace: + return "ThinkingStartedEatingWhitespace" + case thinkingState_Thinking: + return "Thinking" + case thinkingState_ThinkingDoneEatingWhitespace: + return "ThinkingDoneEatingWhitespace" + case thinkingState_ThinkingDone: + return "ThinkingDone" + default: + return "Unknown" + } +} + +type thinkingParser struct { + state thinkingState + openingTag string + closingTag string + acc strings.Builder +} + +// addContent returns the thinking content and the non-thinking content that +// should be immediately sent to the user. It will internally buffer if it needs +// to see more raw content to disambiguate +func (s *thinkingParser) addContent(content string) (string, string) { + s.acc.WriteString(content) + + var thinkingSb, remainingSb strings.Builder + + var thinking, remaining string + keepLooping := true + // we loop because we might pass through multiple parsing states in a single + // call to addContent, and we want to make sure callers don't have to wait for + // data that's already unambiguous + for keepLooping { + thinking, remaining, keepLooping = eat(s) + thinkingSb.WriteString(thinking) + remainingSb.WriteString(remaining) + } + + return thinkingSb.String(), remainingSb.String() +} + +// the additional bool return is true iff we should continue eating +func eat(s *thinkingParser) (string, string, bool) { + switch s.state { + case thinkingState_LookingForOpening: + trimmed := strings.TrimLeftFunc(s.acc.String(), unicode.IsSpace) + if strings.HasPrefix(trimmed, s.openingTag) { + after := strings.Join(strings.Split(trimmed, s.openingTag)[1:], s.openingTag) + after = strings.TrimLeftFunc(after, unicode.IsSpace) + // after might contain more than just thinking tokens, so we continue + // parsing instead of returning it as thinking tokens here + s.acc.Reset() + s.acc.WriteString(after) + if after == "" { + s.state = thinkingState_ThinkingStartedEatingWhitespace + } else { + s.state = thinkingState_Thinking + } + return "", "", true + } else if strings.HasPrefix(s.openingTag, trimmed) { + // partial opening seen, so let's keep accumulating + return "", "", false + } else if trimmed == "" { + // saw whitespace only, so let's keep accumulating + return "", "", false + } else { + // didn't see an opening tag, but we have content, so thinking was skipped + s.state = thinkingState_ThinkingDone + // note that we use the original content, not the trimmed one because we + // don't want to eat any whitespace in the real content if there were no + // thinking tags + return "", s.acc.String(), false + } + case thinkingState_ThinkingStartedEatingWhitespace: + trimmed := strings.TrimLeftFunc(s.acc.String(), unicode.IsSpace) + s.acc.Reset() + if trimmed == "" { + return "", "", false + } else { + s.state = thinkingState_Thinking + s.acc.WriteString(trimmed) + return "", "", true + } + case thinkingState_Thinking: + acc := s.acc.String() + if strings.Contains(acc, s.closingTag) { + split := strings.Split(acc, s.closingTag) + thinking := split[0] + remaining := strings.Join(split[1:], s.closingTag) + remaining = strings.TrimLeftFunc(remaining, unicode.IsSpace) + s.acc.Reset() + if remaining == "" { + s.state = thinkingState_ThinkingDoneEatingWhitespace + } else { + s.state = thinkingState_ThinkingDone + } + return thinking, remaining, false + } else if overlapLen := overlap(acc, s.closingTag); overlapLen > 0 { + thinking := acc[:len(acc)-overlapLen] + remaining := acc[len(acc)-overlapLen:] + s.acc.Reset() + // keep track of the candidate closing tag. We have to buffer it until it + // becomes disambiguated + s.acc.WriteString(remaining) + return thinking, "", false + } else { + // purely just thinking tokens, so we can return them + s.acc.Reset() + return acc, "", false + } + case thinkingState_ThinkingDoneEatingWhitespace: + trimmed := strings.TrimLeftFunc(s.acc.String(), unicode.IsSpace) + s.acc.Reset() + // if we see non-whitespace, we're done eating the leading whitespace of the content + if trimmed != "" { + s.state = thinkingState_ThinkingDone + } + return "", trimmed, false + case thinkingState_ThinkingDone: + acc := s.acc.String() + s.acc.Reset() + return "", acc, false + default: + panic("unknown state") + } +} + +// longest overlap between suffix of s and prefix of delim +func overlap(s, delim string) int { + max := min(len(delim), len(s)) + for i := max; i > 0; i-- { + if strings.HasSuffix(s, delim[:i]) { + return i + } + } + return 0 +} + +func templateVisit(n parse.Node, enterFn func(parse.Node) bool, exitFn func(parse.Node)) { + if n == nil { + return + } + shouldContinue := enterFn(n) + if !shouldContinue { + return + } + switch x := n.(type) { + case *parse.ListNode: + for _, c := range x.Nodes { + templateVisit(c, enterFn, exitFn) + } + case *parse.BranchNode: + if x.Pipe != nil { + templateVisit(x.Pipe, enterFn, exitFn) + } + if x.List != nil { + templateVisit(x.List, enterFn, exitFn) + } + if x.ElseList != nil { + templateVisit(x.ElseList, enterFn, exitFn) + } + case *parse.ActionNode: + templateVisit(x.Pipe, enterFn, exitFn) + case *parse.WithNode: + templateVisit(&x.BranchNode, enterFn, exitFn) + case *parse.RangeNode: + templateVisit(&x.BranchNode, enterFn, exitFn) + case *parse.IfNode: + templateVisit(&x.BranchNode, enterFn, exitFn) + case *parse.TemplateNode: + templateVisit(x.Pipe, enterFn, exitFn) + case *parse.PipeNode: + for _, c := range x.Cmds { + templateVisit(c, enterFn, exitFn) + } + case *parse.CommandNode: + for _, a := range x.Args { + templateVisit(a, enterFn, exitFn) + } + // text, field, number, etc. are leaves – nothing to recurse into + } + if exitFn != nil { + exitFn(n) + } +} + +// We use a heuristic to infer the tags that surround thinking traces: +// We look for a range node that iterates over "Messages" and then look for a +// reference to "Thinking" like `{{.Thinking}}`. We then go up to the nearest +// ListNode and take the first and last TextNodes as the opening and closing +// tags. +func inferThinkingTags(t *template.Template) (string, string) { + ancestors := []parse.Node{} + + openingTag := "" + closingTag := "" + + enterFn := func(n parse.Node) bool { + ancestors = append(ancestors, n) + + switch x := n.(type) { + case *parse.FieldNode: + if len(x.Ident) > 0 && x.Ident[0] == "Thinking" { + var mostRecentRange *parse.RangeNode + for i := len(ancestors) - 1; i >= 0; i-- { + if r, ok := ancestors[i].(*parse.RangeNode); ok { + mostRecentRange = r + break + } + } + if mostRecentRange == nil || !rangeUsesField(mostRecentRange, "Messages") { + return true + } + + // TODO(drifkin): to be more robust, check that it's in the action + // part, not the `if`'s pipeline part. We do match on the nearest list + // that starts and ends with text nodes, which makes this not strictly + // necessary for our heuristic + + // go up to the nearest ancestor that is a *parse.ListNode + for i := len(ancestors) - 1; i >= 0; i-- { + if l, ok := ancestors[i].(*parse.ListNode); ok { + firstNode := l.Nodes[0] + if t, ok := firstNode.(*parse.TextNode); ok { + openingTag = strings.TrimSpace(t.String()) + } + lastNode := l.Nodes[len(l.Nodes)-1] + if t, ok := lastNode.(*parse.TextNode); ok { + closingTag = strings.TrimSpace(t.String()) + } + + break + } + } + } + } + + return true + } + + exitFn := func(n parse.Node) { + ancestors = ancestors[:len(ancestors)-1] + } + + templateVisit(t.Root, enterFn, exitFn) + + return openingTag, closingTag +} + +// checks to see if the given field name is present in the pipeline of the given range node +func rangeUsesField(rangeNode *parse.RangeNode, field string) bool { + found := false + enterFn := func(n parse.Node) bool { + switch x := n.(type) { + case *parse.FieldNode: + if x.Ident[0] == field { + found = true + } + } + return true + } + templateVisit(rangeNode.BranchNode.Pipe, enterFn, nil) + return found +} diff --git a/server/thinking_test.go b/server/thinking_test.go new file mode 100644 index 000000000..a2055635e --- /dev/null +++ b/server/thinking_test.go @@ -0,0 +1,403 @@ +package server + +import ( + "testing" + "text/template" +) + +func TestExtractThinking(t *testing.T) { + tests := []struct { + in, wantContent, wantThink string + }{ + { + in: " internal world", + wantThink: "internal ", + wantContent: "world", + }, + { + in: "abc", + wantThink: "a", + wantContent: "bc", + }, + { + in: "no think", + wantThink: "", + wantContent: "no think", + }, + } + for i, tt := range tests { + parser := thinkingParser{ + openingTag: "", + closingTag: "", + } + gotThinking, gotContent := parser.addContent(tt.in) + if gotContent != tt.wantContent || gotThinking != tt.wantThink { + t.Errorf("case %d: got (%q,%q), want (%q,%q)", i, gotThinking, gotContent, tt.wantThink, tt.wantContent) + } + } +} + +func TestThinkingStreaming(t *testing.T) { + type step struct { + input string + wantThinking string + wantContent string + wantStateAfter thinkingState + } + + cases := []struct { + desc string + skip bool + steps []step + }{ + { + desc: "content without a thinking tag", + steps: []step{ + { + input: " abc", + wantThinking: "", + wantContent: " abc", + wantStateAfter: thinkingState_ThinkingDone, + }, + }, + }, + { + desc: "content before a thinking tag nerfs the thinking tag", + steps: []step{ + { + input: " abc def ghi", + wantThinking: "", + wantContent: " abc def ghi", + wantStateAfter: thinkingState_ThinkingDone, + }, + }, + }, + { + desc: "building up a thinking tag partially", + steps: []step{ + { + input: " a", + wantThinking: "a", + wantContent: "", + wantStateAfter: thinkingState_Thinking, + }, + }, + }, + { + desc: "partial closing tag", + steps: []step{ + { + input: "abcdef", + wantThinking: "", + wantContent: "def", + wantStateAfter: thinkingState_ThinkingDone, + }, + }, + }, + { + desc: "partial closing tag fakeout", + steps: []step{ + { + input: "abcdef", + wantThinking: "def", + wantContent: "", + wantStateAfter: thinkingState_Thinking, + }, + { + input: "ghijkl", + wantThinking: "", + wantContent: "jkl", + wantStateAfter: thinkingState_ThinkingDone, + }, + }, + }, + { + desc: "whitespace after thinking tag", + steps: []step{ + { + input: " abc\n\ndef", + wantThinking: "abc", + wantContent: "def", + wantStateAfter: thinkingState_ThinkingDone, + }, + }, + }, + { + desc: "whitespace after thinking tag (incremental)", + steps: []step{ + { + input: " abc", + wantThinking: "abc", + wantContent: "", + wantStateAfter: thinkingState_ThinkingDoneEatingWhitespace, + }, + { + input: "\n\ndef", + wantThinking: "", + wantContent: "def", + wantStateAfter: thinkingState_ThinkingDone, + }, + }, + }, + { + desc: "whitespace after thinking tag with content and more whitespace", + steps: []step{ + { + input: " abc\n\ndef ", + wantThinking: "abc", + wantContent: "def ", + wantStateAfter: thinkingState_ThinkingDone, + }, + { + input: " ghi", + wantThinking: "", + wantContent: " ghi", + wantStateAfter: thinkingState_ThinkingDone, + }, + }, + }, + { + desc: "token by token", + steps: []step{ + { + input: "", + wantThinking: "", + wantContent: "", + wantStateAfter: thinkingState_ThinkingStartedEatingWhitespace, + }, + { + input: "\n", + wantThinking: "", + wantContent: "", + wantStateAfter: thinkingState_ThinkingStartedEatingWhitespace, + }, + { + input: "", + wantThinking: "", + wantContent: "", + wantStateAfter: thinkingState_ThinkingDoneEatingWhitespace, + }, + { + input: "\n\n", + wantThinking: "", + wantContent: "", + wantStateAfter: thinkingState_ThinkingDoneEatingWhitespace, + }, + { + input: "Hi", + wantThinking: "", + wantContent: "Hi", + wantStateAfter: thinkingState_ThinkingDone, + }, + { + input: " there", + wantThinking: "", + wantContent: " there", + wantStateAfter: thinkingState_ThinkingDone, + }, + }, + }, + { + desc: "leading thinking whitespace", + steps: []step{ + { + input: " \t ", + wantThinking: "", + wantContent: "", + wantStateAfter: thinkingState_ThinkingStartedEatingWhitespace, + }, + { + input: " these are some ", + wantThinking: "these are some ", + wantContent: "", + wantStateAfter: thinkingState_Thinking, + }, + { + input: "thoughts ", + wantThinking: "thoughts ", + wantContent: "", + wantStateAfter: thinkingState_ThinkingDoneEatingWhitespace, + }, + { + input: " more content", + wantThinking: "", + wantContent: "more content", + wantStateAfter: thinkingState_ThinkingDone, + }, + }, + }, + } + + for _, c := range cases { + parser := thinkingParser{ + openingTag: "", + closingTag: "", + } + if c.skip { + continue + } + for i, step := range c.steps { + thinking, content := parser.addContent(step.input) + if content != step.wantContent || thinking != step.wantThinking { + t.Errorf("case %q (step %d): got (%q,%q), want (%q,%q)", c.desc, i, content, thinking, step.wantContent, step.wantThinking) + } + if parser.state != step.wantStateAfter { + t.Errorf("case %q (step %d): got state %s, want %s", c.desc, i, parser.state, step.wantStateAfter) + } + } + } +} + +func TestInferThinkingTags(t *testing.T) { + cases := []struct { + desc string + tmplString string + wantOpeningTag string + wantClosingTag string + }{ + { + desc: "basic", + tmplString: ` + {{ if .Thinking}} + /think + {{ end }} + {{- range $i, $_ := .Messages }} + {{- $last := eq (len (slice $.Messages $i)) 1 -}} + {{ if and $last .Thinking }} + {{ .Thinking }} + {{ end }} + {{ end }} + `, + wantOpeningTag: "", + wantClosingTag: "", + }, + { + desc: "doubly nested range", + tmplString: ` + {{ if .Thinking}} + /think + {{ end }} + {{- range $i, $_ := .Messages }} + {{- range $j, $_ := .NotMessages }} + {{- $last := eq (len (slice $.Messages $i)) 1 -}} + {{ if and $last .Thinking }} + {{ .Thinking }} + {{ end }} + {{ end }} + {{ end }} + `, + wantOpeningTag: "", + wantClosingTag: "", + }, + { + desc: "whitespace is trimmed", + tmplString: ` + {{ if .Thinking}} + /think + {{ end }} + {{- range $i, $_ := .Messages }} + {{- $last := eq (len (slice $.Messages $i)) 1 -}} + {{ if and $last .Thinking }} + Some text before {{ .Thinking }} Some text after + {{ end }} + {{ end }} + `, + wantOpeningTag: "Some text before", + wantClosingTag: "Some text after", + }, + { + desc: "qwen3", + tmplString: ` +{{- if or .System .Tools .Thinking }}<|im_start|>system +{{- if .System }} +{{ .System }} +{{- end }} +{{- if .Tools }} + +# Tools + +You may call one or more functions to assist with the user query. + +You are provided with function signatures within XML tags: + +{{- range .Tools }} +{"type": "function", "function": {{ .Function }}} +{{- end }} + + +For each function call, return a json object with function name and arguments within XML tags: + +{"name": , "arguments": } + +{{- end }} +{{- if .Thinking }} +/think +{{- else }} +/no_think +{{- end }}<|im_end|> +{{ end }} +{{- range $i, $_ := .Messages }} +{{- $last := eq (len (slice $.Messages $i)) 1 -}} +{{- if eq .Role "user" }}<|im_start|>user +{{ .Content }}<|im_end|> +{{ else if eq .Role "assistant" }}<|im_start|>assistant +{{ if and $last .Thinking }} +{{ .Thinking }} +{{ end }} +{{ if .Content }}{{ .Content }} +{{- else if .ToolCalls }} +{{ range .ToolCalls }}{"name": "{{ .Function.Name }}", "arguments": {{ .Function.Arguments }}} +{{ end }} +{{- end }}{{ if not $last }}<|im_end|> +{{ end }} +{{- else if eq .Role "tool" }}<|im_start|>user + +{{ .Content }} +<|im_end|> +{{ end }} +{{- if and (ne .Role "assistant") $last }}<|im_start|>assistant +{{ end }} +{{- end }} + `, + wantOpeningTag: "", + wantClosingTag: "", + }, + } + for _, c := range cases { + tmpl := template.Must(template.New("test").Parse(c.tmplString)) + openingTag, closingTag := inferThinkingTags(tmpl) + if openingTag != c.wantOpeningTag || closingTag != c.wantClosingTag { + t.Errorf("case %q: got (%q,%q), want (%q,%q)", c.desc, openingTag, closingTag, c.wantOpeningTag, c.wantClosingTag) + } + } +} diff --git a/template/template.go b/template/template.go index 5c886cac4..da910afbd 100644 --- a/template/template.go +++ b/template/template.go @@ -167,6 +167,10 @@ type Values struct { api.Tools Prompt string Suffix string + Think bool + // whether or not the user explicitly set the thinking flag (vs. it being + // implicitly false). Templates can't see whether `Think` is nil + IsThinkSet bool // forceLegacy is a flag used to test compatibility with legacy templates forceLegacy bool @@ -222,16 +226,20 @@ func (t *Template) Execute(w io.Writer, v Values) error { system, messages := collate(v.Messages) if v.Prompt != "" && v.Suffix != "" { return t.Template.Execute(w, map[string]any{ - "Prompt": v.Prompt, - "Suffix": v.Suffix, - "Response": "", + "Prompt": v.Prompt, + "Suffix": v.Suffix, + "Response": "", + "Think": v.Think, + "IsThinkSet": v.IsThinkSet, }) } else if !v.forceLegacy && slices.Contains(t.Vars(), "messages") { return t.Template.Execute(w, map[string]any{ - "System": system, - "Messages": messages, - "Tools": v.Tools, - "Response": "", + "System": system, + "Messages": messages, + "Tools": v.Tools, + "Response": "", + "Think": v.Think, + "IsThinkSet": v.IsThinkSet, }) } @@ -241,9 +249,11 @@ func (t *Template) Execute(w io.Writer, v Values) error { for _, m := range messages { execute := func() error { if err := t.Template.Execute(&b, map[string]any{ - "System": system, - "Prompt": prompt, - "Response": response, + "System": system, + "Prompt": prompt, + "Response": response, + "Think": v.Think, + "IsThinkSet": v.IsThinkSet, }); err != nil { return err } @@ -286,9 +296,11 @@ func (t *Template) Execute(w io.Writer, v Values) error { tree := parse.Tree{Root: nodes.(*parse.ListNode)} if err := template.Must(template.New("").AddParseTree("", &tree)).Execute(&b, map[string]any{ - "System": system, - "Prompt": prompt, - "Response": response, + "System": system, + "Prompt": prompt, + "Response": response, + "Think": v.Think, + "IsThinkSet": v.IsThinkSet, }); err != nil { return err } diff --git a/types/model/capability.go b/types/model/capability.go index fb8689403..cde23cee7 100644 --- a/types/model/capability.go +++ b/types/model/capability.go @@ -8,6 +8,7 @@ const ( CapabilityInsert = Capability("insert") CapabilityVision = Capability("vision") CapabilityEmbedding = Capability("embedding") + CapabilityThinking = Capability("thinking") ) func (c Capability) String() string { From f15ffc432061e3d96b3412219a3a0f673b579a12 Mon Sep 17 00:00:00 2001 From: Jesse Gross Date: Tue, 13 May 2025 17:26:46 -0700 Subject: [PATCH 22/54] llm: Make "POST predict" error message more informative "POST predict" basically means that the runner has crashed, which can have many reasons. However, many people think this is a specific error and either report only this message or group together unrelated bugs. This replaces it with a more friendly and helpful message. --- llm/server.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/llm/server.go b/llm/server.go index 4abb569fb..373f6faef 100644 --- a/llm/server.go +++ b/llm/server.go @@ -797,7 +797,8 @@ func (s *llmServer) Completion(ctx context.Context, req CompletionRequest, fn fu res, err := http.DefaultClient.Do(serverReq) if err != nil { - return fmt.Errorf("POST predict: %v", err) + slog.Error("post predict", "error", err) + return errors.New("model runner has unexpectedly stopped, this may be due to resource limitations or an internal error, check ollama server logs for details") } defer res.Body.Close() From aaa7818000c42a82fc030212c35ef83f9799efd7 Mon Sep 17 00:00:00 2001 From: Jesse Gross Date: Thu, 24 Apr 2025 11:48:49 -0700 Subject: [PATCH 23/54] ggml: Export GPU UUIDs This enables matching up devices and information reported by the backend with system management libraries such as nvml to get accurate free memory reporting. --- .../patches/0017-ggml-Export-GPU-UUIDs.patch | 102 ++++++++++++++++++ ml/backend.go | 8 ++ ml/backend/ggml/ggml.go | 6 ++ ml/backend/ggml/ggml/include/ggml-backend.h | 1 + .../ggml/ggml/src/ggml-cuda/ggml-cuda.cu | 33 ++++++ .../ggml/ggml/src/ggml-metal/ggml-metal.m | 1 + 6 files changed, 151 insertions(+) create mode 100644 llama/patches/0017-ggml-Export-GPU-UUIDs.patch diff --git a/llama/patches/0017-ggml-Export-GPU-UUIDs.patch b/llama/patches/0017-ggml-Export-GPU-UUIDs.patch new file mode 100644 index 000000000..a2539034c --- /dev/null +++ b/llama/patches/0017-ggml-Export-GPU-UUIDs.patch @@ -0,0 +1,102 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Jesse Gross +Date: Thu, 24 Apr 2025 14:48:51 -0700 +Subject: [PATCH] ggml: Export GPU UUIDs + +This enables matching up devices and information reported by the backend +with tools (e.g. nvidia-smi) and system management libraries (e.g. nvml). +--- + ggml/include/ggml-backend.h | 1 + + ggml/src/ggml-cuda/ggml-cuda.cu | 33 ++++++++++++++++++++++++++++++++ + ggml/src/ggml-metal/ggml-metal.m | 1 + + 3 files changed, 35 insertions(+) + +diff --git a/ggml/include/ggml-backend.h b/ggml/include/ggml-backend.h +index 74e46716..a880df33 100644 +--- a/ggml/include/ggml-backend.h ++++ b/ggml/include/ggml-backend.h +@@ -152,6 +152,7 @@ extern "C" { + struct ggml_backend_dev_props { + const char * name; + const char * description; ++ const char * uuid; + size_t memory_free; + size_t memory_total; + enum ggml_backend_dev_type type; +diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu +index cb0d8528..4c829153 100644 +--- a/ggml/src/ggml-cuda/ggml-cuda.cu ++++ b/ggml/src/ggml-cuda/ggml-cuda.cu +@@ -2884,6 +2884,7 @@ struct ggml_backend_cuda_device_context { + int device; + std::string name; + std::string description; ++ std::string uuid; + }; + + static const char * ggml_backend_cuda_device_get_name(ggml_backend_dev_t dev) { +@@ -2896,6 +2897,11 @@ static const char * ggml_backend_cuda_device_get_description(ggml_backend_dev_t + return ctx->description.c_str(); + } + ++static const char * ggml_backend_cuda_device_get_uuid(ggml_backend_dev_t dev) { ++ ggml_backend_cuda_device_context * ctx = (ggml_backend_cuda_device_context *)dev->context; ++ return ctx->uuid.c_str(); ++} ++ + static void ggml_backend_cuda_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) { + ggml_backend_cuda_device_context * ctx = (ggml_backend_cuda_device_context *)dev->context; + ggml_cuda_set_device(ctx->device); +@@ -2910,6 +2916,7 @@ static enum ggml_backend_dev_type ggml_backend_cuda_device_get_type(ggml_backend + static void ggml_backend_cuda_device_get_props(ggml_backend_dev_t dev, ggml_backend_dev_props * props) { + props->name = ggml_backend_cuda_device_get_name(dev); + props->description = ggml_backend_cuda_device_get_description(dev); ++ props->uuid = ggml_backend_cuda_device_get_uuid(dev); + props->type = ggml_backend_cuda_device_get_type(dev); + ggml_backend_cuda_device_get_memory(dev, &props->memory_free, &props->memory_total); + +@@ -3458,6 +3465,32 @@ ggml_backend_reg_t ggml_backend_cuda_reg() { + CUDA_CHECK(cudaGetDeviceProperties(&prop, i)); + dev_ctx->description = prop.name; + ++ #if !defined(GGML_USE_HIP) ++ char uuid[64]; ++ snprintf(uuid, sizeof(uuid), ++ "GPU-%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x", ++ (unsigned char)prop.uuid.bytes[0], ++ (unsigned char)prop.uuid.bytes[1], ++ (unsigned char)prop.uuid.bytes[2], ++ (unsigned char)prop.uuid.bytes[3], ++ (unsigned char)prop.uuid.bytes[4], ++ (unsigned char)prop.uuid.bytes[5], ++ (unsigned char)prop.uuid.bytes[6], ++ (unsigned char)prop.uuid.bytes[7], ++ (unsigned char)prop.uuid.bytes[8], ++ (unsigned char)prop.uuid.bytes[9], ++ (unsigned char)prop.uuid.bytes[10], ++ (unsigned char)prop.uuid.bytes[11], ++ (unsigned char)prop.uuid.bytes[12], ++ (unsigned char)prop.uuid.bytes[13], ++ (unsigned char)prop.uuid.bytes[14], ++ (unsigned char)prop.uuid.bytes[15] ++ ); ++ dev_ctx->uuid = uuid; ++ #else ++ dev_ctx->uuid = "GPU-" + std::string(prop.uuid.bytes, 16); ++ #endif ++ + ggml_backend_dev_t dev = new ggml_backend_device { + /* .iface = */ ggml_backend_cuda_device_interface, + /* .reg = */ ®, +diff --git a/ggml/src/ggml-metal/ggml-metal.m b/ggml/src/ggml-metal/ggml-metal.m +index 1b56f858..ee4f2dcb 100644 +--- a/ggml/src/ggml-metal/ggml-metal.m ++++ b/ggml/src/ggml-metal/ggml-metal.m +@@ -5703,6 +5703,7 @@ static enum ggml_backend_dev_type ggml_backend_metal_device_get_type(ggml_backen + static void ggml_backend_metal_device_get_props(ggml_backend_dev_t dev, struct ggml_backend_dev_props * props) { + props->name = ggml_backend_metal_device_get_name(dev); + props->description = ggml_backend_metal_device_get_description(dev); ++ props->uuid = "0"; + props->type = ggml_backend_metal_device_get_type(dev); + ggml_backend_metal_device_get_memory(dev, &props->memory_free, &props->memory_total); + props->caps = (struct ggml_backend_dev_caps) { diff --git a/ml/backend.go b/ml/backend.go index 65f169486..2df6c8923 100644 --- a/ml/backend.go +++ b/ml/backend.go @@ -124,6 +124,10 @@ type DeviceMemory struct { // may not be persistent across instances of the runner. Name string + // UUID is a unique persistent identifier for the device for matching + // with system management libraries + UUID string + // Weights is the per-layer memory needed for the model weights. Weights []Memory @@ -152,6 +156,10 @@ func (m DeviceMemory) LogValue() slog.Value { attrs = append(attrs, slog.Any("Graph", m.Graph)) } + if len(attrs) > 0 && m.UUID != "" { + attrs = append([]slog.Attr{slog.String("UUID", m.UUID)}, attrs...) + } + return slog.GroupValue(attrs...) } diff --git a/ml/backend/ggml/ggml.go b/ml/backend/ggml/ggml.go index 76172ae1a..5a9fe67e5 100644 --- a/ml/backend/ggml/ggml.go +++ b/ml/backend/ggml/ggml.go @@ -136,6 +136,9 @@ func New(modelPath string, params ml.BackendParams) (ml.Backend, error) { } requiredMemory.CPU.Name = C.GoString(C.ggml_backend_dev_name(cpuDeviceBufferType.d)) + var props C.struct_ggml_backend_dev_props + C.ggml_backend_dev_get_props(cpuDeviceBufferType.d, &props) + requiredMemory.CPU.UUID = C.GoString(props.uuid) requiredMemory.CPU.Weights = make([]ml.Memory, blocks+1) requiredMemory.CPU.Cache = make([]ml.Memory, blocks+1) @@ -150,6 +153,9 @@ func New(modelPath string, params ml.BackendParams) (ml.Backend, error) { }) btDeviceMemory[bt] = &requiredMemory.GPUs[i] requiredMemory.GPUs[i].Name = C.GoString(C.ggml_backend_dev_name(d)) + var props C.struct_ggml_backend_dev_props + C.ggml_backend_dev_get_props(d, &props) + requiredMemory.GPUs[i].UUID = C.GoString(props.uuid) requiredMemory.GPUs[i].Weights = make([]ml.Memory, blocks+1) requiredMemory.GPUs[i].Cache = make([]ml.Memory, blocks+1) } diff --git a/ml/backend/ggml/ggml/include/ggml-backend.h b/ml/backend/ggml/ggml/include/ggml-backend.h index 74e467163..a880df33e 100644 --- a/ml/backend/ggml/ggml/include/ggml-backend.h +++ b/ml/backend/ggml/ggml/include/ggml-backend.h @@ -152,6 +152,7 @@ extern "C" { struct ggml_backend_dev_props { const char * name; const char * description; + const char * uuid; size_t memory_free; size_t memory_total; enum ggml_backend_dev_type type; diff --git a/ml/backend/ggml/ggml/src/ggml-cuda/ggml-cuda.cu b/ml/backend/ggml/ggml/src/ggml-cuda/ggml-cuda.cu index cb0d8528d..4c8291532 100644 --- a/ml/backend/ggml/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ml/backend/ggml/ggml/src/ggml-cuda/ggml-cuda.cu @@ -2884,6 +2884,7 @@ struct ggml_backend_cuda_device_context { int device; std::string name; std::string description; + std::string uuid; }; static const char * ggml_backend_cuda_device_get_name(ggml_backend_dev_t dev) { @@ -2896,6 +2897,11 @@ static const char * ggml_backend_cuda_device_get_description(ggml_backend_dev_t return ctx->description.c_str(); } +static const char * ggml_backend_cuda_device_get_uuid(ggml_backend_dev_t dev) { + ggml_backend_cuda_device_context * ctx = (ggml_backend_cuda_device_context *)dev->context; + return ctx->uuid.c_str(); +} + static void ggml_backend_cuda_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) { ggml_backend_cuda_device_context * ctx = (ggml_backend_cuda_device_context *)dev->context; ggml_cuda_set_device(ctx->device); @@ -2910,6 +2916,7 @@ static enum ggml_backend_dev_type ggml_backend_cuda_device_get_type(ggml_backend static void ggml_backend_cuda_device_get_props(ggml_backend_dev_t dev, ggml_backend_dev_props * props) { props->name = ggml_backend_cuda_device_get_name(dev); props->description = ggml_backend_cuda_device_get_description(dev); + props->uuid = ggml_backend_cuda_device_get_uuid(dev); props->type = ggml_backend_cuda_device_get_type(dev); ggml_backend_cuda_device_get_memory(dev, &props->memory_free, &props->memory_total); @@ -3458,6 +3465,32 @@ ggml_backend_reg_t ggml_backend_cuda_reg() { CUDA_CHECK(cudaGetDeviceProperties(&prop, i)); dev_ctx->description = prop.name; + #if !defined(GGML_USE_HIP) + char uuid[64]; + snprintf(uuid, sizeof(uuid), + "GPU-%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x", + (unsigned char)prop.uuid.bytes[0], + (unsigned char)prop.uuid.bytes[1], + (unsigned char)prop.uuid.bytes[2], + (unsigned char)prop.uuid.bytes[3], + (unsigned char)prop.uuid.bytes[4], + (unsigned char)prop.uuid.bytes[5], + (unsigned char)prop.uuid.bytes[6], + (unsigned char)prop.uuid.bytes[7], + (unsigned char)prop.uuid.bytes[8], + (unsigned char)prop.uuid.bytes[9], + (unsigned char)prop.uuid.bytes[10], + (unsigned char)prop.uuid.bytes[11], + (unsigned char)prop.uuid.bytes[12], + (unsigned char)prop.uuid.bytes[13], + (unsigned char)prop.uuid.bytes[14], + (unsigned char)prop.uuid.bytes[15] + ); + dev_ctx->uuid = uuid; + #else + dev_ctx->uuid = "GPU-" + std::string(prop.uuid.bytes, 16); + #endif + ggml_backend_dev_t dev = new ggml_backend_device { /* .iface = */ ggml_backend_cuda_device_interface, /* .reg = */ ®, diff --git a/ml/backend/ggml/ggml/src/ggml-metal/ggml-metal.m b/ml/backend/ggml/ggml/src/ggml-metal/ggml-metal.m index 1b56f858c..ee4f2dcb0 100644 --- a/ml/backend/ggml/ggml/src/ggml-metal/ggml-metal.m +++ b/ml/backend/ggml/ggml/src/ggml-metal/ggml-metal.m @@ -5703,6 +5703,7 @@ static enum ggml_backend_dev_type ggml_backend_metal_device_get_type(ggml_backen static void ggml_backend_metal_device_get_props(ggml_backend_dev_t dev, struct ggml_backend_dev_props * props) { props->name = ggml_backend_metal_device_get_name(dev); props->description = ggml_backend_metal_device_get_description(dev); + props->uuid = "0"; props->type = ggml_backend_metal_device_get_type(dev); ggml_backend_metal_device_get_memory(dev, &props->memory_free, &props->memory_total); props->caps = (struct ggml_backend_dev_caps) { From 65f10c2823d540837a9e79202522957194377735 Mon Sep 17 00:00:00 2001 From: Parth Sareen Date: Fri, 30 May 2025 15:18:09 -0700 Subject: [PATCH 24/54] tools: resiliency upgrade to name and arg extraction from template (#10917) --- tools/tools_utils.go | 37 +++++------ tools/tools_utils_test.go | 131 ++++++++++++++++++++++++-------------- 2 files changed, 98 insertions(+), 70 deletions(-) diff --git a/tools/tools_utils.go b/tools/tools_utils.go index 48531b789..b6f80729e 100644 --- a/tools/tools_utils.go +++ b/tools/tools_utils.go @@ -166,31 +166,26 @@ func extractToolArgs(tmpl *gotmpl.Template) (name, arguments string, err error) return "", "", err } - var obj any - err = json.Unmarshal(b.Bytes(), &obj) - if err != nil { + // Extract JSON object between curly braces + // JSON arrays are also valid as they will not be repeated in the template + output := b.String() + start := strings.Index(output, "{") + end := strings.LastIndex(output, "}") + if start == -1 || end == -1 || start > end { + return "", "", errors.New("no valid JSON object found in template output") + } + jsonStr := output[start : end+1] + + var obj map[string]any + if err := json.Unmarshal([]byte(jsonStr), &obj); err != nil { return "", "", err } - var objs []map[string]any - switch v := obj.(type) { - case map[string]any: - objs = []map[string]any{v} - case []map[string]any: - objs = v - case []any: - objs = collect(v) - } - if len(objs) == 0 { - return "", "", errors.New("no template objects found") - } - - // find the keys that correspond to the name and arguments fields - for k, v := range objs[0] { - switch v.(type) { - case string: + // Find name and arguments fields + for k, v := range obj { + if str, ok := v.(string); ok && str == "@@name@@" { name = k - case map[string]any: + } else if _, ok := v.(map[string]any); ok { arguments = k } } diff --git a/tools/tools_utils_test.go b/tools/tools_utils_test.go index 769183b73..e346117a9 100644 --- a/tools/tools_utils_test.go +++ b/tools/tools_utils_test.go @@ -271,74 +271,99 @@ func TestExtractToolArgs(t *testing.T) { cases := []struct { name string template string - want string - ok bool + wantName string + wantArgs string + wantErr bool }{ { - name: "basic tool call with text after", - template: `{{if .ToolCalls}}tool response{{end}}`, - want: "tool response", - ok: true, + name: "basic tool call", + template: `{{ range .ToolCalls }} +{"name": "{{ .Function.Name }}", "parameters": {{ .Function.Arguments }}}{{ end }}`, + wantName: "name", + wantArgs: "parameters", + wantErr: false, }, { - name: "tool call with mixed content after", - template: `{{if .ToolCalls}}{{.Something}}{{end}}`, - want: "", - ok: true, + name: "tool call with whitespace", + template: `{{range .ToolCalls}} + {"name": "{{.Function.Name}}", "parameters": {{.Function.Arguments}}} +{{end}}`, + wantName: "name", + wantArgs: "parameters", + wantErr: false, }, { - name: "tool call with no text after", - template: `{{if .ToolCalls}}{{.Something}}{{end}}`, - want: "", - ok: true, - }, - { - name: "nested tool call", - template: `{{if .Something}}{{if .ToolCalls}}[TOOL_CALL]{{end}}{{end}}`, - want: "[TOOL_CALL]", - ok: true, + name: "tool call with extra content", + template: `Before {{range .ToolCalls}} +{"name": "{{.Function.Name}}", "arguments": {{.Function.Arguments}}}{{end}} After`, + wantName: "name", + wantArgs: "arguments", + wantErr: false, }, { name: "no tool calls", template: `{{if .Something}}no tools here{{end}}`, - want: "", - ok: false, + wantName: "", + wantArgs: "", + wantErr: true, }, { name: "empty template", template: ``, - want: "", - ok: false, + wantName: "", + wantArgs: "", + wantErr: true, }, { - name: "multiple tool calls sections", - template: `{{if .ToolCalls}}first{{end}}{{if .ToolCalls}}second{{end}}`, - want: "first", - ok: true, + name: "prefix within tool call", + template: `{{- if .ToolCalls }} +{{ range .ToolCalls }} + +{"name": "{{ .Function.Name }}", "arguments": {{ .Function.Arguments }}} +{{ end }}{{- end }}`, + wantName: "name", + wantArgs: "arguments", + wantErr: false, }, { - name: "range over tool calls", - template: `{{if .ToolCalls}}{{range .ToolCalls}}tool{{end}}{{end}}`, - want: "", - ok: true, + name: "JSON array", + template: `{{ range .ToolCalls }} +[{"name": "{{ .Function.Name }}", "arguments": {{ .Function.Arguments }}}]{{ end }}`, + wantName: "name", + wantArgs: "arguments", + wantErr: false, }, { - name: "tool calls with pipe delimiters", - template: `{{if .ToolCalls}}<|tool|>{{end}}`, - want: "<|tool|>", - ok: true, + name: "invalid JSON", + template: `{{ range .ToolCalls }} +{"name": "{{ .Function.Name }}", "arguments": {{ .Function.Arguments }}, invalid}{{ end }}`, + wantName: "", + wantArgs: "", + wantErr: true, }, { - name: "tool calls with nested template", - template: `{{if .ToolCalls}}{{template "tool" .}}{{end}}`, - want: "", - ok: true, + name: "missing name field", + template: `{{ range .ToolCalls }} +{"parameters": {{ .Function.Arguments }}}{{ end }}`, + wantName: "", + wantArgs: "", + wantErr: true, }, { - name: "tool calls with whitespace variations", - template: `{{if .ToolCalls}} tool {{end}}`, - want: " tool ", - ok: true, + name: "missing arguments field", + template: `{{ range .ToolCalls }} +{"name": "{{ .Function.Name }}"}{{ end }}`, + wantName: "", + wantArgs: "", + wantErr: true, + }, + { + name: "malformed JSON", + template: `{{ range .ToolCalls }} +{"name": {{ .Function.Name }}, "arguments": {{ .Function.Arguments }}{{ end }}`, + wantName: "", + wantArgs: "", + wantErr: true, }, } @@ -349,12 +374,20 @@ func TestExtractToolArgs(t *testing.T) { t.Fatalf("failed to parse template: %v", err) } - got, ok := extractToolCallsFormat(tmpl) - if got != tt.want { - t.Errorf("TextAfterToolCalls() got = %q, want %q", got, tt.want) + gotName, gotArgs, err := extractToolArgs(tmpl) + if (err != nil) != tt.wantErr { + t.Errorf("extractToolArgs() error = %v, wantErr %v", err, tt.wantErr) + return } - if ok != tt.ok { - t.Errorf("TextAfterToolCalls() ok = %v, want %v", ok, tt.ok) + if err != nil { + return + } + + if gotName != tt.wantName { + t.Errorf("extractToolArgs() gotName = %q, want %q", gotName, tt.wantName) + } + if gotArgs != tt.wantArgs { + t.Errorf("extractToolArgs() gotArgs = %q, want %q", gotArgs, tt.wantArgs) } }) } From 5c42800fca4da07d1c362c0f190429993e53c3b5 Mon Sep 17 00:00:00 2001 From: HardCodeDev Date: Sat, 31 May 2025 06:50:16 +0400 Subject: [PATCH 25/54] readme: add SimpleOllamaUnity to community integrations (#10817) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index d7cf5bfec..22de41e63 100644 --- a/README.md +++ b/README.md @@ -587,6 +587,7 @@ See the [API documentation](./docs/api.md) for all endpoints. - [Simple-Discord-AI](https://github.com/zyphixor/simple-discord-ai) - [LLM Telegram Bot](https://github.com/innightwolfsleep/llm_telegram_bot) (telegram bot, primary for RP. Oobabooga-like buttons, [A1111](https://github.com/AUTOMATIC1111/stable-diffusion-webui) API integration e.t.c) - [mcp-llm](https://github.com/sammcj/mcp-llm) (MCP Server to allow LLMs to call other LLMs) +- [SimpleOllamaUnity](https://github.com/HardCodeDev777/SimpleOllamaUnity) (Unity Engine extension for communicating with Ollama in a few lines of code. Also works at runtime) - [UnityCodeLama](https://github.com/HardCodeDev777/UnityCodeLama) (Unity Edtior tool to analyze scripts via Ollama) ### Supported backends From 09430011936652cf55925184aaed6f2cebf62a75 Mon Sep 17 00:00:00 2001 From: JasonHonKL <148705846+JasonHonKL@users.noreply.github.com> Date: Thu, 5 Jun 2025 02:39:48 +0800 Subject: [PATCH 26/54] server: add model capabilities to the list endpoint (#10174) --- api/types.go | 13 +++++++------ docs/api.md | 29 +++++++++++++++++++---------- server/routes.go | 14 +++++++++++--- 3 files changed, 37 insertions(+), 19 deletions(-) diff --git a/api/types.go b/api/types.go index 94d492006..a1f896db3 100644 --- a/api/types.go +++ b/api/types.go @@ -457,12 +457,13 @@ type ProcessResponse struct { // ListModelResponse is a single model description in [ListResponse]. type ListModelResponse struct { - Name string `json:"name"` - Model string `json:"model"` - ModifiedAt time.Time `json:"modified_at"` - Size int64 `json:"size"` - Digest string `json:"digest"` - Details ModelDetails `json:"details,omitempty"` + Name string `json:"name"` + Model string `json:"model"` + ModifiedAt time.Time `json:"modified_at"` + Size int64 `json:"size"` + Digest string `json:"digest"` + Capabilities []model.Capability `json:"capabilities,omitempty"` + Details ModelDetails `json:"details,omitempty"` } // ProcessModelResponse is a single model description in [ProcessResponse]. diff --git a/docs/api.md b/docs/api.md index 11eaf73ab..31e18bd5d 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1157,11 +1157,15 @@ A single JSON object will be returned. { "models": [ { - "name": "deepseek-r1:latest", - "model": "deepseek-r1:latest", - "modified_at": "2025-05-10T08:06:48.639712648-07:00", - "size": 4683075271, - "digest": "0a8c266910232fd3291e71e5ba1e058cc5af9d411192cf88b6d30e92b6e73163", + + "model": "codellama:13b", + "modified_at": "2023-11-04T14:56:49.277302595-07:00", + "size": 7365960935, + "digest": "9f438cb9cd581fc025612d27f7c1a6669ff83a8bb0ed86c94fcf4c5440555697", + "capabilities": [ + "completion" + ], + "details": { "parent_model": "", "format": "gguf", @@ -1174,11 +1178,16 @@ A single JSON object will be returned. } }, { - "name": "llama3.2:latest", - "model": "llama3.2:latest", - "modified_at": "2025-05-04T17:37:44.706015396-07:00", - "size": 2019393189, - "digest": "a80c4f17acd55265feec403c7aef86be0c25983ab279d83f3bcd3abbcb5b8b72", + + "model": "llama4:latest", + "modified_at": "2023-12-07T09:32:18.757212583-08:00", + "size": 3825819519, + "digest": "fe938a131f40e6f6d40083c9f0f430a515233eb2edaa6d72eb85c50d64f2300e", + "capabilities": [ + "completion", + "vision" + ], + "details": { "parent_model": "", "format": "gguf", diff --git a/server/routes.go b/server/routes.go index 236f92e22..924ba06ca 100644 --- a/server/routes.go +++ b/server/routes.go @@ -928,8 +928,7 @@ func (s *Server) ListHandler(c *gin.Context) { } } - // tag should never be masked - models = append(models, api.ListModelResponse{ + r := api.ListModelResponse{ Model: n.DisplayShortest(), Name: n.DisplayShortest(), Size: m.Size(), @@ -942,7 +941,16 @@ func (s *Server) ListHandler(c *gin.Context) { ParameterSize: cf.ModelType, QuantizationLevel: cf.FileType, }, - }) + } + + model, err := GetModel(n.String()) + if err != nil { + slog.Warn("bad model details", "name", n, "error", err) + } else { + r.Capabilities = model.Capabilities() + } + + models = append(models, r) } slices.SortStableFunc(models, func(i, j api.ListModelResponse) int { From 0683efa6379ba69384bb5876f1013d9ef38f1ab0 Mon Sep 17 00:00:00 2001 From: Devon Rifkin Date: Thu, 5 Jun 2025 10:22:32 -0700 Subject: [PATCH 27/54] export ThinkingParser --- server/routes.go | 28 ++++++++++++++-------------- server/thinking.go | 26 +++++++++++++------------- server/thinking_test.go | 16 ++++++++-------- 3 files changed, 35 insertions(+), 35 deletions(-) diff --git a/server/routes.go b/server/routes.go index 924ba06ca..d03ac2ece 100644 --- a/server/routes.go +++ b/server/routes.go @@ -282,12 +282,12 @@ func (s *Server) GenerateHandler(c *gin.Context) { prompt = b.String() } - var thinkingState *thinkingParser + var thinkingState *ThinkingParser openingTag, closingTag := inferThinkingTags(m.Template.Template) if req.Think != nil && *req.Think && openingTag != "" && closingTag != "" { - thinkingState = &thinkingParser{ - openingTag: openingTag, - closingTag: closingTag, + thinkingState = &ThinkingParser{ + OpeningTag: openingTag, + ClosingTag: closingTag, } } @@ -316,7 +316,7 @@ func (s *Server) GenerateHandler(c *gin.Context) { } if thinkingState != nil { - thinking, content := thinkingState.addContent(cr.Content) + thinking, content := thinkingState.AddContent(cr.Content) res.Thinking = thinking res.Response = content } @@ -1522,12 +1522,12 @@ func (s *Server) ChatHandler(c *gin.Context) { return } - var thinkingState *thinkingParser + var thinkingState *ThinkingParser openingTag, closingTag := inferThinkingTags(m.Template.Template) if req.Think != nil && *req.Think && openingTag != "" && closingTag != "" { - thinkingState = &thinkingParser{ - openingTag: openingTag, - closingTag: closingTag, + thinkingState = &ThinkingParser{ + OpeningTag: openingTag, + ClosingTag: closingTag, } } @@ -1565,7 +1565,7 @@ func (s *Server) ChatHandler(c *gin.Context) { } if thinkingState != nil { - thinkingContent, remainingContent := thinkingState.addContent(res.Message.Content) + thinkingContent, remainingContent := thinkingState.AddContent(res.Message.Content) if thinkingContent == "" && remainingContent == "" && !r.Done { // need to accumulate more to decide what to send return @@ -1676,11 +1676,11 @@ func filterThinkTags(msgs []api.Message, m *Model) []api.Message { // change the user output), we should probably perform this filtering // for all thinking models (not just qwen3 & deepseek-r1) since it tends // to save tokens and improve quality. - thinkingState := &thinkingParser{ - openingTag: "", - closingTag: "", + thinkingState := &ThinkingParser{ + OpeningTag: "", + ClosingTag: "", } - _, content := thinkingState.addContent(msg.Content) + _, content := thinkingState.AddContent(msg.Content) msgs[i].Content = content } } diff --git a/server/thinking.go b/server/thinking.go index 2213b6b6e..4ef3c1848 100644 --- a/server/thinking.go +++ b/server/thinking.go @@ -46,17 +46,17 @@ func (s thinkingState) String() string { } } -type thinkingParser struct { +type ThinkingParser struct { state thinkingState - openingTag string - closingTag string + OpeningTag string + ClosingTag string acc strings.Builder } -// addContent returns the thinking content and the non-thinking content that +// AddContent returns the thinking content and the non-thinking content that // should be immediately sent to the user. It will internally buffer if it needs // to see more raw content to disambiguate -func (s *thinkingParser) addContent(content string) (string, string) { +func (s *ThinkingParser) AddContent(content string) (string, string) { s.acc.WriteString(content) var thinkingSb, remainingSb strings.Builder @@ -76,12 +76,12 @@ func (s *thinkingParser) addContent(content string) (string, string) { } // the additional bool return is true iff we should continue eating -func eat(s *thinkingParser) (string, string, bool) { +func eat(s *ThinkingParser) (string, string, bool) { switch s.state { case thinkingState_LookingForOpening: trimmed := strings.TrimLeftFunc(s.acc.String(), unicode.IsSpace) - if strings.HasPrefix(trimmed, s.openingTag) { - after := strings.Join(strings.Split(trimmed, s.openingTag)[1:], s.openingTag) + if strings.HasPrefix(trimmed, s.OpeningTag) { + after := strings.Join(strings.Split(trimmed, s.OpeningTag)[1:], s.OpeningTag) after = strings.TrimLeftFunc(after, unicode.IsSpace) // after might contain more than just thinking tokens, so we continue // parsing instead of returning it as thinking tokens here @@ -93,7 +93,7 @@ func eat(s *thinkingParser) (string, string, bool) { s.state = thinkingState_Thinking } return "", "", true - } else if strings.HasPrefix(s.openingTag, trimmed) { + } else if strings.HasPrefix(s.OpeningTag, trimmed) { // partial opening seen, so let's keep accumulating return "", "", false } else if trimmed == "" { @@ -119,10 +119,10 @@ func eat(s *thinkingParser) (string, string, bool) { } case thinkingState_Thinking: acc := s.acc.String() - if strings.Contains(acc, s.closingTag) { - split := strings.Split(acc, s.closingTag) + if strings.Contains(acc, s.ClosingTag) { + split := strings.Split(acc, s.ClosingTag) thinking := split[0] - remaining := strings.Join(split[1:], s.closingTag) + remaining := strings.Join(split[1:], s.ClosingTag) remaining = strings.TrimLeftFunc(remaining, unicode.IsSpace) s.acc.Reset() if remaining == "" { @@ -131,7 +131,7 @@ func eat(s *thinkingParser) (string, string, bool) { s.state = thinkingState_ThinkingDone } return thinking, remaining, false - } else if overlapLen := overlap(acc, s.closingTag); overlapLen > 0 { + } else if overlapLen := overlap(acc, s.ClosingTag); overlapLen > 0 { thinking := acc[:len(acc)-overlapLen] remaining := acc[len(acc)-overlapLen:] s.acc.Reset() diff --git a/server/thinking_test.go b/server/thinking_test.go index a2055635e..90d3f961d 100644 --- a/server/thinking_test.go +++ b/server/thinking_test.go @@ -26,11 +26,11 @@ func TestExtractThinking(t *testing.T) { }, } for i, tt := range tests { - parser := thinkingParser{ - openingTag: "", - closingTag: "", + parser := ThinkingParser{ + OpeningTag: "", + ClosingTag: "", } - gotThinking, gotContent := parser.addContent(tt.in) + gotThinking, gotContent := parser.AddContent(tt.in) if gotContent != tt.wantContent || gotThinking != tt.wantThink { t.Errorf("case %d: got (%q,%q), want (%q,%q)", i, gotThinking, gotContent, tt.wantThink, tt.wantContent) } @@ -259,15 +259,15 @@ func TestThinkingStreaming(t *testing.T) { } for _, c := range cases { - parser := thinkingParser{ - openingTag: "", - closingTag: "", + parser := ThinkingParser{ + OpeningTag: "", + ClosingTag: "", } if c.skip { continue } for i, step := range c.steps { - thinking, content := parser.addContent(step.input) + thinking, content := parser.AddContent(step.input) if content != step.wantContent || thinking != step.wantThinking { t.Errorf("case %q (step %d): got (%q,%q), want (%q,%q)", c.desc, i, content, thinking, step.wantContent, step.wantThinking) } From c6a6d7294dd50b9216918fe72fd92bc4ae572ac0 Mon Sep 17 00:00:00 2001 From: Hunter Wittenborn Date: Fri, 6 Jun 2025 11:07:29 -0500 Subject: [PATCH 28/54] docs: fix typo in development.md (#10998) --- docs/development.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/development.md b/docs/development.md index cf6d91e27..24bcba194 100644 --- a/docs/development.md +++ b/docs/development.md @@ -118,7 +118,7 @@ To run tests, use `go test`: go test ./... ``` -> NOTE: In rare cirumstances, you may nedd to change a package using the new +> NOTE: In rare cirumstances, you may need to change a package using the new > "synctest" package in go1.24. > > If you do not have the "synctest" package enabled, you will not see build or From a3b6886b7da0339e63ebf41e6ba5c6b06438a123 Mon Sep 17 00:00:00 2001 From: Devon Rifkin Date: Fri, 6 Jun 2025 12:02:20 -0700 Subject: [PATCH 29/54] move thinking logic into its own package (#10990) move thinking logic into its own package --- server/images.go | 3 +- server/routes.go | 15 +- server/thinking.go => thinking/parser.go | 137 +----------------- .../parser_test.go | 131 +---------------- thinking/template.go | 134 +++++++++++++++++ thinking/template_test.go | 130 +++++++++++++++++ 6 files changed, 281 insertions(+), 269 deletions(-) rename server/thinking.go => thinking/parser.go (59%) rename server/thinking_test.go => thinking/parser_test.go (66%) create mode 100644 thinking/template.go create mode 100644 thinking/template_test.go diff --git a/server/images.go b/server/images.go index 58fb87dcc..d6cceff4c 100644 --- a/server/images.go +++ b/server/images.go @@ -26,6 +26,7 @@ import ( "github.com/ollama/ollama/fs/ggml" "github.com/ollama/ollama/parser" "github.com/ollama/ollama/template" + "github.com/ollama/ollama/thinking" "github.com/ollama/ollama/types/model" "github.com/ollama/ollama/version" ) @@ -113,7 +114,7 @@ func (m *Model) Capabilities() []model.Capability { } // Check for thinking capability - openingTag, closingTag := inferThinkingTags(m.Template.Template) + openingTag, closingTag := thinking.InferTags(m.Template.Template) if openingTag != "" && closingTag != "" { capabilities = append(capabilities, model.CapabilityThinking) } diff --git a/server/routes.go b/server/routes.go index d03ac2ece..70cb6cef9 100644 --- a/server/routes.go +++ b/server/routes.go @@ -37,6 +37,7 @@ import ( "github.com/ollama/ollama/server/internal/client/ollama" "github.com/ollama/ollama/server/internal/registry" "github.com/ollama/ollama/template" + "github.com/ollama/ollama/thinking" "github.com/ollama/ollama/tools" "github.com/ollama/ollama/types/errtypes" "github.com/ollama/ollama/types/model" @@ -282,10 +283,10 @@ func (s *Server) GenerateHandler(c *gin.Context) { prompt = b.String() } - var thinkingState *ThinkingParser - openingTag, closingTag := inferThinkingTags(m.Template.Template) + var thinkingState *thinking.Parser + openingTag, closingTag := thinking.InferTags(m.Template.Template) if req.Think != nil && *req.Think && openingTag != "" && closingTag != "" { - thinkingState = &ThinkingParser{ + thinkingState = &thinking.Parser{ OpeningTag: openingTag, ClosingTag: closingTag, } @@ -1522,10 +1523,10 @@ func (s *Server) ChatHandler(c *gin.Context) { return } - var thinkingState *ThinkingParser - openingTag, closingTag := inferThinkingTags(m.Template.Template) + var thinkingState *thinking.Parser + openingTag, closingTag := thinking.InferTags(m.Template.Template) if req.Think != nil && *req.Think && openingTag != "" && closingTag != "" { - thinkingState = &ThinkingParser{ + thinkingState = &thinking.Parser{ OpeningTag: openingTag, ClosingTag: closingTag, } @@ -1676,7 +1677,7 @@ func filterThinkTags(msgs []api.Message, m *Model) []api.Message { // change the user output), we should probably perform this filtering // for all thinking models (not just qwen3 & deepseek-r1) since it tends // to save tokens and improve quality. - thinkingState := &ThinkingParser{ + thinkingState := &thinking.Parser{ OpeningTag: "", ClosingTag: "", } diff --git a/server/thinking.go b/thinking/parser.go similarity index 59% rename from server/thinking.go rename to thinking/parser.go index 4ef3c1848..a4d05e35a 100644 --- a/server/thinking.go +++ b/thinking/parser.go @@ -1,9 +1,7 @@ -package server +package thinking import ( "strings" - "text/template" - "text/template/parse" "unicode" ) @@ -46,7 +44,7 @@ func (s thinkingState) String() string { } } -type ThinkingParser struct { +type Parser struct { state thinkingState OpeningTag string ClosingTag string @@ -56,7 +54,7 @@ type ThinkingParser struct { // AddContent returns the thinking content and the non-thinking content that // should be immediately sent to the user. It will internally buffer if it needs // to see more raw content to disambiguate -func (s *ThinkingParser) AddContent(content string) (string, string) { +func (s *Parser) AddContent(content string) (string, string) { s.acc.WriteString(content) var thinkingSb, remainingSb strings.Builder @@ -76,7 +74,7 @@ func (s *ThinkingParser) AddContent(content string) (string, string) { } // the additional bool return is true iff we should continue eating -func eat(s *ThinkingParser) (string, string, bool) { +func eat(s *Parser) (string, string, bool) { switch s.state { case thinkingState_LookingForOpening: trimmed := strings.TrimLeftFunc(s.acc.String(), unicode.IsSpace) @@ -171,130 +169,3 @@ func overlap(s, delim string) int { } return 0 } - -func templateVisit(n parse.Node, enterFn func(parse.Node) bool, exitFn func(parse.Node)) { - if n == nil { - return - } - shouldContinue := enterFn(n) - if !shouldContinue { - return - } - switch x := n.(type) { - case *parse.ListNode: - for _, c := range x.Nodes { - templateVisit(c, enterFn, exitFn) - } - case *parse.BranchNode: - if x.Pipe != nil { - templateVisit(x.Pipe, enterFn, exitFn) - } - if x.List != nil { - templateVisit(x.List, enterFn, exitFn) - } - if x.ElseList != nil { - templateVisit(x.ElseList, enterFn, exitFn) - } - case *parse.ActionNode: - templateVisit(x.Pipe, enterFn, exitFn) - case *parse.WithNode: - templateVisit(&x.BranchNode, enterFn, exitFn) - case *parse.RangeNode: - templateVisit(&x.BranchNode, enterFn, exitFn) - case *parse.IfNode: - templateVisit(&x.BranchNode, enterFn, exitFn) - case *parse.TemplateNode: - templateVisit(x.Pipe, enterFn, exitFn) - case *parse.PipeNode: - for _, c := range x.Cmds { - templateVisit(c, enterFn, exitFn) - } - case *parse.CommandNode: - for _, a := range x.Args { - templateVisit(a, enterFn, exitFn) - } - // text, field, number, etc. are leaves – nothing to recurse into - } - if exitFn != nil { - exitFn(n) - } -} - -// We use a heuristic to infer the tags that surround thinking traces: -// We look for a range node that iterates over "Messages" and then look for a -// reference to "Thinking" like `{{.Thinking}}`. We then go up to the nearest -// ListNode and take the first and last TextNodes as the opening and closing -// tags. -func inferThinkingTags(t *template.Template) (string, string) { - ancestors := []parse.Node{} - - openingTag := "" - closingTag := "" - - enterFn := func(n parse.Node) bool { - ancestors = append(ancestors, n) - - switch x := n.(type) { - case *parse.FieldNode: - if len(x.Ident) > 0 && x.Ident[0] == "Thinking" { - var mostRecentRange *parse.RangeNode - for i := len(ancestors) - 1; i >= 0; i-- { - if r, ok := ancestors[i].(*parse.RangeNode); ok { - mostRecentRange = r - break - } - } - if mostRecentRange == nil || !rangeUsesField(mostRecentRange, "Messages") { - return true - } - - // TODO(drifkin): to be more robust, check that it's in the action - // part, not the `if`'s pipeline part. We do match on the nearest list - // that starts and ends with text nodes, which makes this not strictly - // necessary for our heuristic - - // go up to the nearest ancestor that is a *parse.ListNode - for i := len(ancestors) - 1; i >= 0; i-- { - if l, ok := ancestors[i].(*parse.ListNode); ok { - firstNode := l.Nodes[0] - if t, ok := firstNode.(*parse.TextNode); ok { - openingTag = strings.TrimSpace(t.String()) - } - lastNode := l.Nodes[len(l.Nodes)-1] - if t, ok := lastNode.(*parse.TextNode); ok { - closingTag = strings.TrimSpace(t.String()) - } - - break - } - } - } - } - - return true - } - - exitFn := func(n parse.Node) { - ancestors = ancestors[:len(ancestors)-1] - } - - templateVisit(t.Root, enterFn, exitFn) - - return openingTag, closingTag -} - -// checks to see if the given field name is present in the pipeline of the given range node -func rangeUsesField(rangeNode *parse.RangeNode, field string) bool { - found := false - enterFn := func(n parse.Node) bool { - switch x := n.(type) { - case *parse.FieldNode: - if x.Ident[0] == field { - found = true - } - } - return true - } - templateVisit(rangeNode.BranchNode.Pipe, enterFn, nil) - return found -} diff --git a/server/thinking_test.go b/thinking/parser_test.go similarity index 66% rename from server/thinking_test.go rename to thinking/parser_test.go index 90d3f961d..78c297cd9 100644 --- a/server/thinking_test.go +++ b/thinking/parser_test.go @@ -1,8 +1,7 @@ -package server +package thinking import ( "testing" - "text/template" ) func TestExtractThinking(t *testing.T) { @@ -26,7 +25,7 @@ func TestExtractThinking(t *testing.T) { }, } for i, tt := range tests { - parser := ThinkingParser{ + parser := Parser{ OpeningTag: "", ClosingTag: "", } @@ -259,7 +258,7 @@ func TestThinkingStreaming(t *testing.T) { } for _, c := range cases { - parser := ThinkingParser{ + parser := Parser{ OpeningTag: "", ClosingTag: "", } @@ -277,127 +276,3 @@ func TestThinkingStreaming(t *testing.T) { } } } - -func TestInferThinkingTags(t *testing.T) { - cases := []struct { - desc string - tmplString string - wantOpeningTag string - wantClosingTag string - }{ - { - desc: "basic", - tmplString: ` - {{ if .Thinking}} - /think - {{ end }} - {{- range $i, $_ := .Messages }} - {{- $last := eq (len (slice $.Messages $i)) 1 -}} - {{ if and $last .Thinking }} - {{ .Thinking }} - {{ end }} - {{ end }} - `, - wantOpeningTag: "", - wantClosingTag: "", - }, - { - desc: "doubly nested range", - tmplString: ` - {{ if .Thinking}} - /think - {{ end }} - {{- range $i, $_ := .Messages }} - {{- range $j, $_ := .NotMessages }} - {{- $last := eq (len (slice $.Messages $i)) 1 -}} - {{ if and $last .Thinking }} - {{ .Thinking }} - {{ end }} - {{ end }} - {{ end }} - `, - wantOpeningTag: "", - wantClosingTag: "", - }, - { - desc: "whitespace is trimmed", - tmplString: ` - {{ if .Thinking}} - /think - {{ end }} - {{- range $i, $_ := .Messages }} - {{- $last := eq (len (slice $.Messages $i)) 1 -}} - {{ if and $last .Thinking }} - Some text before {{ .Thinking }} Some text after - {{ end }} - {{ end }} - `, - wantOpeningTag: "Some text before", - wantClosingTag: "Some text after", - }, - { - desc: "qwen3", - tmplString: ` -{{- if or .System .Tools .Thinking }}<|im_start|>system -{{- if .System }} -{{ .System }} -{{- end }} -{{- if .Tools }} - -# Tools - -You may call one or more functions to assist with the user query. - -You are provided with function signatures within XML tags: - -{{- range .Tools }} -{"type": "function", "function": {{ .Function }}} -{{- end }} - - -For each function call, return a json object with function name and arguments within XML tags: - -{"name": , "arguments": } - -{{- end }} -{{- if .Thinking }} -/think -{{- else }} -/no_think -{{- end }}<|im_end|> -{{ end }} -{{- range $i, $_ := .Messages }} -{{- $last := eq (len (slice $.Messages $i)) 1 -}} -{{- if eq .Role "user" }}<|im_start|>user -{{ .Content }}<|im_end|> -{{ else if eq .Role "assistant" }}<|im_start|>assistant -{{ if and $last .Thinking }} -{{ .Thinking }} -{{ end }} -{{ if .Content }}{{ .Content }} -{{- else if .ToolCalls }} -{{ range .ToolCalls }}{"name": "{{ .Function.Name }}", "arguments": {{ .Function.Arguments }}} -{{ end }} -{{- end }}{{ if not $last }}<|im_end|> -{{ end }} -{{- else if eq .Role "tool" }}<|im_start|>user - -{{ .Content }} -<|im_end|> -{{ end }} -{{- if and (ne .Role "assistant") $last }}<|im_start|>assistant -{{ end }} -{{- end }} - `, - wantOpeningTag: "", - wantClosingTag: "", - }, - } - for _, c := range cases { - tmpl := template.Must(template.New("test").Parse(c.tmplString)) - openingTag, closingTag := inferThinkingTags(tmpl) - if openingTag != c.wantOpeningTag || closingTag != c.wantClosingTag { - t.Errorf("case %q: got (%q,%q), want (%q,%q)", c.desc, openingTag, closingTag, c.wantOpeningTag, c.wantClosingTag) - } - } -} diff --git a/thinking/template.go b/thinking/template.go new file mode 100644 index 000000000..20bd65ec1 --- /dev/null +++ b/thinking/template.go @@ -0,0 +1,134 @@ +package thinking + +import ( + "strings" + "text/template" + "text/template/parse" +) + +func templateVisit(n parse.Node, enterFn func(parse.Node) bool, exitFn func(parse.Node)) { + if n == nil { + return + } + shouldContinue := enterFn(n) + if !shouldContinue { + return + } + switch x := n.(type) { + case *parse.ListNode: + for _, c := range x.Nodes { + templateVisit(c, enterFn, exitFn) + } + case *parse.BranchNode: + if x.Pipe != nil { + templateVisit(x.Pipe, enterFn, exitFn) + } + if x.List != nil { + templateVisit(x.List, enterFn, exitFn) + } + if x.ElseList != nil { + templateVisit(x.ElseList, enterFn, exitFn) + } + case *parse.ActionNode: + templateVisit(x.Pipe, enterFn, exitFn) + case *parse.WithNode: + templateVisit(&x.BranchNode, enterFn, exitFn) + case *parse.RangeNode: + templateVisit(&x.BranchNode, enterFn, exitFn) + case *parse.IfNode: + templateVisit(&x.BranchNode, enterFn, exitFn) + case *parse.TemplateNode: + templateVisit(x.Pipe, enterFn, exitFn) + case *parse.PipeNode: + for _, c := range x.Cmds { + templateVisit(c, enterFn, exitFn) + } + case *parse.CommandNode: + for _, a := range x.Args { + templateVisit(a, enterFn, exitFn) + } + // text, field, number, etc. are leaves – nothing to recurse into + } + if exitFn != nil { + exitFn(n) + } +} + +// InferTags uses a heuristic to infer the tags that surround thinking traces: +// We look for a range node that iterates over "Messages" and then look for a +// reference to "Thinking" like `{{.Thinking}}`. We then go up to the nearest +// ListNode and take the first and last TextNodes as the opening and closing +// tags. +func InferTags(t *template.Template) (string, string) { + ancestors := []parse.Node{} + + openingTag := "" + closingTag := "" + + enterFn := func(n parse.Node) bool { + ancestors = append(ancestors, n) + + switch x := n.(type) { + case *parse.FieldNode: + if len(x.Ident) > 0 && x.Ident[0] == "Thinking" { + var mostRecentRange *parse.RangeNode + for i := len(ancestors) - 1; i >= 0; i-- { + if r, ok := ancestors[i].(*parse.RangeNode); ok { + mostRecentRange = r + break + } + } + if mostRecentRange == nil || !rangeUsesField(mostRecentRange, "Messages") { + return true + } + + // TODO(drifkin): to be more robust, check that it's in the action + // part, not the `if`'s pipeline part. We do match on the nearest list + // that starts and ends with text nodes, which makes this not strictly + // necessary for our heuristic + + // go up to the nearest ancestor that is a *parse.ListNode + for i := len(ancestors) - 1; i >= 0; i-- { + if l, ok := ancestors[i].(*parse.ListNode); ok { + firstNode := l.Nodes[0] + if t, ok := firstNode.(*parse.TextNode); ok { + openingTag = strings.TrimSpace(t.String()) + } + lastNode := l.Nodes[len(l.Nodes)-1] + if t, ok := lastNode.(*parse.TextNode); ok { + closingTag = strings.TrimSpace(t.String()) + } + + break + } + } + } + } + + return true + } + + exitFn := func(n parse.Node) { + ancestors = ancestors[:len(ancestors)-1] + } + + templateVisit(t.Root, enterFn, exitFn) + + return openingTag, closingTag +} + +// checks to see if the given field name is present in the pipeline of the given range node +func rangeUsesField(rangeNode *parse.RangeNode, field string) bool { + found := false + enterFn := func(n parse.Node) bool { + switch x := n.(type) { + case *parse.FieldNode: + if x.Ident[0] == field { + found = true + } + } + return true + } + templateVisit(rangeNode.BranchNode.Pipe, enterFn, nil) + return found +} diff --git a/thinking/template_test.go b/thinking/template_test.go new file mode 100644 index 000000000..e63558e28 --- /dev/null +++ b/thinking/template_test.go @@ -0,0 +1,130 @@ +package thinking + +import ( + "testing" + "text/template" +) + +func TestInferThinkingTags(t *testing.T) { + cases := []struct { + desc string + tmplString string + wantOpeningTag string + wantClosingTag string + }{ + { + desc: "basic", + tmplString: ` + {{ if .Thinking}} + /think + {{ end }} + {{- range $i, $_ := .Messages }} + {{- $last := eq (len (slice $.Messages $i)) 1 -}} + {{ if and $last .Thinking }} + {{ .Thinking }} + {{ end }} + {{ end }} + `, + wantOpeningTag: "", + wantClosingTag: "", + }, + { + desc: "doubly nested range", + tmplString: ` + {{ if .Thinking}} + /think + {{ end }} + {{- range $i, $_ := .Messages }} + {{- range $j, $_ := .NotMessages }} + {{- $last := eq (len (slice $.Messages $i)) 1 -}} + {{ if and $last .Thinking }} + {{ .Thinking }} + {{ end }} + {{ end }} + {{ end }} + `, + wantOpeningTag: "", + wantClosingTag: "", + }, + { + desc: "whitespace is trimmed", + tmplString: ` + {{ if .Thinking}} + /think + {{ end }} + {{- range $i, $_ := .Messages }} + {{- $last := eq (len (slice $.Messages $i)) 1 -}} + {{ if and $last .Thinking }} + Some text before {{ .Thinking }} Some text after + {{ end }} + {{ end }} + `, + wantOpeningTag: "Some text before", + wantClosingTag: "Some text after", + }, + { + desc: "qwen3", + tmplString: ` +{{- if or .System .Tools .Thinking }}<|im_start|>system +{{- if .System }} +{{ .System }} +{{- end }} +{{- if .Tools }} + +# Tools + +You may call one or more functions to assist with the user query. + +You are provided with function signatures within XML tags: + +{{- range .Tools }} +{"type": "function", "function": {{ .Function }}} +{{- end }} + + +For each function call, return a json object with function name and arguments within XML tags: + +{"name": , "arguments": } + +{{- end }} +{{- if .Thinking }} +/think +{{- else }} +/no_think +{{- end }}<|im_end|> +{{ end }} +{{- range $i, $_ := .Messages }} +{{- $last := eq (len (slice $.Messages $i)) 1 -}} +{{- if eq .Role "user" }}<|im_start|>user +{{ .Content }}<|im_end|> +{{ else if eq .Role "assistant" }}<|im_start|>assistant +{{ if and $last .Thinking }} +{{ .Thinking }} +{{ end }} +{{ if .Content }}{{ .Content }} +{{- else if .ToolCalls }} +{{ range .ToolCalls }}{"name": "{{ .Function.Name }}", "arguments": {{ .Function.Arguments }}} +{{ end }} +{{- end }}{{ if not $last }}<|im_end|> +{{ end }} +{{- else if eq .Role "tool" }}<|im_start|>user + +{{ .Content }} +<|im_end|> +{{ end }} +{{- if and (ne .Role "assistant") $last }}<|im_start|>assistant +{{ end }} +{{- end }} + `, + wantOpeningTag: "", + wantClosingTag: "", + }, + } + for _, c := range cases { + tmpl := template.Must(template.New("test").Parse(c.tmplString)) + openingTag, closingTag := InferTags(tmpl) + if openingTag != c.wantOpeningTag || closingTag != c.wantClosingTag { + t.Errorf("case %q: got (%q,%q), want (%q,%q)", c.desc, openingTag, closingTag, c.wantOpeningTag, c.wantClosingTag) + } + } +} From 2ae65ae471c9d51d343f401da16c05b98b99a842 Mon Sep 17 00:00:00 2001 From: Daniel Hiltgen Date: Fri, 6 Jun 2025 14:06:09 -0700 Subject: [PATCH 30/54] win: handle more than 2048 processes (#10997) Fix an array out of bounds crash --- cmd/start_windows.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/cmd/start_windows.go b/cmd/start_windows.go index bcc51057b..1b648d9d2 100644 --- a/cmd/start_windows.go +++ b/cmd/start_windows.go @@ -74,7 +74,16 @@ func isProcRunning(procName string) []uint32 { slog.Debug("failed to check for running installers", "error", err) return nil } - pids = pids[:ret] + if ret > uint32(len(pids)) { + pids = make([]uint32, ret+10) + if err := windows.EnumProcesses(pids, &ret); err != nil || ret == 0 { + slog.Debug("failed to check for running installers", "error", err) + return nil + } + } + if ret < uint32(len(pids)) { + pids = pids[:ret] + } var matches []uint32 for _, pid := range pids { if pid == 0 { From a8ed68bd9383ffec346fd1b3cf60d94c032bbec8 Mon Sep 17 00:00:00 2001 From: Daniel Hiltgen Date: Fri, 6 Jun 2025 14:06:29 -0700 Subject: [PATCH 31/54] launch app hidden (#10962) When starting the app in the background, start it hidden. --- cmd/start_darwin.go | 2 +- cmd/start_windows.go | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/cmd/start_darwin.go b/cmd/start_darwin.go index 1a9a1ae87..3cb726eaf 100644 --- a/cmd/start_darwin.go +++ b/cmd/start_darwin.go @@ -23,7 +23,7 @@ func startApp(ctx context.Context, client *api.Client) error { return errors.New("could not find ollama app") } path := strings.Split(link, "Ollama.app") - if err := exec.Command("/usr/bin/open", "-a", path[0]+"Ollama.app").Run(); err != nil { + if err := exec.Command("/usr/bin/open", "-j", "-a", path[0]+"Ollama.app").Run(); err != nil { return err } return waitForServer(ctx, client) diff --git a/cmd/start_windows.go b/cmd/start_windows.go index 1b648d9d2..635b50770 100644 --- a/cmd/start_windows.go +++ b/cmd/start_windows.go @@ -45,14 +45,11 @@ func startApp(ctx context.Context, client *api.Client) error { } } } - // log.Printf("XXX attempting to start app %s", appExe) cmd_path := "c:\\Windows\\system32\\cmd.exe" - cmd := exec.Command(cmd_path, "/c", appExe) - // TODO - these hide flags aren't working - still pops up a command window for some reason + cmd := exec.Command(cmd_path, "/c", appExe, "hidden") cmd.SysProcAttr = &syscall.SysProcAttr{CreationFlags: 0x08000000, HideWindow: true} - // TODO this didn't help either... cmd.Stdin = strings.NewReader("") cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr From 09d308d6b6c7995e3fb565e0ecfa184d49205f00 Mon Sep 17 00:00:00 2001 From: Jeffrey Morgan Date: Fri, 6 Jun 2025 23:29:14 -0400 Subject: [PATCH 32/54] Revert "server: add model capabilities to the list endpoint (#10174)" (#11004) This reverts commit 09430011936652cf55925184aaed6f2cebf62a75. --- api/types.go | 13 ++++++------- docs/api.md | 29 ++++++++++------------------- server/routes.go | 14 +++----------- 3 files changed, 19 insertions(+), 37 deletions(-) diff --git a/api/types.go b/api/types.go index a1f896db3..94d492006 100644 --- a/api/types.go +++ b/api/types.go @@ -457,13 +457,12 @@ type ProcessResponse struct { // ListModelResponse is a single model description in [ListResponse]. type ListModelResponse struct { - Name string `json:"name"` - Model string `json:"model"` - ModifiedAt time.Time `json:"modified_at"` - Size int64 `json:"size"` - Digest string `json:"digest"` - Capabilities []model.Capability `json:"capabilities,omitempty"` - Details ModelDetails `json:"details,omitempty"` + Name string `json:"name"` + Model string `json:"model"` + ModifiedAt time.Time `json:"modified_at"` + Size int64 `json:"size"` + Digest string `json:"digest"` + Details ModelDetails `json:"details,omitempty"` } // ProcessModelResponse is a single model description in [ProcessResponse]. diff --git a/docs/api.md b/docs/api.md index 31e18bd5d..11eaf73ab 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1157,15 +1157,11 @@ A single JSON object will be returned. { "models": [ { - - "model": "codellama:13b", - "modified_at": "2023-11-04T14:56:49.277302595-07:00", - "size": 7365960935, - "digest": "9f438cb9cd581fc025612d27f7c1a6669ff83a8bb0ed86c94fcf4c5440555697", - "capabilities": [ - "completion" - ], - + "name": "deepseek-r1:latest", + "model": "deepseek-r1:latest", + "modified_at": "2025-05-10T08:06:48.639712648-07:00", + "size": 4683075271, + "digest": "0a8c266910232fd3291e71e5ba1e058cc5af9d411192cf88b6d30e92b6e73163", "details": { "parent_model": "", "format": "gguf", @@ -1178,16 +1174,11 @@ A single JSON object will be returned. } }, { - - "model": "llama4:latest", - "modified_at": "2023-12-07T09:32:18.757212583-08:00", - "size": 3825819519, - "digest": "fe938a131f40e6f6d40083c9f0f430a515233eb2edaa6d72eb85c50d64f2300e", - "capabilities": [ - "completion", - "vision" - ], - + "name": "llama3.2:latest", + "model": "llama3.2:latest", + "modified_at": "2025-05-04T17:37:44.706015396-07:00", + "size": 2019393189, + "digest": "a80c4f17acd55265feec403c7aef86be0c25983ab279d83f3bcd3abbcb5b8b72", "details": { "parent_model": "", "format": "gguf", diff --git a/server/routes.go b/server/routes.go index 70cb6cef9..8eda5c73f 100644 --- a/server/routes.go +++ b/server/routes.go @@ -929,7 +929,8 @@ func (s *Server) ListHandler(c *gin.Context) { } } - r := api.ListModelResponse{ + // tag should never be masked + models = append(models, api.ListModelResponse{ Model: n.DisplayShortest(), Name: n.DisplayShortest(), Size: m.Size(), @@ -942,16 +943,7 @@ func (s *Server) ListHandler(c *gin.Context) { ParameterSize: cf.ModelType, QuantizationLevel: cf.FileType, }, - } - - model, err := GetModel(n.String()) - if err != nil { - slog.Warn("bad model details", "name", n, "error", err) - } else { - r.Capabilities = model.Capabilities() - } - - models = append(models, r) + }) } slices.SortStableFunc(models, func(i, j api.ListModelResponse) int { From fc0309615e42c32989e060e733d871e16617874e Mon Sep 17 00:00:00 2001 From: Krzysztof Jeziorny <872730+krzysztofjeziorny@users.noreply.github.com> Date: Sat, 7 Jun 2025 05:30:04 +0200 Subject: [PATCH 33/54] docs: update link to AMD drivers in linux.md (#10973) --- docs/linux.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/linux.md b/docs/linux.md index 2dda87f3a..72a5ff019 100644 --- a/docs/linux.md +++ b/docs/linux.md @@ -112,8 +112,8 @@ sudo systemctl status ollama > While AMD has contributed the `amdgpu` driver upstream to the official linux > kernel source, the version is older and may not support all ROCm features. We > recommend you install the latest driver from -> https://www.amd.com/en/support/linux-drivers for best support of your Radeon -> GPU. +> [AMD](https://www.amd.com/en/support/download/linux-drivers.html) for best support +> of your Radeon GPU. ## Customizing From feeabdadd2b272b40747f3e7e74957c40ba2800c Mon Sep 17 00:00:00 2001 From: Daniel Hiltgen Date: Sun, 8 Jun 2025 09:34:52 -0700 Subject: [PATCH 34/54] spawn desktop quickly (#11011) Give the desktop app a hint to start fast. --- cmd/start_darwin.go | 2 +- cmd/start_windows.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/start_darwin.go b/cmd/start_darwin.go index 3cb726eaf..83af12353 100644 --- a/cmd/start_darwin.go +++ b/cmd/start_darwin.go @@ -23,7 +23,7 @@ func startApp(ctx context.Context, client *api.Client) error { return errors.New("could not find ollama app") } path := strings.Split(link, "Ollama.app") - if err := exec.Command("/usr/bin/open", "-j", "-a", path[0]+"Ollama.app").Run(); err != nil { + if err := exec.Command("/usr/bin/open", "-j", "-a", path[0]+"Ollama.app", "--args", "--fast-startup").Run(); err != nil { return err } return waitForServer(ctx, client) diff --git a/cmd/start_windows.go b/cmd/start_windows.go index 635b50770..9505e1bba 100644 --- a/cmd/start_windows.go +++ b/cmd/start_windows.go @@ -47,7 +47,7 @@ func startApp(ctx context.Context, client *api.Client) error { } cmd_path := "c:\\Windows\\system32\\cmd.exe" - cmd := exec.Command(cmd_path, "/c", appExe, "hidden") + cmd := exec.Command(cmd_path, "/c", appExe, "--hide", "--fast-startup") cmd.SysProcAttr = &syscall.SysProcAttr{CreationFlags: 0x08000000, HideWindow: true} cmd.Stdin = strings.NewReader("") From 82ad1dbc07dc2db39c6f502eb148ff3ce00d96b8 Mon Sep 17 00:00:00 2001 From: Daniel Hiltgen Date: Mon, 9 Jun 2025 16:29:57 -0700 Subject: [PATCH 35/54] mac: handle "keep" named apps (#11031) When a user elects to keep the existing app, the new Ollama is named `Ollama 2.app` This fixes the app startup flow to handle this naming pattern. --- cmd/start_darwin.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cmd/start_darwin.go b/cmd/start_darwin.go index 83af12353..05a4551e1 100644 --- a/cmd/start_darwin.go +++ b/cmd/start_darwin.go @@ -5,7 +5,7 @@ import ( "errors" "os" "os/exec" - "strings" + "regexp" "github.com/ollama/ollama/api" ) @@ -19,11 +19,12 @@ func startApp(ctx context.Context, client *api.Client) error { if err != nil { return err } - if !strings.Contains(link, "Ollama.app") { + r := regexp.MustCompile(`^.*/Ollama\s?\d*.app`) + m := r.FindStringSubmatch(link) + if len(m) != 1 { return errors.New("could not find ollama app") } - path := strings.Split(link, "Ollama.app") - if err := exec.Command("/usr/bin/open", "-j", "-a", path[0]+"Ollama.app", "--args", "--fast-startup").Run(); err != nil { + if err := exec.Command("/usr/bin/open", "-j", "-a", m[0], "--args", "--fast-startup").Run(); err != nil { return err } return waitForServer(ctx, client) From f63d7f68eb206cf403fb9c7dca7978d16204e268 Mon Sep 17 00:00:00 2001 From: Jeffrey Morgan Date: Tue, 10 Jun 2025 09:33:54 -0700 Subject: [PATCH 36/54] readme: update quickstart example to Gemma 3 --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 22de41e63..af064a63b 100644 --- a/README.md +++ b/README.md @@ -40,10 +40,10 @@ The official [Ollama Docker image](https://hub.docker.com/r/ollama/ollama) `olla ## Quickstart -To run and chat with [Llama 3.2](https://ollama.com/library/llama3.2): +To run and chat with [Llama 3.2](https://ollama.com/library/gemma3): ```shell -ollama run llama3.2 +ollama run gemma3 ``` ## Model library From af21a5ac397c1d2ce62881e0962bbff9d6da31f2 Mon Sep 17 00:00:00 2001 From: Jeffrey Morgan Date: Tue, 10 Jun 2025 09:34:23 -0700 Subject: [PATCH 37/54] readme: update quickstart link text to Gemma 3 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index af064a63b..90e41be8b 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ The official [Ollama Docker image](https://hub.docker.com/r/ollama/ollama) `olla ## Quickstart -To run and chat with [Llama 3.2](https://ollama.com/library/gemma3): +To run and chat with [Gemma 3](https://ollama.com/library/gemma3): ```shell ollama run gemma3 From deaabe292d86b712e061bebe7fdd6be6690f539b Mon Sep 17 00:00:00 2001 From: Attogram Project Date: Tue, 10 Jun 2025 23:14:51 +0200 Subject: [PATCH 38/54] readme: add ollama-multirun to community integrations (#11038) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 90e41be8b..ffaec6281 100644 --- a/README.md +++ b/README.md @@ -451,6 +451,7 @@ See the [API documentation](./docs/api.md) for all endpoints. - [orca-cli](https://github.com/molbal/orca-cli) Ollama Registry CLI Application - Browse, pull, and download models from Ollama Registry in your terminal. - [GGUF-to-Ollama](https://github.com/jonathanhecl/gguf-to-ollama) - Importing GGUF to Ollama made easy (multiplatform) - [AWS-Strands-With-Ollama](https://github.com/rapidarchitect/ollama_strands) - AWS Strands Agents with Ollama Examples +- [ollama-multirun](https://github.com/attogram/ollama-multirun) - A bash shell script to run a single prompt against any or all of your locally installed ollama models, saving the output and performance statistics as easily navigable web pages. ([Demo](https://attogram.github.io/ai_test_zone/)) ### Apple Vision Pro From 2e77aa1ae70372388bd4b08b9957e5198d566a22 Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Wed, 11 Jun 2025 12:10:15 -0700 Subject: [PATCH 39/54] use nn.Linear in place of ml.Tensor (#11049) while nn.Linear.Forward isn't applicable for sparse MLP, it's still a nice container for the tensors --- model/models/llama4/model_text.go | 12 ++++++------ model/models/qwen3/model.go | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/model/models/llama4/model_text.go b/model/models/llama4/model_text.go index 27935f401..045ab403f 100644 --- a/model/models/llama4/model_text.go +++ b/model/models/llama4/model_text.go @@ -63,9 +63,9 @@ func (mlp *TextMLP) Forward(ctx ml.Context, hiddenStates ml.Tensor, opts *TextOp } type TextExperts struct { - Gate ml.Tensor `gguf:"ffn_gate_exps.weight"` - Up ml.Tensor `gguf:"ffn_up_exps.weight"` - Down ml.Tensor `gguf:"ffn_down_exps.weight"` + Gate *nn.Linear `gguf:"ffn_gate_exps"` + Up *nn.Linear `gguf:"ffn_up_exps"` + Down *nn.Linear `gguf:"ffn_down_exps"` } func (e *TextExperts) Forward(ctx ml.Context, hiddenStates, routerLogits ml.Tensor, opts *TextOptions) ml.Tensor { @@ -76,9 +76,9 @@ func (e *TextExperts) Forward(ctx ml.Context, hiddenStates, routerLogits ml.Tens hiddenStates = hiddenStates.Repeat(ctx, 1, opts.numExpertsUsed) hiddenStates = hiddenStates.Mul(ctx, scores) - upStates := e.Up.MulmatID(ctx, hiddenStates, experts) - gateStates := e.Gate.MulmatID(ctx, hiddenStates, experts) - downStates := e.Down.MulmatID(ctx, upStates.Mul(ctx, gateStates.SILU(ctx)), experts) + upStates := e.Up.Weight.MulmatID(ctx, hiddenStates, experts) + gateStates := e.Gate.Weight.MulmatID(ctx, hiddenStates, experts) + downStates := e.Down.Weight.MulmatID(ctx, upStates.Mul(ctx, gateStates.SILU(ctx)), experts) nextStates := downStates.View(ctx, 0, hiddenStates.Dim(0), downStates.Stride(2), hiddenStates.Dim(2)) for i := 1; i < opts.numExpertsUsed; i++ { diff --git a/model/models/qwen3/model.go b/model/models/qwen3/model.go index 1930da7e2..7a83e0d04 100644 --- a/model/models/qwen3/model.go +++ b/model/models/qwen3/model.go @@ -66,9 +66,9 @@ type MLP interface { type sparse struct { Router *nn.Linear `gguf:"ffn_gate_inp"` - Gate ml.Tensor `gguf:"ffn_gate_exps.weight"` - Up ml.Tensor `gguf:"ffn_up_exps.weight"` - Down ml.Tensor `gguf:"ffn_down_exps.weight"` + Gate *nn.Linear `gguf:"ffn_gate_exps"` + Up *nn.Linear `gguf:"ffn_up_exps"` + Down *nn.Linear `gguf:"ffn_down_exps"` } func (mlp *sparse) Forward(ctx ml.Context, hiddenStates ml.Tensor, opts *Options) ml.Tensor { @@ -87,13 +87,13 @@ func (mlp *sparse) Forward(ctx ml.Context, hiddenStates ml.Tensor, opts *Options hiddenStates = hiddenStates.Reshape(ctx, hiddenStates.Dim(0), 1, hiddenStates.Dim(1)) - upStates := mlp.Up.MulmatID(ctx, hiddenStates, selectedExperts) + upStates := mlp.Up.Weight.MulmatID(ctx, hiddenStates, selectedExperts) - hiddenStates = mlp.Gate.MulmatID(ctx, hiddenStates, selectedExperts) + hiddenStates = mlp.Gate.Weight.MulmatID(ctx, hiddenStates, selectedExperts) hiddenStates = hiddenStates.SILU(ctx) hiddenStates = hiddenStates.Mul(ctx, upStates) - experts := mlp.Down.MulmatID(ctx, hiddenStates, selectedExperts) + experts := mlp.Down.Weight.MulmatID(ctx, hiddenStates, selectedExperts) experts = experts.Mul(ctx, routingWeights) nextStates := experts.View(ctx, 0, experts.Dim(0), experts.Stride(2), experts.Dim(2)) From 0dabb4ef6a1aab240a59b6bb4ef82372d335e3a9 Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Wed, 11 Jun 2025 12:10:35 -0700 Subject: [PATCH 40/54] skip tokenizer.model if possible (#11050) if tokenizer.json is already copied, skip tokenizer.model --- parser/parser.go | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/parser/parser.go b/parser/parser.go index 96eae9c04..d40a79c29 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -292,13 +292,18 @@ func filesForModel(path string) ([]string, error) { } files = append(files, js...) - if tks, _ := glob(filepath.Join(path, "tokenizer.model"), "application/octet-stream"); len(tks) > 0 { - // add tokenizer.model if it exists, tokenizer.json is automatically picked up by the previous glob - // tokenizer.model might be a unresolved git lfs reference; error if it is - files = append(files, tks...) - } else if tks, _ := glob(filepath.Join(path, "**/tokenizer.model"), "text/plain"); len(tks) > 0 { - // some times tokenizer.model is in a subdirectory (e.g. meta-llama/Meta-Llama-3-8B) - files = append(files, tks...) + // only include tokenizer.model is tokenizer.json is not present + if !slices.ContainsFunc(files, func(s string) bool { + return slices.Contains(strings.Split(s, string(os.PathSeparator)), "tokenizer.json") + }) { + if tks, _ := glob(filepath.Join(path, "tokenizer.model"), "application/octet-stream"); len(tks) > 0 { + // add tokenizer.model if it exists, tokenizer.json is automatically picked up by the previous glob + // tokenizer.model might be a unresolved git lfs reference; error if it is + files = append(files, tks...) + } else if tks, _ := glob(filepath.Join(path, "**/tokenizer.model"), "text/plain"); len(tks) > 0 { + // some times tokenizer.model is in a subdirectory (e.g. meta-llama/Meta-Llama-3-8B) + files = append(files, tks...) + } } return files, nil From 45f56355d557b7130c7c07bbd6e1b634a758d946 Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Wed, 11 Jun 2025 12:10:54 -0700 Subject: [PATCH 41/54] feat: uneven splits (#11048) The current splitDim function only operates on tensors that are split evenly which isn't always the case, e.g. a QKV tensor. This change allows the function to be used for arbitrary splits --- convert/convert_qwen25vl.go | 10 +- convert/tensor.go | 50 ++++-- convert/tensor_test.go | 304 ++++++++++++++++++++++++++++++++++++ 3 files changed, 344 insertions(+), 20 deletions(-) create mode 100644 convert/tensor_test.go diff --git a/convert/convert_qwen25vl.go b/convert/convert_qwen25vl.go index c2d5a633b..6e4c96408 100644 --- a/convert/convert_qwen25vl.go +++ b/convert/convert_qwen25vl.go @@ -65,17 +65,17 @@ func (q *qwen25VLModel) Tensors(ts []Tensor) []*ggml.Tensor { for _, t := range ts { if strings.Contains(t.Name(), "patch_embed.proj") { for t := range splitDim(t, 2, - strings.NewReplacer("patch_embed.proj", "patch_embd_0"), - strings.NewReplacer("patch_embed.proj", "patch_embd_1"), + split{Replacer: strings.NewReplacer("patch_embed.proj", "patch_embd_0")}, + split{Replacer: strings.NewReplacer("patch_embed.proj", "patch_embd_1")}, ) { t.Shape = slices.DeleteFunc(t.Shape, func(i uint64) bool { return i == 1 }) out = append(out, t) } } else if strings.Contains(t.Name(), "attn.qkv") { out = append(out, slices.Collect(splitDim(t, 0, - strings.NewReplacer("attn.qkv", "attn_q"), - strings.NewReplacer("attn.qkv", "attn_k"), - strings.NewReplacer("attn.qkv", "attn_v"), + split{Replacer: strings.NewReplacer("attn.qkv", "attn_q")}, + split{Replacer: strings.NewReplacer("attn.qkv", "attn_k")}, + split{Replacer: strings.NewReplacer("attn.qkv", "attn_v")}, ))...) } else { out = append(out, &ggml.Tensor{ diff --git a/convert/tensor.go b/convert/tensor.go index ffb22ead9..9d6919e36 100644 --- a/convert/tensor.go +++ b/convert/tensor.go @@ -1,53 +1,73 @@ package convert import ( + "cmp" "iter" "slices" "strings" - "github.com/ollama/ollama/fs/ggml" "github.com/pdevine/tensor" "github.com/pdevine/tensor/native" + + "github.com/ollama/ollama/fs/ggml" ) +type split struct { + *strings.Replacer + dim int + + // fn is an optional function to apply to the tensor after slicing + fn func(tensor.Tensor) (tensor.Tensor, error) +} + // splitDim splits a tensor along a specified dimension into multiple tensors. The dimension -// is split evenly based on the number of replacers provided. -func splitDim(t Tensor, dim int, replacers ...*strings.Replacer) iter.Seq[*ggml.Tensor] { +// is split evenly based on the number of replacers provided unless a specific count is given. +func splitDim(t Tensor, dim int, splits ...split) iter.Seq[*ggml.Tensor] { return func(yield func(*ggml.Tensor) bool) { - for i, replacer := range replacers { + var offset int + for _, split := range splits { + t := t.Clone() shape := slices.Clone(t.Shape()) - shape[dim] = shape[dim] / uint64(len(replacers)) + shape[dim] = cmp.Or(uint64(split.dim), shape[dim]/uint64(len(splits))) slice := slices.Repeat([]tensor.Slice{nil}, len(shape)) - slice[dim] = tensor.S(i*int(shape[dim]), (i+1)*int(shape[dim])) + slice[dim] = tensor.S(offset, offset+int(shape[dim])) + offset += int(shape[dim]) - tt := t.Clone() - tt.SetRepacker(func(_ string, data []float32, shape []uint64) ([]float32, error) { + t.SetRepacker(func(_ string, data []float32, shape []uint64) ([]float32, error) { dims := make([]int, len(shape)) for i := range shape { dims[i] = int(shape[i]) } - var t tensor.Tensor = tensor.New(tensor.WithShape(dims...), tensor.WithBacking(data)) - t, err := t.Slice(slice...) + var tt tensor.Tensor = tensor.New(tensor.WithShape(dims...), tensor.WithBacking(data)) + tt, err := tt.Slice(slice...) if err != nil { return nil, err } - t = tensor.Materialize(t) + tt = tensor.Materialize(tt) + + if split.fn != nil { + tt, err = split.fn(tt) + if err != nil { + return nil, err + } + } + // flatten tensor so it can be written as a vector - if err := t.Reshape(t.Shape().TotalSize()); err != nil { + if err := tt.Reshape(tt.Shape().TotalSize()); err != nil { return nil, err } - return native.VectorF32(t.(*tensor.Dense)) + return native.VectorF32(tt.(*tensor.Dense)) }) if !yield(&ggml.Tensor{ - Name: replacer.Replace(t.Name()), + Name: split.Replace(t.Name()), Kind: t.Kind(), Shape: shape, - WriterTo: tt, + WriterTo: t, }) { break } diff --git a/convert/tensor_test.go b/convert/tensor_test.go new file mode 100644 index 000000000..ea12d0f59 --- /dev/null +++ b/convert/tensor_test.go @@ -0,0 +1,304 @@ +package convert + +import ( + "bytes" + "encoding/binary" + "io" + "iter" + "slices" + "strings" + "testing" + + "github.com/pdevine/tensor" +) + +type fakeTensor struct { + name string + shape []uint64 + data []float32 + + repacker Repacker +} + +func (f fakeTensor) Name() string { + return f.name +} + +func (f fakeTensor) Shape() []uint64 { + return f.shape +} + +func (f fakeTensor) Kind() uint32 { + return 0 +} + +func (f *fakeTensor) SetRepacker(fn Repacker) { + f.repacker = fn +} + +func (f fakeTensor) Clone() Tensor { + return &fakeTensor{ + name: f.name, + shape: slices.Clone(f.shape), + data: slices.Clone(f.data), + repacker: f.repacker, + } +} + +func (f fakeTensor) WriteTo(w io.Writer) (n int64, err error) { + data := f.data + if f.repacker != nil { + data, err = f.repacker(f.name, data, f.shape) + if err != nil { + return 0, err + } + } + + if err := binary.Write(w, binary.LittleEndian, data); err != nil { + return 0, err + } + + return int64(len(data) * 4), nil +} + +func mul(shape []uint64) int { + n := 1 + for _, dim := range shape { + n *= int(dim) + } + return n +} + +func TestSplitDim(t *testing.T) { + r := fakeTensor{ + name: "a.b", + shape: []uint64{3, 4}, + data: []float32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}, + } + + t.Run("no split", func(t *testing.T) { + for tt := range splitDim(&r, 0, split{Replacer: strings.NewReplacer("a", "x")}) { + if tt.Name != "x.b" { + t.Fatalf("expected name 'x', got '%s'", tt.Name) + } + + if !slices.Equal(tt.Shape, []uint64{3, 4}) { + t.Fatalf("expected shape [3, 4], got %v", tt.Shape) + } + + var b bytes.Buffer + if _, err := tt.WriteTo(&b); err != nil { + t.Fatal(err) + } + + f32s := make([]float32, mul(tt.Shape)) + if err := binary.Read(&b, binary.LittleEndian, &f32s); err != nil { + t.Fatal(err) + } + + if !slices.Equal(f32s, []float32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}) { + t.Fatalf("expected data [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], got %v", f32s) + } + } + }) + + t.Run("even split", func(t *testing.T) { + next, stop := iter.Pull(splitDim(&r, 1, + split{Replacer: strings.NewReplacer("a", "x")}, + split{Replacer: strings.NewReplacer("b", "y")}, + )) + defer stop() + + { + tt, ok := next() + if !ok { + t.Fatal("expected at least one split") + } + + if tt.Name != "x.b" { + t.Fatal("expected name 'x.b', got", tt.Name) + } + + if !slices.Equal(tt.Shape, []uint64{3, 2}) { + t.Fatal("expected shape [3, 2], got", tt.Shape) + } + + var b bytes.Buffer + if _, err := tt.WriteTo(&b); err != nil { + t.Fatal(err) + } + + f32s := make([]float32, mul(tt.Shape)) + if err := binary.Read(&b, binary.LittleEndian, &f32s); err != nil { + t.Fatal(err) + } + + if !slices.Equal(f32s, []float32{0, 1, 4, 5, 8, 9}) { + t.Fatal("expected data [0, 1, 4, 5, 8, 9], got", f32s) + } + } + + { + tt, ok := next() + if !ok { + t.Fatal("expected at least one split") + } + + if tt.Name != "a.y" { + t.Fatal("expected name 'a.y', got", tt.Name) + } + + if !slices.Equal(tt.Shape, []uint64{3, 2}) { + t.Fatal("expected shape [3, 2], got", tt.Shape) + } + + var b bytes.Buffer + if _, err := tt.WriteTo(&b); err != nil { + t.Fatal(err) + } + + f32s := make([]float32, mul(tt.Shape)) + if err := binary.Read(&b, binary.LittleEndian, &f32s); err != nil { + t.Fatal(err) + } + + if !slices.Equal(f32s, []float32{2, 3, 6, 7, 10, 11}) { + t.Fatal("expected data [2, 3, 6, 7, 10, 11], got", f32s) + } + } + }) + + t.Run("uneven split", func(t *testing.T) { + next, stop := iter.Pull(splitDim(&r, 0, + split{Replacer: strings.NewReplacer("a", "x"), dim: 2}, + split{Replacer: strings.NewReplacer("b", "y"), dim: 1}, + )) + defer stop() + + { + tt, ok := next() + if !ok { + t.Fatal("expected at least one split") + } + + if tt.Name != "x.b" { + t.Fatal("expected name 'x.b', got", tt.Name) + } + + if !slices.Equal(tt.Shape, []uint64{2, 4}) { + t.Fatal("expected shape [2, 4], got", tt.Shape) + } + + var b bytes.Buffer + if _, err := tt.WriteTo(&b); err != nil { + t.Fatal(err) + } + + f32s := make([]float32, mul(tt.Shape)) + if err := binary.Read(&b, binary.LittleEndian, &f32s); err != nil { + t.Fatal(err) + } + + if !slices.Equal(f32s, []float32{0, 1, 2, 3, 4, 5, 6, 7}) { + t.Fatal("expected data [0, 1, 2, 3, 4, 5, 6, 7], got", f32s) + } + } + + { + tt, ok := next() + if !ok { + t.Fatal("expected at least one split") + } + + if tt.Name != "a.y" { + t.Fatal("expected name 'a.y', got", tt.Name) + } + + if !slices.Equal(tt.Shape, []uint64{1, 4}) { + t.Fatal("expected shape [1, 4], got", tt.Shape) + } + + var b bytes.Buffer + if _, err := tt.WriteTo(&b); err != nil { + t.Fatal(err) + } + + f32s := make([]float32, mul(tt.Shape)) + if err := binary.Read(&b, binary.LittleEndian, &f32s); err != nil { + t.Fatal(err) + } + + if !slices.Equal(f32s, []float32{8, 9, 10, 11}) { + t.Fatal("expected data [8, 9, 10, 11], got", f32s) + } + } + }) + + t.Run("split with transpose", func(t *testing.T) { + next, stop := iter.Pull(splitDim(&r, 1, + split{Replacer: strings.NewReplacer("a", "x")}, + split{Replacer: strings.NewReplacer("b", "y"), fn: func(tt tensor.Tensor) (tensor.Tensor, error) { + return tensor.Transpose(tt, 1, 0) + }}, + )) + defer stop() + + { + tt, ok := next() + if !ok { + t.Fatal("expected at least one split") + } + + if tt.Name != "x.b" { + t.Fatal("expected name 'x.b', got", tt.Name) + } + + if !slices.Equal(tt.Shape, []uint64{3, 2}) { + t.Fatal("expected shape [3, 2], got", tt.Shape) + } + + var b bytes.Buffer + if _, err := tt.WriteTo(&b); err != nil { + t.Fatal(err) + } + + f32s := make([]float32, mul(tt.Shape)) + if err := binary.Read(&b, binary.LittleEndian, &f32s); err != nil { + t.Fatal(err) + } + + if !slices.Equal(f32s, []float32{0, 1, 4, 5, 8, 9}) { + t.Fatal("expected data [0, 1, 4, 5, 8, 9], got", f32s) + } + } + + { + tt, ok := next() + if !ok { + t.Fatal("expected at least one split") + } + + if tt.Name != "a.y" { + t.Fatal("expected name 'a.y', got", tt.Name) + } + + if !slices.Equal(tt.Shape, []uint64{3, 2}) { + t.Fatal("expected shape [3, 2], got", tt.Shape) + } + + var b bytes.Buffer + if _, err := tt.WriteTo(&b); err != nil { + t.Fatal(err) + } + + f32s := make([]float32, mul(tt.Shape)) + if err := binary.Read(&b, binary.LittleEndian, &f32s); err != nil { + t.Fatal(err) + } + + if !slices.Equal(f32s, []float32{2, 6, 10, 3, 7, 11}) { + t.Fatal("expected data [2, 6, 10, 3, 7, 11], got", f32s) + } + } + }) +} From 6b04cad7e816d1a119559e092d59f4fbaa6c3a0b Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Thu, 12 Jun 2025 11:04:11 -0700 Subject: [PATCH 42/54] feat: incremental gguf parser (#10822) * incremental gguf parser * gguf: update test to not rely on gguf on disc * re-use existing create gguf * read capabilities from gguf kv * kv exists * update tests * s/doneFunc/successFunc/g * new buffered reader --------- Co-authored-by: Bruce MacDonald --- fs/gguf/gguf.go | 347 ++++++++++++++++++++++++++++++++++++ fs/gguf/gguf_test.go | 249 ++++++++++++++++++++++++++ fs/gguf/keyvalue.go | 90 ++++++++++ fs/gguf/keyvalue_test.go | 208 +++++++++++++++++++++ fs/gguf/lazy.go | 89 +++++++++ fs/gguf/reader.go | 23 +++ fs/gguf/tensor.go | 288 ++++++++++++++++++++++++++++++ go.mod | 2 +- go.sum | 4 +- server/images.go | 24 ++- server/images_test.go | 165 +++++------------ server/quantization_test.go | 12 +- server/sched_test.go | 20 +-- 13 files changed, 1357 insertions(+), 164 deletions(-) create mode 100644 fs/gguf/gguf.go create mode 100644 fs/gguf/gguf_test.go create mode 100644 fs/gguf/keyvalue.go create mode 100644 fs/gguf/keyvalue_test.go create mode 100644 fs/gguf/lazy.go create mode 100644 fs/gguf/reader.go create mode 100644 fs/gguf/tensor.go diff --git a/fs/gguf/gguf.go b/fs/gguf/gguf.go new file mode 100644 index 000000000..ebb9286f3 --- /dev/null +++ b/fs/gguf/gguf.go @@ -0,0 +1,347 @@ +package gguf + +import ( + "bytes" + "cmp" + "encoding/binary" + "errors" + "fmt" + "io" + "iter" + "os" + "slices" + "strings" +) + +const ( + typeUint8 uint32 = iota + typeInt8 + typeUint16 + typeInt16 + typeUint32 + typeInt32 + typeFloat32 + typeBool + typeString + typeArray + typeUint64 + typeInt64 + typeFloat64 +) + +var ErrUnsupported = errors.New("unsupported") + +type File struct { + Magic [4]byte + Version uint32 + + keyValues *lazy[KeyValue] + tensors *lazy[TensorInfo] + offset int64 + + file *os.File + reader *bufferedReader + bts []byte +} + +func Open(path string) (f *File, err error) { + f = &File{bts: make([]byte, 4096)} + f.file, err = os.Open(path) + if err != nil { + return nil, err + } + + f.reader = newBufferedReader(f.file, 32<<10) + + if err := binary.Read(f.reader, binary.LittleEndian, &f.Magic); err != nil { + return nil, err + } + + if bytes.Equal(f.Magic[:], []byte("gguf")) { + return nil, fmt.Errorf("%w file type %v", ErrUnsupported, f.Magic) + } + + if err := binary.Read(f.reader, binary.LittleEndian, &f.Version); err != nil { + return nil, err + } + + if f.Version != 3 { + return nil, fmt.Errorf("%w version %v", ErrUnsupported, f.Version) + } + + f.tensors, err = newLazy(f, f.readTensor) + if err != nil { + return nil, err + } + + f.tensors.successFunc = func() error { + offset := f.reader.offset + + alignment := cmp.Or(f.KeyValue("general.alignment").Int(), 32) + f.offset = offset + (alignment-offset%alignment)%alignment + return nil + } + + f.keyValues, err = newLazy(f, f.readKeyValue) + if err != nil { + return nil, err + } + + return f, nil +} + +func (f *File) readTensor() (TensorInfo, error) { + name, err := readString(f) + if err != nil { + return TensorInfo{}, err + } + + dims, err := read[uint32](f) + if err != nil { + return TensorInfo{}, err + } + + shape := make([]uint64, dims) + for i := range dims { + shape[i], err = read[uint64](f) + if err != nil { + return TensorInfo{}, err + } + } + + type_, err := read[uint32](f) + if err != nil { + return TensorInfo{}, err + } + + offset, err := read[uint64](f) + if err != nil { + return TensorInfo{}, err + } + + return TensorInfo{ + Name: name, + Offset: offset, + Shape: shape, + Type: TensorType(type_), + }, nil +} + +func (f *File) readKeyValue() (KeyValue, error) { + key, err := readString(f) + if err != nil { + return KeyValue{}, err + } + + t, err := read[uint32](f) + if err != nil { + return KeyValue{}, err + } + + value, err := func() (any, error) { + switch t { + case typeUint8: + return read[uint8](f) + case typeInt8: + return read[int8](f) + case typeUint16: + return read[uint16](f) + case typeInt16: + return read[int16](f) + case typeUint32: + return read[uint32](f) + case typeInt32: + return read[int32](f) + case typeUint64: + return read[uint64](f) + case typeInt64: + return read[int64](f) + case typeFloat32: + return read[float32](f) + case typeFloat64: + return read[float64](f) + case typeBool: + return read[bool](f) + case typeString: + return readString(f) + case typeArray: + return readArray(f) + default: + return nil, fmt.Errorf("%w type %d", ErrUnsupported, t) + } + }() + if err != nil { + return KeyValue{}, err + } + + return KeyValue{ + Key: key, + Value: Value{value}, + }, nil +} + +func read[T any](f *File) (t T, err error) { + err = binary.Read(f.reader, binary.LittleEndian, &t) + return t, err +} + +func readString(f *File) (string, error) { + n, err := read[uint64](f) + if err != nil { + return "", err + } + + if int(n) > len(f.bts) { + f.bts = make([]byte, n) + } + + bts := f.bts[:n] + if _, err := io.ReadFull(f.reader, bts); err != nil { + return "", err + } + defer clear(bts) + + return string(bts), nil +} + +func readArray(f *File) (any, error) { + t, err := read[uint32](f) + if err != nil { + return nil, err + } + + n, err := read[uint64](f) + if err != nil { + return nil, err + } + + switch t { + case typeUint8: + return readArrayData[uint8](f, n) + case typeInt8: + return readArrayData[int8](f, n) + case typeUint16: + return readArrayData[uint16](f, n) + case typeInt16: + return readArrayData[int16](f, n) + case typeUint32: + return readArrayData[uint32](f, n) + case typeInt32: + return readArrayData[int32](f, n) + case typeUint64: + return readArrayData[uint64](f, n) + case typeInt64: + return readArrayData[int64](f, n) + case typeFloat32: + return readArrayData[float32](f, n) + case typeFloat64: + return readArrayData[float64](f, n) + case typeBool: + return readArrayData[bool](f, n) + case typeString: + return readArrayString(f, n) + default: + return nil, fmt.Errorf("%w type %d", ErrUnsupported, t) + } +} + +func readArrayData[T any](f *File, n uint64) (s []T, err error) { + s = make([]T, n) + for i := range n { + e, err := read[T](f) + if err != nil { + return nil, err + } + + s[i] = e + } + + return s, nil +} + +func readArrayString(f *File, n uint64) (s []string, err error) { + s = make([]string, n) + for i := range n { + e, err := readString(f) + if err != nil { + return nil, err + } + + s[i] = e + } + + return s, nil +} + +func (f *File) Close() error { + f.keyValues.stop() + f.tensors.stop() + return f.file.Close() +} + +func (f *File) KeyValue(key string) KeyValue { + if !strings.HasPrefix(key, "general.") && !strings.HasPrefix(key, "tokenizer.") { + key = f.KeyValue("general.architecture").String() + "." + key + } + + if index := slices.IndexFunc(f.keyValues.values, func(kv KeyValue) bool { + return kv.Key == key + }); index >= 0 { + return f.keyValues.values[index] + } + + for keyValue, ok := f.keyValues.next(); ok; keyValue, ok = f.keyValues.next() { + if keyValue.Key == key { + return keyValue + } + } + + return KeyValue{} +} + +func (f *File) NumKeyValues() int { + return int(f.keyValues.count) +} + +func (f *File) KeyValues() iter.Seq2[int, KeyValue] { + return f.keyValues.All() +} + +func (f *File) TensorInfo(name string) TensorInfo { + if index := slices.IndexFunc(f.tensors.values, func(t TensorInfo) bool { + return t.Name == name + }); index >= 0 { + return f.tensors.values[index] + } + + // fast-forward through key values if we haven't already + _ = f.keyValues.rest() + for tensor, ok := f.tensors.next(); ok; tensor, ok = f.tensors.next() { + if tensor.Name == name { + return tensor + } + } + + return TensorInfo{} +} + +func (f *File) NumTensors() int { + return int(f.tensors.count) +} + +func (f *File) TensorInfos() iter.Seq2[int, TensorInfo] { + // fast forward through key values if we haven't already + f.keyValues.rest() + return f.tensors.All() +} + +func (f *File) TensorReader(name string) (TensorInfo, io.Reader, error) { + t := f.TensorInfo(name) + if t.NumBytes() == 0 { + return TensorInfo{}, nil, fmt.Errorf("tensor %s not found", name) + } + + // fast forward through tensor info if we haven't already + _ = f.tensors.rest() + return t, io.NewSectionReader(f.file, f.offset+int64(t.Offset), t.NumBytes()), nil +} diff --git a/fs/gguf/gguf_test.go b/fs/gguf/gguf_test.go new file mode 100644 index 000000000..eea28a480 --- /dev/null +++ b/fs/gguf/gguf_test.go @@ -0,0 +1,249 @@ +package gguf_test + +import ( + "bytes" + "os" + "strconv" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/ollama/ollama/fs/ggml" + "github.com/ollama/ollama/fs/gguf" +) + +func createBinFile(tb testing.TB) string { + tb.Helper() + f, err := os.CreateTemp(tb.TempDir(), "") + if err != nil { + tb.Fatal(err) + } + defer f.Close() + + kv := ggml.KV{ + "general.architecture": "llama", + "llama.block_count": uint32(8), + "llama.embedding_length": uint32(3), + "llama.attention.head_count": uint32(2), + "llama.attention.head_count_kv": uint32(2), + "llama.attention.key_length": uint32(3), + "llama.rope.dimension_count": uint32(4), + "llama.rope.freq_base": float32(10000.0), + "llama.rope.freq_scale": float32(1.0), + "llama.attention.layer_norm_rms_epsilon": float32(1e-6), + "tokenizer.ggml.eos_token_id": uint32(0), + "tokenizer.ggml.eos_token_ids": []int32{1, 2, 3}, + "tokenizer.ggml.tokens": []string{"hello", "world"}, + "tokenizer.ggml.scores": []float32{0, 1}, + } + + tensors := []*ggml.Tensor{ + { + Name: "token_embd.weight", + Kind: 0, + Shape: []uint64{2, 3}, + WriterTo: bytes.NewBuffer(make([]byte, 4*2*3)), + }, + { + Name: "output.weight", + Kind: 0, + Shape: []uint64{3, 2}, + WriterTo: bytes.NewBuffer(make([]byte, 4*3*2)), + }, + } + + for i := range 8 { + tensors = append(tensors, &ggml.Tensor{ + Name: "blk." + strconv.Itoa(i) + ".attn_q.weight", + Kind: 0, + Shape: []uint64{3, 3}, + WriterTo: bytes.NewBuffer(make([]byte, 4*3*3)), + }, &ggml.Tensor{ + Name: "blk." + strconv.Itoa(i) + ".attn_k.weight", + Kind: 0, + Shape: []uint64{3, 3}, + WriterTo: bytes.NewBuffer(make([]byte, 4*3*3)), + }, &ggml.Tensor{ + Name: "blk." + strconv.Itoa(i) + ".attn_v.weight", + Kind: 0, + Shape: []uint64{3, 3}, + WriterTo: bytes.NewBuffer(make([]byte, 4*3*3)), + }, &ggml.Tensor{ + Name: "blk." + strconv.Itoa(i) + ".attn_output.weight", + Kind: 0, + Shape: []uint64{3, 3}, + WriterTo: bytes.NewBuffer(make([]byte, 4*3*3)), + }) + } + + if err := ggml.WriteGGUF(f, kv, tensors); err != nil { + tb.Fatal(err) + } + + return f.Name() +} + +func TestRead(t *testing.T) { + f, err := gguf.Open(createBinFile(t)) + if err != nil { + t.Fatal(err) + } + defer f.Close() + + if got := f.KeyValue("does.not.exist").Valid(); got { + t.Errorf(`KeyValue("does.not.exist").Exists() = %v, want false`, got) + } + + if got := f.KeyValue("general.architecture").String(); got != "llama" { + t.Errorf(`KeyValue("general.architecture").String() = %q, want %q`, got, "llama") + } + + if got := f.TensorInfo("token_embd.weight"); got.Name != "token_embd.weight" { + t.Errorf(`TensorInfo("token_embd.weight").Name = %q, want %q`, got.Name, "token_embd.weight") + } else if diff := cmp.Diff(got.Shape, []uint64{2, 3}); diff != "" { + t.Errorf(`TensorInfo("token_embd.weight").Shape mismatch (-got +want):\n%s`, diff) + } else if got.Type != gguf.TensorTypeF32 { + t.Errorf(`TensorInfo("token_embd.weight").Type = %d, want %d`, got.Type, gguf.TensorTypeF32) + } + + if got := f.KeyValue("block_count").Uint(); got != 8 { + t.Errorf(`KeyValue("block_count").Uint() = %d, want %d`, got, 8) + } + + if diff := cmp.Diff(f.KeyValue("tokenizer.ggml.tokens").Strings(), []string{"hello", "world"}); diff != "" { + t.Errorf("KeyValue(\"tokenizer.ggml.tokens\").Strings() mismatch (-got +want):\n%s", diff) + } + + if diff := cmp.Diff(f.KeyValue("tokenizer.ggml.scores").Floats(), []float64{0, 1}); diff != "" { + t.Errorf("KeyValue(\"tokenizer.ggml.scores\").Ints() mismatch (-got +want):\n%s", diff) + } + + var kvs []string + for _, kv := range f.KeyValues() { + if !kv.Valid() { + t.Error("found invalid key-value pair:", kv) + } + + kvs = append(kvs, kv.Key) + } + + if len(kvs) != f.NumKeyValues() { + t.Errorf("iterated key count = %d, want %d", len(kvs), f.NumKeyValues()) + } + + if diff := cmp.Diff(kvs, []string{ + "general.architecture", + "llama.block_count", + "llama.embedding_length", + "llama.attention.head_count", + "llama.attention.head_count_kv", + "llama.attention.key_length", + "llama.rope.dimension_count", + "llama.rope.freq_base", + "llama.rope.freq_scale", + "llama.attention.layer_norm_rms_epsilon", + "tokenizer.ggml.eos_token_id", + "tokenizer.ggml.eos_token_ids", + "tokenizer.ggml.tokens", + "tokenizer.ggml.scores", + }, cmpopts.SortSlices(strings.Compare)); diff != "" { + t.Errorf("KeyValues() mismatch (-got +want):\n%s", diff) + } + + var tis []string + for _, ti := range f.TensorInfos() { + if !ti.Valid() { + t.Error("found invalid tensor info:", ti) + } + + tis = append(tis, ti.Name) + } + + if len(tis) != f.NumTensors() { + t.Errorf("iterated tensor count = %d, want %d", len(tis), f.NumTensors()) + } + + if diff := cmp.Diff(tis, []string{ + "token_embd.weight", + "output.weight", + "blk.0.attn_q.weight", + "blk.0.attn_k.weight", + "blk.0.attn_v.weight", + "blk.0.attn_output.weight", + "blk.1.attn_q.weight", + "blk.1.attn_k.weight", + "blk.1.attn_v.weight", + "blk.1.attn_output.weight", + "blk.2.attn_q.weight", + "blk.2.attn_k.weight", + "blk.2.attn_v.weight", + "blk.2.attn_output.weight", + "blk.3.attn_q.weight", + "blk.3.attn_k.weight", + "blk.3.attn_v.weight", + "blk.3.attn_output.weight", + "blk.4.attn_q.weight", + "blk.4.attn_k.weight", + "blk.4.attn_v.weight", + "blk.4.attn_output.weight", + "blk.5.attn_q.weight", + "blk.5.attn_k.weight", + "blk.5.attn_v.weight", + "blk.5.attn_output.weight", + "blk.6.attn_q.weight", + "blk.6.attn_k.weight", + "blk.6.attn_v.weight", + "blk.6.attn_output.weight", + "blk.7.attn_q.weight", + "blk.7.attn_k.weight", + "blk.7.attn_v.weight", + "blk.7.attn_output.weight", + }, cmpopts.SortSlices(strings.Compare)); diff != "" { + t.Errorf("TensorInfos() mismatch (-got +want):\n%s", diff) + } + + ti, r, err := f.TensorReader("output.weight") + if err != nil { + t.Fatalf(`TensorReader("output.weight") error: %v`, err) + } + + if ti.Name != "output.weight" { + t.Errorf(`TensorReader("output.weight").Name = %q, want %q`, ti.Name, "output.weight") + } else if diff := cmp.Diff(ti.Shape, []uint64{3, 2}); diff != "" { + t.Errorf(`TensorReader("output.weight").Shape mismatch (-got +want):\n%s`, diff) + } else if ti.Type != gguf.TensorTypeF32 { + t.Errorf(`TensorReader("output.weight").Type = %d, want %d`, ti.Type, gguf.TensorTypeF32) + } + + var b bytes.Buffer + if _, err := b.ReadFrom(r); err != nil { + t.Fatalf(`ReadFrom TensorReader("output.weight") error: %v`, err) + } + + if b.Len() != int(ti.NumBytes()) { + t.Errorf(`ReadFrom TensorReader("output.weight") length = %d, want %d`, b.Len(), ti.NumBytes()) + } +} + +func BenchmarkRead(b *testing.B) { + b.ReportAllocs() + + p := createBinFile(b) + for b.Loop() { + f, err := gguf.Open(p) + if err != nil { + b.Fatal(err) + } + + if got := f.KeyValue("general.architecture").String(); got != "llama" { + b.Errorf("got = %q, want %q", got, "llama") + } + + // Iterate through some tensors + for range f.TensorInfos() { + } + + f.Close() + } +} diff --git a/fs/gguf/keyvalue.go b/fs/gguf/keyvalue.go new file mode 100644 index 000000000..5843326c1 --- /dev/null +++ b/fs/gguf/keyvalue.go @@ -0,0 +1,90 @@ +package gguf + +import ( + "reflect" + "slices" +) + +type KeyValue struct { + Key string + Value +} + +func (kv KeyValue) Valid() bool { + return kv.Key != "" && kv.Value.value != nil +} + +type Value struct { + value any +} + +func value[T any](v Value, kinds ...reflect.Kind) (t T) { + vv := reflect.ValueOf(v.value) + if slices.Contains(kinds, vv.Kind()) { + t = vv.Convert(reflect.TypeOf(t)).Interface().(T) + } + return +} + +func values[T any](v Value, kinds ...reflect.Kind) (ts []T) { + switch vv := reflect.ValueOf(v.value); vv.Kind() { + case reflect.Slice: + if slices.Contains(kinds, vv.Type().Elem().Kind()) { + ts = make([]T, vv.Len()) + for i := range vv.Len() { + ts[i] = vv.Index(i).Convert(reflect.TypeOf(ts[i])).Interface().(T) + } + } + } + return +} + +// Int returns Value as a signed integer. If it is not a signed integer, it returns 0. +func (v Value) Int() int64 { + return value[int64](v, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64) +} + +// Ints returns Value as a signed integer slice. If it is not a signed integer slice, it returns nil. +func (v Value) Ints() (i64s []int64) { + return values[int64](v, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64) +} + +// Uint converts an unsigned integer value to uint64. If the value is not a unsigned integer, it returns 0. +func (v Value) Uint() uint64 { + return value[uint64](v, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64) +} + +// Uints returns Value as a unsigned integer slice. If it is not a unsigned integer slice, it returns nil. +func (v Value) Uints() (u64s []uint64) { + return values[uint64](v, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64) +} + +// Float returns Value as a float. If it is not a float, it returns 0. +func (v Value) Float() float64 { + return value[float64](v, reflect.Float32, reflect.Float64) +} + +// Floats returns Value as a float slice. If it is not a float slice, it returns nil. +func (v Value) Floats() (f64s []float64) { + return values[float64](v, reflect.Float32, reflect.Float64) +} + +// Bool returns Value as a boolean. If it is not a boolean, it returns false. +func (v Value) Bool() bool { + return value[bool](v, reflect.Bool) +} + +// Bools returns Value as a boolean slice. If it is not a boolean slice, it returns nil. +func (v Value) Bools() (bools []bool) { + return values[bool](v, reflect.Bool) +} + +// String returns Value as a string. If it is not a string, it returns an empty string. +func (v Value) String() string { + return value[string](v, reflect.String) +} + +// Strings returns Value as a string slice. If it is not a string slice, it returns nil. +func (v Value) Strings() (strings []string) { + return values[string](v, reflect.String) +} diff --git a/fs/gguf/keyvalue_test.go b/fs/gguf/keyvalue_test.go new file mode 100644 index 000000000..2caacc538 --- /dev/null +++ b/fs/gguf/keyvalue_test.go @@ -0,0 +1,208 @@ +package gguf + +import ( + "testing" + + "github.com/google/go-cmp/cmp" +) + +func split(name string, values map[string][]any) (matched []any, unmatched []any) { + for key, value := range values { + if key == name { + matched = value + } else { + unmatched = append(unmatched, value...) + } + } + return +} + +func TestValue(t *testing.T) { + values := map[string][]any{ + "int64": {int(42), int8(42), int16(42), int32(42), int64(42)}, + "uint64": {uint(42), uint8(42), uint16(42), uint32(42), uint64(42)}, + "float64": {float32(42), float64(42)}, + "string": {"42", "hello"}, + "bool": {true, false}, + } + + t.Run("int64", func(t *testing.T) { + matched, unmatched := split("int64", values) + for _, v := range matched { + kv := KeyValue{"key", Value{v}} + if i64 := kv.Int(); i64 != 42 { + t.Errorf("expected 42, got %d", i64) + } + } + + for _, v := range unmatched { + kv := KeyValue{"key", Value{v}} + if i64 := kv.Int(); i64 != 0 { + t.Errorf("expected 42, got %d", i64) + } + } + }) + + t.Run("uint64", func(t *testing.T) { + matched, unmatched := split("uint64", values) + for _, v := range matched { + kv := KeyValue{"key", Value{v}} + if u64 := kv.Uint(); u64 != 42 { + t.Errorf("expected 42, got %d", u64) + } + } + + for _, v := range unmatched { + kv := KeyValue{"key", Value{v}} + if u64 := kv.Uint(); u64 != 0 { + t.Errorf("expected 42, got %d", u64) + } + } + }) + + t.Run("float64", func(t *testing.T) { + matched, unmatched := split("float64", values) + for _, v := range matched { + kv := KeyValue{"key", Value{v}} + if f64 := kv.Float(); f64 != 42 { + t.Errorf("expected 42, got %f", f64) + } + } + + for _, v := range unmatched { + kv := KeyValue{"key", Value{v}} + if f64 := kv.Float(); f64 != 0 { + t.Errorf("expected 42, got %f", f64) + } + } + }) + + t.Run("string", func(t *testing.T) { + matched, unmatched := split("string", values) + for _, v := range matched { + kv := KeyValue{"key", Value{v}} + if s := kv.String(); s != v { + t.Errorf("expected 42, got %s", s) + } + } + + for _, v := range unmatched { + kv := KeyValue{"key", Value{v}} + if s := kv.String(); s != "" { + t.Errorf("expected 42, got %s", s) + } + } + }) + + t.Run("bool", func(t *testing.T) { + matched, unmatched := split("bool", values) + for _, v := range matched { + kv := KeyValue{"key", Value{v}} + if b := kv.Bool(); b != v { + t.Errorf("expected true, got %v", b) + } + } + + for _, v := range unmatched { + kv := KeyValue{"key", Value{v}} + if b := kv.Bool(); b != false { + t.Errorf("expected false, got %v", b) + } + } + }) +} + +func TestValues(t *testing.T) { + values := map[string][]any{ + "int64s": {[]int{42}, []int8{42}, []int16{42}, []int32{42}, []int64{42}}, + "uint64s": {[]uint{42}, []uint8{42}, []uint16{42}, []uint32{42}, []uint64{42}}, + "float64s": {[]float32{42}, []float64{42}}, + "strings": {[]string{"42"}, []string{"hello"}}, + "bools": {[]bool{true}, []bool{false}}, + } + + t.Run("int64s", func(t *testing.T) { + matched, unmatched := split("int64s", values) + for _, v := range matched { + kv := KeyValue{"key", Value{v}} + if diff := cmp.Diff(kv.Ints(), []int64{42}); diff != "" { + t.Errorf("diff: %s", diff) + } + } + + for _, v := range unmatched { + kv := KeyValue{"key", Value{v}} + if i64s := kv.Ints(); i64s != nil { + t.Errorf("expected nil, got %v", i64s) + } + } + }) + + t.Run("uint64s", func(t *testing.T) { + matched, unmatched := split("uint64s", values) + for _, v := range matched { + kv := KeyValue{"key", Value{v}} + if diff := cmp.Diff(kv.Uints(), []uint64{42}); diff != "" { + t.Errorf("diff: %s", diff) + } + } + + for _, v := range unmatched { + kv := KeyValue{"key", Value{v}} + if u64s := kv.Uints(); u64s != nil { + t.Errorf("expected nil, got %v", u64s) + } + } + }) + + t.Run("float64s", func(t *testing.T) { + matched, unmatched := split("float64s", values) + for _, v := range matched { + kv := KeyValue{"key", Value{v}} + if diff := cmp.Diff(kv.Floats(), []float64{42}); diff != "" { + t.Errorf("diff: %s", diff) + } + } + + for _, v := range unmatched { + kv := KeyValue{"key", Value{v}} + if f64s := kv.Floats(); f64s != nil { + t.Errorf("expected nil, got %v", f64s) + } + } + }) + + t.Run("strings", func(t *testing.T) { + matched, unmatched := split("strings", values) + for _, v := range matched { + kv := KeyValue{"key", Value{v}} + if diff := cmp.Diff(kv.Strings(), v); diff != "" { + t.Errorf("diff: %s", diff) + } + } + + for _, v := range unmatched { + kv := KeyValue{"key", Value{v}} + if s := kv.Strings(); s != nil { + t.Errorf("expected nil, got %v", s) + } + } + }) + + t.Run("bools", func(t *testing.T) { + matched, unmatched := split("bools", values) + for _, v := range matched { + kv := KeyValue{"key", Value{v}} + if diff := cmp.Diff(kv.Bools(), v); diff != "" { + t.Errorf("diff: %s", diff) + } + } + + for _, v := range unmatched { + kv := KeyValue{"key", Value{v}} + if b := kv.Bools(); b != nil { + t.Errorf("expected nil, got %v", b) + } + } + }) +} diff --git a/fs/gguf/lazy.go b/fs/gguf/lazy.go new file mode 100644 index 000000000..16ab99093 --- /dev/null +++ b/fs/gguf/lazy.go @@ -0,0 +1,89 @@ +package gguf + +import ( + "encoding/binary" + "iter" + "log/slog" +) + +type lazy[T any] struct { + count uint64 + next func() (T, bool) + stop func() + values []T + + // successFunc is called when all values have been successfully read. + successFunc func() error +} + +func newLazy[T any](f *File, fn func() (T, error)) (*lazy[T], error) { + it := lazy[T]{} + if err := binary.Read(f.reader, binary.LittleEndian, &it.count); err != nil { + return nil, err + } + + it.values = make([]T, 0) + it.next, it.stop = iter.Pull(func(yield func(T) bool) { + for i := range it.count { + t, err := fn() + if err != nil { + slog.Error("error reading tensor", "index", i, "error", err) + return + } + + it.values = append(it.values, t) + if !yield(t) { + break + } + } + + if it.successFunc != nil { + it.successFunc() + } + }) + + return &it, nil +} + +func (g *lazy[T]) Values() iter.Seq[T] { + return func(yield func(T) bool) { + for _, v := range g.All() { + if !yield(v) { + break + } + } + } +} + +func (g *lazy[T]) All() iter.Seq2[int, T] { + return func(yield func(int, T) bool) { + for i := range int(g.count) { + if i < len(g.values) { + if !yield(i, g.values[i]) { + break + } + } else { + t, ok := g.next() + if !ok { + break + } + + if !yield(i, t) { + break + } + } + } + } +} + +func (g *lazy[T]) rest() (collected bool) { + for { + _, ok := g.next() + collected = collected || ok + if !ok { + break + } + } + + return collected +} diff --git a/fs/gguf/reader.go b/fs/gguf/reader.go new file mode 100644 index 000000000..0bd761840 --- /dev/null +++ b/fs/gguf/reader.go @@ -0,0 +1,23 @@ +package gguf + +import ( + "bufio" + "io" +) + +type bufferedReader struct { + offset int64 + *bufio.Reader +} + +func newBufferedReader(rs io.ReadSeeker, size int) *bufferedReader { + return &bufferedReader{ + Reader: bufio.NewReaderSize(rs, size), + } +} + +func (rs *bufferedReader) Read(p []byte) (n int, err error) { + n, err = rs.Reader.Read(p) + rs.offset += int64(n) + return n, err +} diff --git a/fs/gguf/tensor.go b/fs/gguf/tensor.go new file mode 100644 index 000000000..194c1d739 --- /dev/null +++ b/fs/gguf/tensor.go @@ -0,0 +1,288 @@ +package gguf + +import ( + "log/slog" + "strings" +) + +type TensorInfo struct { + Name string + Offset uint64 + Shape []uint64 + Type TensorType +} + +func (ti TensorInfo) Valid() bool { + return ti.Name != "" && ti.NumBytes() > 0 +} + +func (ti TensorInfo) NumValues() int64 { + var numItems int64 = 1 + for _, dim := range ti.Shape { + numItems *= int64(dim) + } + return numItems +} + +// NumBytes returns the number of bytes in the tensor. +func (ti TensorInfo) NumBytes() int64 { + return int64(float64(ti.NumValues()) * ti.Type.NumBytes()) +} + +func (ti TensorInfo) LogValue() slog.Value { + return slog.GroupValue( + slog.String("name", ti.Name), + slog.Int64("offset", int64(ti.Offset)), + slog.Any("shape", ti.Shape), + slog.Int64("num_values", ti.NumValues()), + slog.Int64("num_bytes", ti.NumBytes()), + slog.Any("type", ti.Type), + ) +} + +type TensorType uint32 + +const ( + TensorTypeF32 TensorType = iota + TensorTypeF16 + TensorTypeQ4_0 + TensorTypeQ4_1 + + // unexported // unused in gguf + tensorTypeQ4_2 + tensorTypeQ4_3 + + TensorTypeQ5_0 + TensorTypeQ5_1 + TensorTypeQ8_0 + TensorTypeQ8_1 + TensorTypeQ2_K + TensorTypeQ3_K + TensorTypeQ4_K + TensorTypeQ5_K + TensorTypeQ6_K + TensorTypeQ8_K + + // unexported // unquantizable by ollama + tensorTypeIQ2_XXS + tensorTypeIQ2_XS + tensorTypeIQ3_XXS + tensorTypeIQ1_S + tensorTypeIQ4_NL + tensorTypeIQ3_S + tensorTypeIQ2_S + tensorTypeIQ4_XS + + TensorTypeI8 + TensorTypeI16 + TensorTypeI32 + TensorTypeI64 + TensorTypeF64 + + // unexported // unquantizable by ollama + tensorTypeIQ1_M + + TensorTypeBF16 + + // unexported // unused in gguf + tensorTypeQ4_0_4_4 + tensorTypeQ4_0_4_8 + tensorTypeQ4_0_8_8 + + // unexported // unquantizable by ollama + tensorTypeTQ1_0 + tensorTypeTQ2_0 + + // unexported // unused in gguf + tensorTypeIQ4_NL_4_4 + tensorTypeIQ4_NL_4_8 + tensorTypeIQ4_NL_8_8 +) + +func (tt TensorType) NumBytes() float64 { + return float64(tt.typeSize()) / float64(tt.blockSize()) +} + +func (tt TensorType) typeSize() int64 { + switch tt { + case TensorTypeF32: + return 4 + case TensorTypeF16: + return 2 + case TensorTypeQ4_0: + return 2 + tt.blockSize()/2 + case TensorTypeQ4_1: + return 2 + 2 + tt.blockSize()/2 + case TensorTypeQ5_0: + return 2 + 4 + tt.blockSize()/2 + case TensorTypeQ5_1: + return 2 + 2 + 4 + tt.blockSize()/2 + case TensorTypeQ8_0: + return 2 + tt.blockSize() + case TensorTypeQ8_1: + return 2 + 2 + tt.blockSize() + case TensorTypeQ2_K: + return tt.blockSize()/16 + tt.blockSize()/4 + 2 + 2 + case TensorTypeQ3_K: + return tt.blockSize()/8 + tt.blockSize()/4 + 12 + 2 + case TensorTypeQ4_K: + return 2 + 2 + 12 + tt.blockSize()/2 + case TensorTypeQ5_K: + return 2 + 2 + 12 + tt.blockSize()/8 + tt.blockSize()/2 + case TensorTypeQ6_K: + return tt.blockSize()/2 + tt.blockSize()/4 + tt.blockSize()/16 + 2 + case TensorTypeQ8_K: + return 4 + tt.blockSize() + 2*tt.blockSize()/16 + case tensorTypeIQ2_XXS: + return 2 + 2*tt.blockSize()/8 + case tensorTypeIQ2_XS: + return 2 + 2*tt.blockSize()/8 + tt.blockSize()/32 + case tensorTypeIQ3_XXS: + return 2 + tt.blockSize()/4 + tt.blockSize()/8 + case tensorTypeIQ1_S: + return 2 + tt.blockSize()/8 + tt.blockSize()/16 + case tensorTypeIQ4_NL: + return 2 + tt.blockSize()/2 + case tensorTypeIQ3_S: + return 2 + tt.blockSize()/4 + tt.blockSize()/8 + tt.blockSize()/32 + 4 + case tensorTypeIQ2_S: + return 2 + tt.blockSize()/4 + tt.blockSize()/16 + case tensorTypeIQ4_XS: + return 2 + 2 + tt.blockSize()/2 + tt.blockSize()/64 + case TensorTypeI8: + return 1 + case TensorTypeI16: + return 2 + case TensorTypeI32: + return 4 + case TensorTypeI64: + return 8 + case TensorTypeF64: + return 8 + case tensorTypeIQ1_M: + return tt.blockSize()/8 + tt.blockSize()/16 + tt.blockSize()/32 + case TensorTypeBF16: + return 2 + default: + return 0 + } +} + +func (tt TensorType) blockSize() int64 { + switch tt { + case TensorTypeF32, + TensorTypeF16, + TensorTypeI8, + TensorTypeI16, + TensorTypeI32, + TensorTypeI64, + TensorTypeF64, + TensorTypeBF16: + return 1 + case TensorTypeQ4_0, + TensorTypeQ4_1, + TensorTypeQ5_0, + TensorTypeQ5_1, + TensorTypeQ8_0, + TensorTypeQ8_1, + tensorTypeIQ4_NL: + return 32 + default: + return 256 + } +} + +func (tt TensorType) String() string { + switch tt { + case TensorTypeF32: + return "f32" + case TensorTypeF16: + return "f16" + case TensorTypeQ4_0: + return "q4_0" + case TensorTypeQ4_1: + return "q4_1" + case tensorTypeQ4_2: + return "q4_2" + case tensorTypeQ4_3: + return "q4_3" + case TensorTypeQ5_0: + return "q5_0" + case TensorTypeQ5_1: + return "q5_1" + case TensorTypeQ8_0: + return "q8_0" + case TensorTypeQ8_1: + return "q8_1" + case TensorTypeQ2_K: + return "q2_k" + case TensorTypeQ3_K: + return "q3_k" + case TensorTypeQ4_K: + return "q4_k" + case TensorTypeQ5_K: + return "q5_k" + case TensorTypeQ6_K: + return "q6_k" + case TensorTypeQ8_K: + return "q8_k" + case tensorTypeIQ2_XXS: + return "iq2_xxs" + case tensorTypeIQ2_XS: + return "iq2_xs" + case tensorTypeIQ3_XXS: + return "iq3_xxs" + case tensorTypeIQ1_S: + return "iq1_s" + case tensorTypeIQ4_NL: + return "iq4_nl" + case tensorTypeIQ3_S: + return "iq3_s" + case tensorTypeIQ2_S: + return "iq2_s" + case tensorTypeIQ4_XS: + return "iq4_xs" + case TensorTypeI8: + return "i8" + case TensorTypeI16: + return "i16" + case TensorTypeI32: + return "i32" + case TensorTypeI64: + return "i64" + case TensorTypeF64: + return "f64" + case tensorTypeIQ1_M: + return "iq1_m" + case TensorTypeBF16: + return "bf16" + case tensorTypeQ4_0_4_4: + return "q4_0_4_4" + case tensorTypeQ4_0_4_8: + return "q4_0_4_8" + case tensorTypeQ4_0_8_8: + return "q4_0_8_8" + case tensorTypeTQ1_0: + return "tq1_0" + case tensorTypeTQ2_0: + return "tq2_0" + case tensorTypeIQ4_NL_4_4: + return "iq4_nl_4_4" + case tensorTypeIQ4_NL_4_8: + return "iq4_nl_4_8" + case tensorTypeIQ4_NL_8_8: + return "iq4_nl_8_8" + default: + return "unknown" + } +} + +func (tt TensorType) LogValue() slog.Value { + return slog.GroupValue( + slog.Uint64("value", uint64(tt)), + slog.String("name", strings.ToUpper(tt.String())), + slog.Int64("size", tt.typeSize()), + slog.Int64("block_size", tt.blockSize()), + slog.Float64("num_bytes", tt.NumBytes()), + ) +} diff --git a/go.mod b/go.mod index 283286b7d..6de5959be 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/d4l3k/go-bfloat16 v0.0.0-20211005043715-690c3bdd05f1 github.com/dlclark/regexp2 v1.11.4 github.com/emirpasic/gods/v2 v2.0.0-alpha - github.com/google/go-cmp v0.6.0 + github.com/google/go-cmp v0.7.0 github.com/mattn/go-runewidth v0.0.14 github.com/nlpodyssey/gopickle v0.3.0 github.com/pdevine/tensor v0.0.0-20240510204454-f88f4562727c diff --git a/go.sum b/go.sum index 5755616f6..c0ab53aab 100644 --- a/go.sum +++ b/go.sum @@ -112,8 +112,8 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= diff --git a/server/images.go b/server/images.go index d6cceff4c..38505cc51 100644 --- a/server/images.go +++ b/server/images.go @@ -23,7 +23,7 @@ import ( "github.com/ollama/ollama/api" "github.com/ollama/ollama/envconfig" - "github.com/ollama/ollama/fs/ggml" + "github.com/ollama/ollama/fs/gguf" "github.com/ollama/ollama/parser" "github.com/ollama/ollama/template" "github.com/ollama/ollama/thinking" @@ -73,22 +73,18 @@ func (m *Model) Capabilities() []model.Capability { capabilities := []model.Capability{} // Check for completion capability - r, err := os.Open(m.ModelPath) + f, err := gguf.Open(m.ModelPath) if err == nil { - defer r.Close() + defer f.Close() - f, err := ggml.Decode(r, 1024) - if err == nil { - if _, ok := f.KV()[fmt.Sprintf("%s.pooling_type", f.KV().Architecture())]; ok { - capabilities = append(capabilities, model.CapabilityEmbedding) - } else { - capabilities = append(capabilities, model.CapabilityCompletion) - } - if _, ok := f.KV()[fmt.Sprintf("%s.vision.block_count", f.KV().Architecture())]; ok { - capabilities = append(capabilities, model.CapabilityVision) - } + if f.KeyValue("pooling_type").Valid() { + capabilities = append(capabilities, model.CapabilityEmbedding) } else { - slog.Error("couldn't decode ggml", "error", err) + // If no embedding is specified, we assume the model supports completion + capabilities = append(capabilities, model.CapabilityCompletion) + } + if f.KeyValue("vision.block_count").Valid() { + capabilities = append(capabilities, model.CapabilityVision) } } else { slog.Error("couldn't open model file", "error", err) diff --git a/server/images_test.go b/server/images_test.go index 363b298e1..a2fba8d98 100644 --- a/server/images_test.go +++ b/server/images_test.go @@ -1,123 +1,42 @@ package server import ( - "bytes" - "encoding/binary" - "errors" - "os" - "path/filepath" "strings" "testing" + "github.com/ollama/ollama/fs/ggml" "github.com/ollama/ollama/template" "github.com/ollama/ollama/types/model" ) -// Constants for GGUF magic bytes and version -var ( - ggufMagic = []byte{0x47, 0x47, 0x55, 0x46} // "GGUF" - ggufVer = uint32(3) // Version 3 -) - -// Helper function to create mock GGUF data -func createMockGGUFData(architecture string, vision bool) []byte { - var buf bytes.Buffer - - // Write GGUF header - buf.Write(ggufMagic) - binary.Write(&buf, binary.LittleEndian, ggufVer) - - // Write tensor count (0 for our test) - var numTensors uint64 = 0 - binary.Write(&buf, binary.LittleEndian, numTensors) - - // Calculate number of metadata entries - numMetaEntries := uint64(1) // architecture entry - if vision { - numMetaEntries++ - } - // Add embedding entry if architecture is "bert" - if architecture == "bert" { - numMetaEntries++ - } - binary.Write(&buf, binary.LittleEndian, numMetaEntries) - - // Write architecture metadata - archKey := "general.architecture" - keyLen := uint64(len(archKey)) - binary.Write(&buf, binary.LittleEndian, keyLen) - buf.WriteString(archKey) - - // String type (8) - var strType uint32 = 8 - binary.Write(&buf, binary.LittleEndian, strType) - - // String length - strLen := uint64(len(architecture)) - binary.Write(&buf, binary.LittleEndian, strLen) - buf.WriteString(architecture) - - if vision { - visionKey := architecture + ".vision.block_count" - keyLen = uint64(len(visionKey)) - binary.Write(&buf, binary.LittleEndian, keyLen) - buf.WriteString(visionKey) - - // uint32 type (4) - var uint32Type uint32 = 4 - binary.Write(&buf, binary.LittleEndian, uint32Type) - - // uint32 value (1) - var countVal uint32 = 1 - binary.Write(&buf, binary.LittleEndian, countVal) - } - // Write embedding metadata if architecture is "bert" - if architecture == "bert" { - poolKey := architecture + ".pooling_type" - keyLen = uint64(len(poolKey)) - binary.Write(&buf, binary.LittleEndian, keyLen) - buf.WriteString(poolKey) - - // uint32 type (4) - var uint32Type uint32 = 4 - binary.Write(&buf, binary.LittleEndian, uint32Type) - - // uint32 value (1) - var poolingVal uint32 = 1 - binary.Write(&buf, binary.LittleEndian, poolingVal) - } - - return buf.Bytes() -} - func TestModelCapabilities(t *testing.T) { - // Create a temporary directory for test files - tempDir := t.TempDir() + // Create completion model (llama architecture without vision) + completionModelPath, _ := createBinFile(t, ggml.KV{ + "general.architecture": "llama", + }, []*ggml.Tensor{}) - // Create different types of mock model files - completionModelPath := filepath.Join(tempDir, "model.bin") - visionModelPath := filepath.Join(tempDir, "vision_model.bin") - embeddingModelPath := filepath.Join(tempDir, "embedding_model.bin") - // Create a simple model file for tests that don't depend on GGUF content - simpleModelPath := filepath.Join(tempDir, "simple_model.bin") + // Create vision model (llama architecture with vision block count) + visionModelPath, _ := createBinFile(t, ggml.KV{ + "general.architecture": "llama", + "llama.vision.block_count": uint32(1), + }, []*ggml.Tensor{}) - if err := errors.Join( - os.WriteFile(completionModelPath, createMockGGUFData("llama", false), 0o644), - os.WriteFile(visionModelPath, createMockGGUFData("llama", true), 0o644), - os.WriteFile(embeddingModelPath, createMockGGUFData("bert", false), 0o644), - os.WriteFile(simpleModelPath, []byte("dummy model data"), 0o644), - ); err != nil { - t.Fatalf("Failed to create model files: %v", err) - } + // Create embedding model (bert architecture with pooling type) + embeddingModelPath, _ := createBinFile(t, ggml.KV{ + "general.architecture": "bert", + "bert.pooling_type": uint32(1), + }, []*ggml.Tensor{}) toolsInsertTemplate, err := template.Parse("{{ .prompt }}{{ if .tools }}{{ .tools }}{{ end }}{{ if .suffix }}{{ .suffix }}{{ end }}") if err != nil { t.Fatalf("Failed to parse template: %v", err) } + chatTemplate, err := template.Parse("{{ .prompt }}") if err != nil { t.Fatalf("Failed to parse template: %v", err) } + toolsTemplate, err := template.Parse("{{ .prompt }}{{ if .tools }}{{ .tools }}{{ end }}") if err != nil { t.Fatalf("Failed to parse template: %v", err) @@ -145,21 +64,13 @@ func TestModelCapabilities(t *testing.T) { }, expectedCaps: []model.Capability{model.CapabilityCompletion, model.CapabilityTools, model.CapabilityInsert}, }, - { - name: "model with tools and insert capability", - model: Model{ - ModelPath: simpleModelPath, - Template: toolsInsertTemplate, - }, - expectedCaps: []model.Capability{model.CapabilityTools, model.CapabilityInsert}, - }, { name: "model with tools capability", model: Model{ - ModelPath: simpleModelPath, + ModelPath: completionModelPath, Template: toolsTemplate, }, - expectedCaps: []model.Capability{model.CapabilityTools}, + expectedCaps: []model.Capability{model.CapabilityCompletion, model.CapabilityTools}, }, { name: "model with vision capability", @@ -224,29 +135,33 @@ func TestModelCapabilities(t *testing.T) { } func TestModelCheckCapabilities(t *testing.T) { - // Create a temporary directory for test files - tempDir := t.TempDir() + // Create simple model file for tests that don't depend on GGUF content + completionModelPath, _ := createBinFile(t, ggml.KV{ + "general.architecture": "llama", + }, []*ggml.Tensor{}) - visionModelPath := filepath.Join(tempDir, "vision_model.bin") - simpleModelPath := filepath.Join(tempDir, "model.bin") - embeddingModelPath := filepath.Join(tempDir, "embedding_model.bin") + // Create vision model (llama architecture with vision block count) + visionModelPath, _ := createBinFile(t, ggml.KV{ + "general.architecture": "llama", + "llama.vision.block_count": uint32(1), + }, []*ggml.Tensor{}) - if err := errors.Join( - os.WriteFile(simpleModelPath, []byte("dummy model data"), 0o644), - os.WriteFile(visionModelPath, createMockGGUFData("llama", true), 0o644), - os.WriteFile(embeddingModelPath, createMockGGUFData("bert", false), 0o644), - ); err != nil { - t.Fatalf("Failed to create model files: %v", err) - } + // Create embedding model (bert architecture with pooling type) + embeddingModelPath, _ := createBinFile(t, ggml.KV{ + "general.architecture": "bert", + "bert.pooling_type": uint32(1), + }, []*ggml.Tensor{}) toolsInsertTemplate, err := template.Parse("{{ .prompt }}{{ if .tools }}{{ .tools }}{{ end }}{{ if .suffix }}{{ .suffix }}{{ end }}") if err != nil { t.Fatalf("Failed to parse template: %v", err) } + chatTemplate, err := template.Parse("{{ .prompt }}") if err != nil { t.Fatalf("Failed to parse template: %v", err) } + toolsTemplate, err := template.Parse("{{ .prompt }}{{ if .tools }}{{ .tools }}{{ end }}") if err != nil { t.Fatalf("Failed to parse template: %v", err) @@ -261,7 +176,7 @@ func TestModelCheckCapabilities(t *testing.T) { { name: "completion model without tools capability", model: Model{ - ModelPath: simpleModelPath, + ModelPath: completionModelPath, Template: chatTemplate, }, checkCaps: []model.Capability{model.CapabilityTools}, @@ -270,7 +185,7 @@ func TestModelCheckCapabilities(t *testing.T) { { name: "model with all needed capabilities", model: Model{ - ModelPath: simpleModelPath, + ModelPath: completionModelPath, Template: toolsInsertTemplate, }, checkCaps: []model.Capability{model.CapabilityTools, model.CapabilityInsert}, @@ -278,7 +193,7 @@ func TestModelCheckCapabilities(t *testing.T) { { name: "model missing insert capability", model: Model{ - ModelPath: simpleModelPath, + ModelPath: completionModelPath, Template: toolsTemplate, }, checkCaps: []model.Capability{model.CapabilityInsert}, @@ -287,7 +202,7 @@ func TestModelCheckCapabilities(t *testing.T) { { name: "model missing vision capability", model: Model{ - ModelPath: simpleModelPath, + ModelPath: completionModelPath, Template: toolsTemplate, }, checkCaps: []model.Capability{model.CapabilityVision}, @@ -312,7 +227,7 @@ func TestModelCheckCapabilities(t *testing.T) { { name: "unknown capability", model: Model{ - ModelPath: simpleModelPath, + ModelPath: completionModelPath, Template: chatTemplate, }, checkCaps: []model.Capability{"unknown"}, diff --git a/server/quantization_test.go b/server/quantization_test.go index 4f717c2c2..8b726c836 100644 --- a/server/quantization_test.go +++ b/server/quantization_test.go @@ -257,16 +257,8 @@ func TestQuantizeModel(t *testing.T) { for _, tt := range cases { t.Run(tt.name, func(t *testing.T) { - f, err := os.CreateTemp(t.TempDir(), tt.name) - if err != nil { - t.Fatal(err.Error()) - } - defer f.Close() - err = fsggml.WriteGGUF(f, tt.kv, tt.tensors) - if err != nil { - t.Fatalf("failed to create initial model: %s", err) - } - fp, err := os.Open(f.Name()) + p, _ := createBinFile(t, tt.kv, tt.tensors) + fp, err := os.Open(p) if err != nil { t.Fatal(err.Error()) } diff --git a/server/sched_test.go b/server/sched_test.go index 01fb9a703..3892fbbab 100644 --- a/server/sched_test.go +++ b/server/sched_test.go @@ -112,11 +112,7 @@ func newScenarioRequest(t *testing.T, ctx context.Context, modelName string, est b.ctx, b.ctxDone = context.WithCancel(ctx) t.Helper() - f, err := os.CreateTemp(t.TempDir(), modelName) - require.NoError(t, err) - defer f.Close() - - require.NoError(t, ggml.WriteGGUF(f, ggml.KV{ + p, _ := createBinFile(t, ggml.KV{ "general.architecture": "llama", "llama.context_length": uint32(32), "llama.embedding_length": uint32(4096), @@ -129,14 +125,14 @@ func newScenarioRequest(t *testing.T, ctx context.Context, modelName string, est }, []*ggml.Tensor{ {Name: "blk.0.attn.weight", Kind: uint32(0), Offset: uint64(0), Shape: []uint64{1, 1, 1, 1}, WriterTo: bytes.NewReader(make([]byte, 32))}, {Name: "output.weight", Kind: uint32(0), Offset: uint64(0), Shape: []uint64{1, 1, 1, 1}, WriterTo: bytes.NewReader(make([]byte, 32))}, - })) - require.NoError(t, err) - - fname := f.Name() - model := &Model{Name: modelName, ModelPath: fname} - b.f, err = llm.LoadModel(model.ModelPath, 0) - require.NoError(t, err) + }) + model := &Model{Name: modelName, ModelPath: p} + f, err := llm.LoadModel(model.ModelPath, 0) + if err != nil { + t.Fatal(err) + } + b.f = f if duration == nil { duration = &api.Duration{Duration: 5 * time.Millisecond} } From 9f8a18ec050ef67fca11d4f9bea0508eece93a68 Mon Sep 17 00:00:00 2001 From: Jeffrey Morgan Date: Thu, 12 Jun 2025 14:18:54 -0700 Subject: [PATCH 43/54] tools: loosen tool parsing to allow for more formats (#11030) --- server/routes.go | 8 +- tools/template.go | 156 +++ tools/template_test.go | 139 +++ tools/testdata/command-r-plus.gotmpl | 67 -- tools/testdata/command-r-plus.out | 39 - tools/testdata/firefunction.gotmpl | 31 - tools/testdata/firefunction.out | 17 - tools/testdata/llama3-groq-tool-use.gotmpl | 43 - tools/testdata/llama3-groq-tool-use.out | 24 - tools/testdata/llama3.2.gotmpl | 44 - tools/testdata/llama3.2.out | 24 - tools/testdata/messages.json | 39 - tools/testdata/mistral.gotmpl | 15 - tools/testdata/mistral.out | 3 - tools/testdata/nemotron.gotmpl | 33 - tools/testdata/nemotron.out | 18 - tools/testdata/qwen2.5.gotmpl | 51 - tools/testdata/qwen2.5.out | 31 - tools/testdata/qwen3.gotmpl | 50 - tools/testdata/qwen3.out | 31 - tools/testdata/tools.json | 30 - tools/testdata/xlam.gotmpl | 45 - tools/testdata/xlam.out | 40 - tools/tools.go | 470 ++++---- tools/tools_test.go | 1246 +++++++++++--------- tools/tools_utils.go | 222 ---- tools/tools_utils_test.go | 497 -------- 27 files changed, 1238 insertions(+), 2175 deletions(-) create mode 100644 tools/template.go create mode 100644 tools/template_test.go delete mode 100644 tools/testdata/command-r-plus.gotmpl delete mode 100644 tools/testdata/command-r-plus.out delete mode 100644 tools/testdata/firefunction.gotmpl delete mode 100644 tools/testdata/firefunction.out delete mode 100644 tools/testdata/llama3-groq-tool-use.gotmpl delete mode 100644 tools/testdata/llama3-groq-tool-use.out delete mode 100644 tools/testdata/llama3.2.gotmpl delete mode 100644 tools/testdata/llama3.2.out delete mode 100644 tools/testdata/messages.json delete mode 100644 tools/testdata/mistral.gotmpl delete mode 100644 tools/testdata/mistral.out delete mode 100644 tools/testdata/nemotron.gotmpl delete mode 100644 tools/testdata/nemotron.out delete mode 100644 tools/testdata/qwen2.5.gotmpl delete mode 100644 tools/testdata/qwen2.5.out delete mode 100644 tools/testdata/qwen3.gotmpl delete mode 100644 tools/testdata/qwen3.out delete mode 100644 tools/testdata/tools.json delete mode 100644 tools/testdata/xlam.gotmpl delete mode 100644 tools/testdata/xlam.out delete mode 100644 tools/tools_utils.go delete mode 100644 tools/tools_utils_test.go diff --git a/server/routes.go b/server/routes.go index 8eda5c73f..cb46cef11 100644 --- a/server/routes.go +++ b/server/routes.go @@ -1526,12 +1526,7 @@ func (s *Server) ChatHandler(c *gin.Context) { var toolParser *tools.Parser if len(req.Tools) > 0 { - toolParser, err = tools.NewParser(m.Template.Template) - if err != nil { - slog.Error("failed to create tool parser", "error", err) - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } + toolParser = tools.NewParser(m.Template.Template, req.Tools) } ch := make(chan any) @@ -1584,6 +1579,7 @@ func (s *Server) ChatHandler(c *gin.Context) { // don't return } else { if r.Done { + res.Message.Content = toolParser.Content() ch <- res } return diff --git a/tools/template.go b/tools/template.go new file mode 100644 index 000000000..e22f06754 --- /dev/null +++ b/tools/template.go @@ -0,0 +1,156 @@ +package tools + +import ( + "bytes" + "log/slog" + "slices" + "strings" + "text/template" + "text/template/parse" +) + +// parseTag finds the tool calling tag from a Go template +// often [TOOL_CALL] or similar by finding the +// first text node after .ToolCalls and returning the content +// if no tag is found, return "{" to indicate that json objects +// should be attempted to be parsed as tool calls +func parseTag(tmpl *template.Template) string { + if tmpl == nil || tmpl.Tree == nil { + slog.Debug("template or tree is nil") + return "{" + } + + tc := findToolCallNode(tmpl.Tree.Root.Nodes) + if tc == nil { + return "{" + } + + tn := findTextNode(tc.List.Nodes) + if tn == nil { + return "{" + } + + tag := string(tn.Text) + tag = strings.ReplaceAll(tag, "\r\n", "\n") + + // avoid parsing { onwards as this may be a tool call + // however keep '{' as a prefix if there is no tag + // so that all json objects will be attempted to + // be parsed as tool calls + tag, _, _ = strings.Cut(tag, "{") + tag = strings.TrimSpace(tag) + if tag == "" { + tag = "{" + } + + return tag +} + +// findToolCallNode searches for and returns an IfNode with .ToolCalls +func findToolCallNode(nodes []parse.Node) *parse.IfNode { + isToolCallsNode := func(n *parse.IfNode) bool { + for _, cmd := range n.Pipe.Cmds { + for _, arg := range cmd.Args { + if field, ok := arg.(*parse.FieldNode); ok { + if slices.Contains(field.Ident, "ToolCalls") { + return true + } + } + } + } + return false + } + + for _, node := range nodes { + switch n := node.(type) { + case *parse.IfNode: + if isToolCallsNode(n) { + return n + } + // Recursively search in nested IfNodes + if result := findToolCallNode(n.List.Nodes); result != nil { + return result + } + if n.ElseList != nil { + if result := findToolCallNode(n.ElseList.Nodes); result != nil { + return result + } + } + case *parse.ListNode: + if result := findToolCallNode(n.Nodes); result != nil { + return result + } + case *parse.RangeNode: + if result := findToolCallNode(n.List.Nodes); result != nil { + return result + } + if n.ElseList != nil { + if result := findToolCallNode(n.ElseList.Nodes); result != nil { + return result + } + } + case *parse.WithNode: + if result := findToolCallNode(n.List.Nodes); result != nil { + return result + } + if n.ElseList != nil { + if result := findToolCallNode(n.ElseList.Nodes); result != nil { + return result + } + } + } + } + return nil +} + +// findTextNode does a depth-first search for the first text content in nodes, +// stopping at template constructs to avoid parsing text after the tool calls +func findTextNode(nodes []parse.Node) *parse.TextNode { + for _, node := range nodes { + switch n := node.(type) { + case *parse.TextNode: + // skip whitespace-only text nodes + if len(bytes.TrimSpace(n.Text)) == 0 { + continue + } + return n + case *parse.IfNode: + if text := findTextNode(n.List.Nodes); text != nil { + return text + } + if n.ElseList != nil { + if text := findTextNode(n.ElseList.Nodes); text != nil { + return text + } + } + return nil + case *parse.ListNode: + if text := findTextNode(n.Nodes); text != nil { + return text + } + case *parse.RangeNode: + if text := findTextNode(n.List.Nodes); text != nil { + return text + } + if n.ElseList != nil { + if text := findTextNode(n.ElseList.Nodes); text != nil { + return text + } + } + return nil + case *parse.WithNode: + if text := findTextNode(n.List.Nodes); text != nil { + return text + } + if n.ElseList != nil { + if text := findTextNode(n.ElseList.Nodes); text != nil { + return text + } + } + return nil + case *parse.ActionNode: + return nil + } + } + return nil +} diff --git a/tools/template_test.go b/tools/template_test.go new file mode 100644 index 000000000..970c0d599 --- /dev/null +++ b/tools/template_test.go @@ -0,0 +1,139 @@ +package tools + +import ( + "testing" + "text/template" +) + +func TestParseTag(t *testing.T) { + cases := []struct { + name string + template string + want string + }{ + { + name: "empty", + template: "", + want: "{", + }, + { + name: "no tag", + template: "{{if .ToolCalls}}{{end}}", + want: "{", + }, + { + name: "no tag with range", + template: "{{if .ToolCalls}}{{range .ToolCalls}}{{ . }}{{end}}{{end}}", + want: "{", + }, + { + name: "tool call with json format", + template: "{{if .ToolCalls}}```json\n{{end}}", + want: "```json", + }, + { + name: "square brackets", + template: "{{if .ToolCalls}}[{{range .ToolCalls}}{{ . }}{{end}}]{{end}}", + want: "[", + }, + { + name: "square brackets with whitespace", + template: "{{if .ToolCalls}}\n [ {{range .ToolCalls}}{{ . }}{{end}}]{{end}}", + want: "[", + }, + { + name: "tailing ]", + template: "{{if .ToolCalls}}{{range .ToolCalls}}{{ . }}{{end}}]{{end}}", + want: "{", + }, + { + name: "whitespace only", + template: "{{if .ToolCalls}} {{range .ToolCalls}}{{ . }}{{end}}{{end}}", + want: "{", + }, + { + name: "whitespace only in range", + template: "{{if .ToolCalls}}{{range .ToolCalls}}\n{{ . }}\n{{end}}{{end}}", + want: "{", + }, + { + name: "json objects", + template: `{{if .ToolCalls}}{{range .ToolCalls}}{"name": "{{ .Function.Name }}", "arguments": {{ .Function.Arguments }}}{{end}}{{end}}`, + want: "{", + }, + { + name: "json objects with whitespace", + template: "{{if .ToolCalls}}{{range .ToolCalls}}\n{\"name\": \"{{ .Function.Name }}\", \"arguments\": {{ .Function.Arguments }}}{{end}}{{end}}", + want: "{", + }, + { + name: "json objects with CRLF", + template: "{{if .ToolCalls}}{{range .ToolCalls}}\r\n{\"name\": \"{{ .Function.Name }}\", \"arguments\": {{ .Function.Arguments }}}{{end}}{{end}}", + want: "{", + }, + { + name: "json objects with whitespace before and after range", + template: "{{if .ToolCalls}}\n{{range .ToolCalls}}\n{\"name\": \"{{ .Function.Name }}\", \"arguments\": {{ .Function.Arguments }}}\r\n{{end}}\r\n{{end}}", + want: "{", + }, + { + name: "before and after range", + template: "{{if .ToolCalls}}<|tool▁calls▁begin|>{{range .ToolCalls}}<|tool▁call▁begin|>functionget_current_weather\n```json\n{\"location\": \"Tokyo\"}\n```<|tool▁call▁end|>\n{{end}}<|tool▁calls▁end|>{{end}}", + want: "<|tool▁calls▁begin|>", + }, + { + name: "after range", + template: "{{if .ToolCalls}}{{range .ToolCalls}}{\"name\": \"{{ .Function.Name }}\", \"arguments\": {{ .Function.Arguments }}}{{end}}{{end}}", + want: "", + }, + { + name: "after range with leading whitespace before range", + template: "{{if .ToolCalls}}\n{{range .ToolCalls}}{\"name\": \"{{ .Function.Name }}\", \"arguments\": {{ .Function.Arguments }}}{{end}}{{end}}", + want: "", + }, + { + name: "tool call in range with {", + template: `{{if .ToolCalls}}{{range .ToolCalls}}{"name": "{{ .Function.Name }}", "arguments": {{ .Function.Arguments }}}{{end}}{{end}}`, + want: "", + }, + { + name: "tool call with multiple text nodes", + template: "{{if .ToolCalls}}First text{{if .Something}}inner{{end}}Second text{{end}}", + want: "First text", + }, + { + name: "action tag", + template: "{{if .ToolCalls}}Action: ```json{{end}}", + want: "Action: ```json", + }, + { + name: "incomplete functools bracket", + template: "{{if .ToolCalls}}functools[{{end}}", + want: "functools[", + }, + { + name: "uppercase tool call with incomplete bracket", + template: "{{if .ToolCalls}}[TOOL_CALL] [{{end}}", + want: "[TOOL_CALL] [", + }, + { + name: "uppercase tool call with adjacent bracket", + template: "{{if .ToolCalls}}[TOOL_CALL][{{end}}", + want: "[TOOL_CALL][", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + tmpl, err := template.New("test").Parse(tc.template) + if err != nil && tc.template != "" { + t.Fatalf("failed to parse template: %v", err) + } + + got := parseTag(tmpl) + if got != tc.want { + t.Errorf("got text %q, want %q", got, tc.want) + } + }) + } +} diff --git a/tools/testdata/command-r-plus.gotmpl b/tools/testdata/command-r-plus.gotmpl deleted file mode 100644 index f30124e37..000000000 --- a/tools/testdata/command-r-plus.gotmpl +++ /dev/null @@ -1,67 +0,0 @@ -{{- if or .Tools .System }}<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|> -{{- if .Tools }}# Safety Preamble -The instructions in this section override those in the task description and style guide sections. Don't answer questions that are harmful or immoral. - -# System Preamble -## Basic Rules -You are a powerful conversational AI trained by Cohere to help people. You are augmented by a number of tools, and your job is to use and consume the output of these tools to best help the user. You will see a conversation history between yourself and a user, ending with an utterance from the user. You will then see a specific instruction instructing you what kind of response to generate. When you answer the user's requests, you cite your sources in your answers, according to those instructions. - -{{ if .System }}# User Preamble -{{ .System }} -{{- end }} - -## Available Tools -Here is a list of tools that you have available to you: -{{- range .Tools }} - -```python -def {{ .Function.Name }}( -{{- range $name, $property := .Function.Parameters.Properties }}{{ $name }}: {{ $property.Type }}, {{ end }}) -> List[Dict]: - """{{ .Function.Description }} - -{{- if .Function.Parameters.Properties }} - - Args: -{{- range $name, $property := .Function.Parameters.Properties }} - {{ $name }} ({{ $property.Type }}): {{ $property.Description }} -{{- end }} -{{- end }} - """ - pass -``` -{{- end }} -{{- else if .System }}{{ .System }} -{{- end }}<|END_OF_TURN_TOKEN|> -{{- end }} -{{- range .Messages }} -{{- if eq .Role "system" }} -{{- continue }} -{{- end }}<|START_OF_TURN_TOKEN|> -{{- if eq .Role "user" }}<|USER_TOKEN|>{{ .Content }} -{{- else if eq .Role "assistant" }}<|CHATBOT_TOKEN|> -{{- if .Content }}{{ .Content }} -{{- else if .ToolCalls }} -Action: ```json -[ -{{- range .ToolCalls }} - { - "tool_name": "{{ .Function.Name }}", - "parameters": {{ .Function.Arguments }} - } -{{- end }} -]``` -{{ continue }} -{{ end }} -{{- else if eq .Role "tool" }}<|SYSTEM_TOKEN|> -{{ .Content }} -{{- end }}<|END_OF_TURN_TOKEN|> -{{- end }} -{{- if .Tools }}<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>Write 'Action:' followed by a json-formatted list of actions that you want to perform in order to produce a good response to the user's last input. You can use any of the supplied tools any number of times, but you should aim to execute the minimum number of necessary actions for the input. You should use the `directly-answer` tool if calling the other tools is unnecessary. The list of actions you want to call should be formatted as a list of json objects, for example: -```json -[ - { - "tool_name": title of the tool in the specification, - "parameters": a dict of parameters to input into the tool as they are defined in the specs, or {} if it takes no parameters - } -]``` -{{- end }}<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|> \ No newline at end of file diff --git a/tools/testdata/command-r-plus.out b/tools/testdata/command-r-plus.out deleted file mode 100644 index 8193d40c9..000000000 --- a/tools/testdata/command-r-plus.out +++ /dev/null @@ -1,39 +0,0 @@ -<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|># Safety Preamble -The instructions in this section override those in the task description and style guide sections. Don't answer questions that are harmful or immoral. - -# System Preamble -## Basic Rules -You are a powerful conversational AI trained by Cohere to help people. You are augmented by a number of tools, and your job is to use and consume the output of these tools to best help the user. You will see a conversation history between yourself and a user, ending with an utterance from the user. You will then see a specific instruction instructing you what kind of response to generate. When you answer the user's requests, you cite your sources in your answers, according to those instructions. - -# User Preamble -You are a knowledgeable assistant. You can answer questions and perform tasks. - -## Available Tools -Here is a list of tools that you have available to you: - -```python -def get_current_weather(format: string, location: string, ) -> List[Dict]: - """Get the current weather - - Args: - format (string): The temperature unit to use. Infer this from the user's location. - location (string): The city and state, e.g. San Francisco, CA - """ - pass -```<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|USER_TOKEN|>What's the weather like today in Paris?<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|> -Action: ```json -[ - { - "tool_name": "get_current_weather", - "parameters": {"format":"celsius","location":"Paris, France"} - } -]``` -<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|> -22<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>The current temperature in Paris, France is 22 degrees Celsius.<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|USER_TOKEN|>What's the weather like today in San Francisco and Toronto?<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>Write 'Action:' followed by a json-formatted list of actions that you want to perform in order to produce a good response to the user's last input. You can use any of the supplied tools any number of times, but you should aim to execute the minimum number of necessary actions for the input. You should use the `directly-answer` tool if calling the other tools is unnecessary. The list of actions you want to call should be formatted as a list of json objects, for example: -```json -[ - { - "tool_name": title of the tool in the specification, - "parameters": a dict of parameters to input into the tool as they are defined in the specs, or {} if it takes no parameters - } -]```<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|> \ No newline at end of file diff --git a/tools/testdata/firefunction.gotmpl b/tools/testdata/firefunction.gotmpl deleted file mode 100644 index 312be205c..000000000 --- a/tools/testdata/firefunction.gotmpl +++ /dev/null @@ -1,31 +0,0 @@ -{{- if or .System .Tools }}<|start_header_id|>system<|end_header_id|> -{{- if .System }} -{{ .System }} -{{- end }} -In addition to plain text responses, you can chose to call one or more of the provided functions. - -Use the following rule to decide when to call a function: - * if the response can be generated from your internal knowledge (e.g., as in the case of queries like "What is the capital of Poland?"), do so - * if you need external information that can be obtained by calling one or more of the provided functions, generate a function calls - -If you decide to call functions: - * prefix function calls with functools marker (no closing marker required) - * all function calls should be generated in a single JSON list formatted as functools[{"name": [function name], "arguments": [function arguments as JSON]}, ...] - * follow the provided JSON schema. Do not hallucinate arguments or values. Do to blindly copy values from the provided samples - * respect the argument type formatting. E.g., if the type if number and format is float, write value 7 as 7.0 - * make sure you pick the right functions that match the user intent - -Available functions as JSON spec: -{{- if .Tools }} -{{ .Tools }} -{{- end }}<|eot_id|> -{{- end }} -{{- range .Messages }}<|start_header_id|> -{{- if or (eq .Role "user") (eq .Role "assistant") (eq .Role "tool") }}{{ .Role }} -{{- end }}<|end_header_id|> -{{- if .Content }}{{ .Content }} -{{- else if .ToolCalls }} functools[ -{{- range .ToolCalls }}{{ "{" }}"name": "{{ .Function.Name }}", "arguments": {{ .Function.Arguments }}{{ "}" }} -{{- end }}] -{{- end }}<|eot_id|> -{{- end }}<|start_header_id|>assistant<|end_header_id|> \ No newline at end of file diff --git a/tools/testdata/firefunction.out b/tools/testdata/firefunction.out deleted file mode 100644 index 144f5e428..000000000 --- a/tools/testdata/firefunction.out +++ /dev/null @@ -1,17 +0,0 @@ -<|start_header_id|>system<|end_header_id|> -You are a knowledgeable assistant. You can answer questions and perform tasks. -In addition to plain text responses, you can chose to call one or more of the provided functions. - -Use the following rule to decide when to call a function: - * if the response can be generated from your internal knowledge (e.g., as in the case of queries like "What is the capital of Poland?"), do so - * if you need external information that can be obtained by calling one or more of the provided functions, generate a function calls - -If you decide to call functions: - * prefix function calls with functools marker (no closing marker required) - * all function calls should be generated in a single JSON list formatted as functools[{"name": [function name], "arguments": [function arguments as JSON]}, ...] - * follow the provided JSON schema. Do not hallucinate arguments or values. Do to blindly copy values from the provided samples - * respect the argument type formatting. E.g., if the type if number and format is float, write value 7 as 7.0 - * make sure you pick the right functions that match the user intent - -Available functions as JSON spec: -[{"type":"function","function":{"name":"get_current_weather","description":"Get the current weather","parameters":{"type":"object","required":["location","format"],"properties":{"format":{"type":"string","description":"The temperature unit to use. Infer this from the user's location.","enum":["celsius","fahrenheit"]},"location":{"type":"string","description":"The city and state, e.g. San Francisco, CA"}}}}}]<|eot_id|><|start_header_id|><|end_header_id|>You are a knowledgeable assistant. You can answer questions and perform tasks.<|eot_id|><|start_header_id|>user<|end_header_id|>What's the weather like today in Paris?<|eot_id|><|start_header_id|>assistant<|end_header_id|> functools[{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Paris, France"}}]<|eot_id|><|start_header_id|>tool<|end_header_id|>22<|eot_id|><|start_header_id|>assistant<|end_header_id|>The current temperature in Paris, France is 22 degrees Celsius.<|eot_id|><|start_header_id|>user<|end_header_id|>What's the weather like today in San Francisco and Toronto?<|eot_id|><|start_header_id|>assistant<|end_header_id|> \ No newline at end of file diff --git a/tools/testdata/llama3-groq-tool-use.gotmpl b/tools/testdata/llama3-groq-tool-use.gotmpl deleted file mode 100644 index 45e9b462f..000000000 --- a/tools/testdata/llama3-groq-tool-use.gotmpl +++ /dev/null @@ -1,43 +0,0 @@ -{{- if .Messages }} -{{- if or .System .Tools }}<|start_header_id|>system<|end_header_id|> - -{{ .System }} -{{- if .Tools }} You are provided with function signatures within XML tags. You may call one or more functions to assist with the user query. Don't make assumptions about what values to plug into functions. For each function call return a json object with function name and arguments within XML tags as follows: - -{"name": ,"arguments": } - - -Here are the available tools: - -{{- range .Tools }} {{ .Function }} -{{- end }} -{{- end }} -{{- end }}<|eot_id|> -{{- range .Messages }} -{{- if ne .Role "system" }}<|start_header_id|>{{ .Role }}<|end_header_id|> - -{{ if eq .Role "user" }}{{ .Content }} -{{- else if eq .Role "assistant" }} -{{- if .Content }}{{ .Content }} -{{- else if .ToolCalls }} -{{ range .ToolCalls }}{"name": "{{ .Function.Name }}", "arguments": {{ .Function.Arguments }}} -{{- end }} - -{{- end }} -{{- else if eq .Role "tool" }} -{{ .Content }} - -{{- end }}<|eot_id|> -{{- end }} -{{- end }}<|start_header_id|>assistant<|end_header_id|> - -{{ else }} -{{ if .System }}<|start_header_id|>system<|end_header_id|> - -{{ .System }}<|eot_id|>{{ end }}{{ if .Prompt }}<|start_header_id|>user<|end_header_id|> - -{{ .Prompt }}<|eot_id|>{{ end }}<|start_header_id|>assistant<|end_header_id|> - -{{ end }}{{ .Response }} -{{- if .Response }}<|eot_id|> -{{- end }} \ No newline at end of file diff --git a/tools/testdata/llama3-groq-tool-use.out b/tools/testdata/llama3-groq-tool-use.out deleted file mode 100644 index 912ad11ca..000000000 --- a/tools/testdata/llama3-groq-tool-use.out +++ /dev/null @@ -1,24 +0,0 @@ -<|start_header_id|>system<|end_header_id|> - -You are a knowledgeable assistant. You can answer questions and perform tasks. You are provided with function signatures within XML tags. You may call one or more functions to assist with the user query. Don't make assumptions about what values to plug into functions. For each function call return a json object with function name and arguments within XML tags as follows: - -{"name": ,"arguments": } - - -Here are the available tools: - {"name":"get_current_weather","description":"Get the current weather","parameters":{"type":"object","required":["location","format"],"properties":{"format":{"type":"string","description":"The temperature unit to use. Infer this from the user's location.","enum":["celsius","fahrenheit"]},"location":{"type":"string","description":"The city and state, e.g. San Francisco, CA"}}}} <|eot_id|><|start_header_id|>user<|end_header_id|> - -What's the weather like today in Paris?<|eot_id|><|start_header_id|>assistant<|end_header_id|> - - -{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Paris, France"}} -<|eot_id|><|start_header_id|>tool<|end_header_id|> - - -22 -<|eot_id|><|start_header_id|>assistant<|end_header_id|> - -The current temperature in Paris, France is 22 degrees Celsius.<|eot_id|><|start_header_id|>user<|end_header_id|> - -What's the weather like today in San Francisco and Toronto?<|eot_id|><|start_header_id|>assistant<|end_header_id|> - diff --git a/tools/testdata/llama3.2.gotmpl b/tools/testdata/llama3.2.gotmpl deleted file mode 100644 index b132423e5..000000000 --- a/tools/testdata/llama3.2.gotmpl +++ /dev/null @@ -1,44 +0,0 @@ -<|start_header_id|>system<|end_header_id|> - -Cutting Knowledge Date: December 2023 - -{{ if .System }}{{ .System }} -{{- end }} -{{- if .Tools }}When you receive a tool call response, use the output to format an answer to the orginal user question. - -You are a helpful assistant with tool calling capabilities. -{{- end }}<|eot_id|> -{{- range $i, $_ := .Messages }} -{{- $last := eq (len (slice $.Messages $i)) 1 }} -{{- if eq .Role "user" }}<|start_header_id|>user<|end_header_id|> -{{- if and $.Tools $last }} - -Given the following functions, please respond with a JSON for a function call with its proper arguments that best answers the given prompt. - -Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}. Do not use variables. - -{{ range $.Tools }} -{{- . }} -{{ end }} -{{ .Content }}<|eot_id|> -{{- else }} - -{{ .Content }}<|eot_id|> -{{- end }}{{ if $last }}<|start_header_id|>assistant<|end_header_id|> - -{{ end }} -{{- else if eq .Role "assistant" }}<|start_header_id|>assistant<|end_header_id|> -{{- if .ToolCalls }} -{{ range .ToolCalls }} -{"name": "{{ .Function.Name }}", "parameters": {{ .Function.Arguments }}}{{ end }} -{{- else }} - -{{ .Content }} -{{- end }}{{ if not $last }}<|eot_id|>{{ end }} -{{- else if eq .Role "tool" }}<|start_header_id|>ipython<|end_header_id|> - -{{ .Content }}<|eot_id|>{{ if $last }}<|start_header_id|>assistant<|end_header_id|> - -{{ end }} -{{- end }} -{{- end }} \ No newline at end of file diff --git a/tools/testdata/llama3.2.out b/tools/testdata/llama3.2.out deleted file mode 100644 index a27c6eafc..000000000 --- a/tools/testdata/llama3.2.out +++ /dev/null @@ -1,24 +0,0 @@ -<|start_header_id|>system<|end_header_id|> - -Cutting Knowledge Date: December 2023 - -You are a knowledgeable assistant. You can answer questions and perform tasks.When you receive a tool call response, use the output to format an answer to the orginal user question. - -You are a helpful assistant with tool calling capabilities.<|eot_id|><|start_header_id|>user<|end_header_id|> - -What's the weather like today in Paris?<|eot_id|><|start_header_id|>assistant<|end_header_id|> - -{"name": "get_current_weather", "parameters": {"format":"celsius","location":"Paris, France"}}<|eot_id|><|start_header_id|>ipython<|end_header_id|> - -22<|eot_id|><|start_header_id|>assistant<|end_header_id|> - -The current temperature in Paris, France is 22 degrees Celsius.<|eot_id|><|start_header_id|>user<|end_header_id|> - -Given the following functions, please respond with a JSON for a function call with its proper arguments that best answers the given prompt. - -Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}. Do not use variables. - -{"type":"function","function":{"name":"get_current_weather","description":"Get the current weather","parameters":{"type":"object","required":["location","format"],"properties":{"format":{"type":"string","description":"The temperature unit to use. Infer this from the user's location.","enum":["celsius","fahrenheit"]},"location":{"type":"string","description":"The city and state, e.g. San Francisco, CA"}}}}} - -What's the weather like today in San Francisco and Toronto?<|eot_id|><|start_header_id|>assistant<|end_header_id|> - diff --git a/tools/testdata/messages.json b/tools/testdata/messages.json deleted file mode 100644 index 42de4711c..000000000 --- a/tools/testdata/messages.json +++ /dev/null @@ -1,39 +0,0 @@ -[ - { - "role": "system", - "content": "You are a knowledgeable assistant. You can answer questions and perform tasks." - }, - { - "role": "user", - "content": "What's the weather like today in Paris?" - }, - { - "role": "assistant", - "tool_calls": [ - { - "id": "89a1e453-0bce-4de3-a456-c54bed09c520", - "type": "function", - "function": { - "name": "get_current_weather", - "arguments": { - "location": "Paris, France", - "format": "celsius" - } - } - } - ] - }, - { - "role": "tool", - "tool_call_id": "89a1e453-0bce-4de3-a456-c54bed09c520", - "content": "22" - }, - { - "role": "assistant", - "content": "The current temperature in Paris, France is 22 degrees Celsius." - }, - { - "role": "user", - "content": "What's the weather like today in San Francisco and Toronto?" - } -] diff --git a/tools/testdata/mistral.gotmpl b/tools/testdata/mistral.gotmpl deleted file mode 100644 index b08d6c2c1..000000000 --- a/tools/testdata/mistral.gotmpl +++ /dev/null @@ -1,15 +0,0 @@ -{{- range $index, $_ := .Messages }} -{{- if eq .Role "user" }} -{{- if and (eq (len (slice $.Messages $index)) 1) $.Tools }}[AVAILABLE_TOOLS] {{ $.Tools }}[/AVAILABLE_TOOLS] -{{- end }}[INST] {{ if and (eq (len (slice $.Messages $index)) 1) $.System }}{{ $.System }} - -{{ end }}{{ .Content }}[/INST] -{{- else if eq .Role "assistant" }} -{{- if .Content }} {{ .Content }} -{{- else if .ToolCalls }}[TOOL_CALLS] [ -{{- range .ToolCalls }}{"name": "{{ .Function.Name }}", "arguments": {{ .Function.Arguments }}} -{{- end }}] -{{- end }} -{{- else if eq .Role "tool" }}[TOOL_RESULTS] {"content": {{ .Content }}}[/TOOL_RESULTS] -{{- end }} -{{- end }} \ No newline at end of file diff --git a/tools/testdata/mistral.out b/tools/testdata/mistral.out deleted file mode 100644 index 6956e3920..000000000 --- a/tools/testdata/mistral.out +++ /dev/null @@ -1,3 +0,0 @@ -[INST] What's the weather like today in Paris?[/INST][TOOL_CALLS] [{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Paris, France"}}][TOOL_RESULTS] {"content": 22}[/TOOL_RESULTS] The current temperature in Paris, France is 22 degrees Celsius.[AVAILABLE_TOOLS] [{"type":"function","function":{"name":"get_current_weather","description":"Get the current weather","parameters":{"type":"object","required":["location","format"],"properties":{"format":{"type":"string","description":"The temperature unit to use. Infer this from the user's location.","enum":["celsius","fahrenheit"]},"location":{"type":"string","description":"The city and state, e.g. San Francisco, CA"}}}}}][/AVAILABLE_TOOLS][INST] You are a knowledgeable assistant. You can answer questions and perform tasks. - -What's the weather like today in San Francisco and Toronto?[/INST] \ No newline at end of file diff --git a/tools/testdata/nemotron.gotmpl b/tools/testdata/nemotron.gotmpl deleted file mode 100644 index 1b6b89ecb..000000000 --- a/tools/testdata/nemotron.gotmpl +++ /dev/null @@ -1,33 +0,0 @@ -{{- if (or .Tools .System) }}System -{{ if .System }}{{ .System }} - - -{{ end }} -{{- if .Tools }} -{{- range .Tools }} {{ . }} {{ end }} - - -{{ end }} -{{- end }} -{{- range $i, $m := .Messages }} -{{- $last := eq (len (slice $.Messages $i)) 1 -}} -{{- if eq .Role "user" }}User -{{ .Content }} -{{- if $last }} -Assistant -{{- end }} -{{ else if eq .Role "tool" }}Tool -{{ .Content }} -{{- if $last }} -Assistant -{{- end }} -{{ else if eq .Role "assistant" }}Assistant -{{- if .ToolCalls }} -{{ range .ToolCalls }} {"name": "{{ .Function.Name }}", "arguments": {{ .Function.Arguments }}} {{ end }} -{{ else }} -{{ .Content }} -{{- if not $last }} -{{ end }} -{{- end }} -{{- end }} -{{- end }} \ No newline at end of file diff --git a/tools/testdata/nemotron.out b/tools/testdata/nemotron.out deleted file mode 100644 index 486889ca1..000000000 --- a/tools/testdata/nemotron.out +++ /dev/null @@ -1,18 +0,0 @@ -System -You are a knowledgeable assistant. You can answer questions and perform tasks. - - - {"type":"function","function":{"name":"get_current_weather","description":"Get the current weather","parameters":{"type":"object","required":["location","format"],"properties":{"format":{"type":"string","description":"The temperature unit to use. Infer this from the user's location.","enum":["celsius","fahrenheit"]},"location":{"type":"string","description":"The city and state, e.g. San Francisco, CA"}}}}} - - -User -What's the weather like today in Paris? -Assistant - {"name": "get_current_weather", "arguments": {"format":"celsius","location":"Paris, France"}} -Tool -22 -Assistant -The current temperature in Paris, France is 22 degrees Celsius. -User -What's the weather like today in San Francisco and Toronto? -Assistant diff --git a/tools/testdata/qwen2.5.gotmpl b/tools/testdata/qwen2.5.gotmpl deleted file mode 100644 index cbd7302c4..000000000 --- a/tools/testdata/qwen2.5.gotmpl +++ /dev/null @@ -1,51 +0,0 @@ -{{- if .Suffix }}<|fim_prefix|>{{ .Prompt }}<|fim_suffix|>{{ .Suffix }}<|fim_middle|> -{{- else if .Messages }} -{{- if or .System .Tools }}<|im_start|>system -{{- if .System }} -{{ .System }} -{{- end }} -{{- if .Tools }} - -# Tools - -You may call one or more functions to assist with the user query. - -You are provided with function signatures within XML tags: - -{{- range .Tools }} -{"type": "function", "function": {{ .Function }}} -{{- end }} - - -For each function call, return a json object with function name and arguments within XML tags: - -{"name": , "arguments": } - -{{- end }}<|im_end|> -{{ end }} -{{- range $i, $_ := .Messages }} -{{- $last := eq (len (slice $.Messages $i)) 1 -}} -{{- if eq .Role "user" }}<|im_start|>user -{{ .Content }}<|im_end|> -{{ else if eq .Role "assistant" }}<|im_start|>assistant -{{ if .Content }}{{ .Content }} -{{- else if .ToolCalls }} -{{ range .ToolCalls }}{"name": "{{ .Function.Name }}", "arguments": {{ .Function.Arguments }}} -{{ end }} -{{- end }}{{ if not $last }}<|im_end|> -{{ end }} -{{- else if eq .Role "tool" }}<|im_start|>user - -{{ .Content }} -<|im_end|> -{{ end }} -{{- if and (ne .Role "assistant") $last }}<|im_start|>assistant -{{ end }} -{{- end }} -{{- else }} -{{- if .System }}<|im_start|>system -{{ .System }}<|im_end|> -{{ end }}{{ if .Prompt }}<|im_start|>user -{{ .Prompt }}<|im_end|> -{{ end }}<|im_start|>assistant -{{ end }}{{ .Response }}{{ if .Response }}<|im_end|>{{ end }} \ No newline at end of file diff --git a/tools/testdata/qwen2.5.out b/tools/testdata/qwen2.5.out deleted file mode 100644 index 76bfbfa98..000000000 --- a/tools/testdata/qwen2.5.out +++ /dev/null @@ -1,31 +0,0 @@ -<|im_start|>system -You are a knowledgeable assistant. You can answer questions and perform tasks. - -# Tools - -You may call one or more functions to assist with the user query. - -You are provided with function signatures within XML tags: - -{"type": "function", "function": {"name":"get_current_weather","description":"Get the current weather","parameters":{"type":"object","required":["location","format"],"properties":{"format":{"type":"string","description":"The temperature unit to use. Infer this from the user's location.","enum":["celsius","fahrenheit"]},"location":{"type":"string","description":"The city and state, e.g. San Francisco, CA"}}}}} - - -For each function call, return a json object with function name and arguments within XML tags: - -{"name": , "arguments": } -<|im_end|> -<|im_start|>user -What's the weather like today in Paris?<|im_end|> -<|im_start|>assistant - -{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Paris, France"}} -<|im_end|> -<|im_start|>user - -22 -<|im_end|> -<|im_start|>assistant -The current temperature in Paris, France is 22 degrees Celsius.<|im_end|> -<|im_start|>user -What's the weather like today in San Francisco and Toronto?<|im_end|> -<|im_start|>assistant diff --git a/tools/testdata/qwen3.gotmpl b/tools/testdata/qwen3.gotmpl deleted file mode 100644 index 26f6656fa..000000000 --- a/tools/testdata/qwen3.gotmpl +++ /dev/null @@ -1,50 +0,0 @@ -{{- if .Messages }} -{{- if or .System .Tools }}<|im_start|>system -{{- if .System }} -{{ .System }} -{{- end }} -{{- if .Tools }} - -# Tools - -You may call one or more functions to assist with the user query. - -You are provided with function signatures within XML tags: - -{{- range .Tools }} -{"type": "function", "function": {{ .Function }}} -{{- end }} - - -For each function call, return a json object with function name and arguments within XML tags: - -{"name": , "arguments": } - -{{- end }}<|im_end|> -{{ end }} -{{- range $i, $_ := .Messages }} -{{- $last := eq (len (slice $.Messages $i)) 1 -}} -{{- if eq .Role "user" }}<|im_start|>user -{{ .Content }}<|im_end|> -{{ else if eq .Role "assistant" }}<|im_start|>assistant -{{ if .Content }}{{ .Content }} -{{- else if .ToolCalls }} -{{ range .ToolCalls }}{"name": "{{ .Function.Name }}", "arguments": {{ .Function.Arguments }}} -{{ end }} -{{- end }}{{ if not $last }}<|im_end|> -{{ end }} -{{- else if eq .Role "tool" }}<|im_start|>user - -{{ .Content }} -<|im_end|> -{{ end }} -{{- if and (ne .Role "assistant") $last }}<|im_start|>assistant -{{ end }} -{{- end }} -{{- else }} -{{- if .System }}<|im_start|>system -{{ .System }}<|im_end|> -{{ end }}{{ if .Prompt }}<|im_start|>user -{{ .Prompt }}<|im_end|> -{{ end }}<|im_start|>assistant -{{ end }}{{ .Response }}{{ if .Response }}<|im_end|>{{ end }} \ No newline at end of file diff --git a/tools/testdata/qwen3.out b/tools/testdata/qwen3.out deleted file mode 100644 index 76bfbfa98..000000000 --- a/tools/testdata/qwen3.out +++ /dev/null @@ -1,31 +0,0 @@ -<|im_start|>system -You are a knowledgeable assistant. You can answer questions and perform tasks. - -# Tools - -You may call one or more functions to assist with the user query. - -You are provided with function signatures within XML tags: - -{"type": "function", "function": {"name":"get_current_weather","description":"Get the current weather","parameters":{"type":"object","required":["location","format"],"properties":{"format":{"type":"string","description":"The temperature unit to use. Infer this from the user's location.","enum":["celsius","fahrenheit"]},"location":{"type":"string","description":"The city and state, e.g. San Francisco, CA"}}}}} - - -For each function call, return a json object with function name and arguments within XML tags: - -{"name": , "arguments": } -<|im_end|> -<|im_start|>user -What's the weather like today in Paris?<|im_end|> -<|im_start|>assistant - -{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Paris, France"}} -<|im_end|> -<|im_start|>user - -22 -<|im_end|> -<|im_start|>assistant -The current temperature in Paris, France is 22 degrees Celsius.<|im_end|> -<|im_start|>user -What's the weather like today in San Francisco and Toronto?<|im_end|> -<|im_start|>assistant diff --git a/tools/testdata/tools.json b/tools/testdata/tools.json deleted file mode 100644 index edde4ae0b..000000000 --- a/tools/testdata/tools.json +++ /dev/null @@ -1,30 +0,0 @@ -[ - { - "type": "function", - "function": { - "name": "get_current_weather", - "description": "Get the current weather", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA" - }, - "format": { - "type": "string", - "enum": [ - "celsius", - "fahrenheit" - ], - "description": "The temperature unit to use. Infer this from the user's location." - } - }, - "required": [ - "location", - "format" - ] - } - } - } -] diff --git a/tools/testdata/xlam.gotmpl b/tools/testdata/xlam.gotmpl deleted file mode 100644 index 51513d698..000000000 --- a/tools/testdata/xlam.gotmpl +++ /dev/null @@ -1,45 +0,0 @@ -{{- if .System }}{{ .System }} -{{ end }} -{{- range $i, $_ := .Messages }} -{{- if eq .Role "user" }}### Instruction: -{{- if and $.Tools (le (len (slice $.Messages $i)) 2) }} -[BEGIN OF TASK INSTRUCTION] -You are an expert in composing functions. You are given a question and a set of possible functions. -Based on the question, you will need to make one or more function/tool calls to achieve the purpose. -If none of the functions can be used, point it out and refuse to answer. -If the given question lacks the parameters required by the function, also point it out. -[END OF TASK INSTRUCTION] - -[BEGIN OF AVAILABLE TOOLS] -{{ $.Tools }} -[END OF AVAILABLE TOOLS] - -[BEGIN OF FORMAT INSTRUCTION] -The output MUST strictly adhere to the following JSON format, and NO other text MUST be included. -The example format is as follows. Please make sure the parameter type is correct. If no function call is needed, please make tool_calls an empty list '[]'. -``` -{ - "tool_calls": [ - {"name": "func_name1", "arguments": {"argument1": "value1", "argument2": "value2"}}, - ... (more tool calls as required) - ] -} -``` -[END OF FORMAT INSTRUCTION] - -[BEGIN OF QUERY] -{{ .Content }} -[END OF QUERY] - - -{{ else }} -{{ .Content }} -{{ end }} -{{- else if .ToolCalls }}### Response: -{"tool_calls": [{{ range .ToolCalls }}{"name": "{{ .Function.Name }}", "arguments": {{ .Function.Arguments }}}{{ end }}]} -<|EOT|> -{{ else if eq .Role "assistant" }}### Response: -{{ .Content }} -<|EOT|> -{{ end }} -{{- end }}### Response: \ No newline at end of file diff --git a/tools/testdata/xlam.out b/tools/testdata/xlam.out deleted file mode 100644 index 5d8065327..000000000 --- a/tools/testdata/xlam.out +++ /dev/null @@ -1,40 +0,0 @@ -You are a knowledgeable assistant. You can answer questions and perform tasks. -### Instruction: -What's the weather like today in Paris? -### Response: -{"tool_calls": [{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Paris, France"}}]} -<|EOT|> -### Response: -The current temperature in Paris, France is 22 degrees Celsius. -<|EOT|> -### Instruction: -[BEGIN OF TASK INSTRUCTION] -You are an expert in composing functions. You are given a question and a set of possible functions. -Based on the question, you will need to make one or more function/tool calls to achieve the purpose. -If none of the functions can be used, point it out and refuse to answer. -If the given question lacks the parameters required by the function, also point it out. -[END OF TASK INSTRUCTION] - -[BEGIN OF AVAILABLE TOOLS] -[{"type":"function","function":{"name":"get_current_weather","description":"Get the current weather","parameters":{"type":"object","required":["location","format"],"properties":{"format":{"type":"string","description":"The temperature unit to use. Infer this from the user's location.","enum":["celsius","fahrenheit"]},"location":{"type":"string","description":"The city and state, e.g. San Francisco, CA"}}}}}] -[END OF AVAILABLE TOOLS] - -[BEGIN OF FORMAT INSTRUCTION] -The output MUST strictly adhere to the following JSON format, and NO other text MUST be included. -The example format is as follows. Please make sure the parameter type is correct. If no function call is needed, please make tool_calls an empty list '[]'. -``` -{ - "tool_calls": [ - {"name": "func_name1", "arguments": {"argument1": "value1", "argument2": "value2"}}, - ... (more tool calls as required) - ] -} -``` -[END OF FORMAT INSTRUCTION] - -[BEGIN OF QUERY] -What's the weather like today in San Francisco and Toronto? -[END OF QUERY] - - -### Response: \ No newline at end of file diff --git a/tools/tools.go b/tools/tools.go index 914a5eaf0..efeaeee0c 100644 --- a/tools/tools.go +++ b/tools/tools.go @@ -1,253 +1,287 @@ package tools import ( + "bytes" "encoding/json" - "errors" - "log/slog" "strings" - gotmpl "text/template" + "text/template" "github.com/ollama/ollama/api" - "github.com/ollama/ollama/template" ) -var ( - errInvalidToolCall = errors.New("invalid tool call format") - errAccumulateMore = errors.New("need to accumulate more content") +type toolsState int + +const ( + toolsState_LookingForTag toolsState = iota + toolsState_ToolCalling + toolsState_Done ) type Parser struct { - greedyParseJSON bool - prefix string - prefixFound bool - tmpl gotmpl.Template - sb strings.Builder - index int - name string - arguments string + tag string + names []string + properties []string + + state toolsState + buffer []byte + n int } -// parseJSONToolCalls attempts to parse a JSON string into a slice of ToolCalls. -// -// Parameters: -// - s: The string to parse -// - name: The field name from template that identifies the tool call name -// - arguments: The field name from template that identifies the tool call arguments -// -// Returns: -// - []api.ToolCall: The parsed tool calls if successful -// - error: ErrAccumulateMore if braces unbalanced, ErrInvalidToolCall if invalid, or nil if successful -func parseJSONToolCalls(s string, name, arguments string, prefix string) ([]api.ToolCall, error) { - // Check for balanced braces before attempting to parse - braceCount := 0 - squareCount := 0 - startIndex := -1 - var rawToolCalls []string - s = strings.TrimSpace(s) - - // Only track these if we don't have a prefix as it will be cut off from the prefix. Also track in the parseLeadingJSON case. - trackSquareBrackets := prefix == "" || !strings.HasSuffix(prefix, "[") || strings.HasPrefix(s, "[") - for i, c := range s { - switch c { - case '{': - braceCount++ - if startIndex == -1 { - startIndex = i - } - case '}': - braceCount-- - if braceCount == 0 { - rawToolCalls = append(rawToolCalls, s[startIndex:i+1]) - startIndex = -1 - } - case '[': - if trackSquareBrackets { - squareCount++ - } - case ']': - if trackSquareBrackets { - squareCount-- - } - } - - // Negative means we have an extra closing brace/bracket - if braceCount < 0 || squareCount < 0 { - return nil, errInvalidToolCall - } - } - - // If braces/brackets aren't balanced, need more input - if braceCount > 0 || squareCount > 0 { - return nil, errAccumulateMore - } - - t := strings.TrimSpace(s) - if len(t) == 0 { - return nil, errAccumulateMore - } - // If the input is a single square bracket, it's not a valid tool call - if t[0] == '[' && len(t) == 1 { - return nil, errAccumulateMore - } - - // Attempt full unmarshal of the JSON - var toolCalls []api.ToolCall - for _, rawToolCall := range rawToolCalls { - var resp map[string]any - if err := json.Unmarshal([]byte(rawToolCall), &resp); err != nil { - continue - } - - // Collect nested objects that could contain tool calls - objs := collect(resp) - if len(objs) == 0 { - continue - } - - // Extract tool calls from objects - for _, kv := range objs { - n, nok := kv[name].(string) - a, aok := kv[arguments].(map[string]any) - if nok && aok { - toolCalls = append(toolCalls, api.ToolCall{ - Function: api.ToolCallFunction{ - Name: n, - Arguments: a, - }, - }) - } else { - slog.Debug("No valid tool call found in object.", "object", kv) - } - } - } - - // Valid JSON, no tool calls found - if len(toolCalls) == 0 { - slog.Debug("No valid tool calls found in any raw tool calls.", "rawToolCalls", rawToolCalls) - return nil, errInvalidToolCall - } - - return toolCalls, nil +// NewParser creates a new tool call parser from a model's chat +// template and a list of provided tools. +func NewParser(tmpl *template.Template, tools []api.Tool) *Parser { + return NewParserWithTag(tools, parseTag(tmpl)) } -// checkPrefix processes a string to find and handle a prefix pattern. -// -// Returns: -// - The processed string with prefix removed if found -// - error: ErrAccumulateMore if prefix is incomplete, or nil if successful -func (p *Parser) checkPrefix(s string) (string, error) { - if s == "" || p.prefix == "" { - return s, nil +func NewParserWithTag(tools []api.Tool, tag string) *Parser { + var p Parser + for _, t := range tools { + p.names = append(p.names, t.Function.Name) + for r := range t.Function.Parameters.Properties { + p.properties = append(p.properties, r) + } } - - // Check for prefix at start of string - if cut, hasPrefix := strings.CutPrefix(s, p.prefix); hasPrefix { - // Found prefix at start - accumulate for potential tool - p.prefixFound = true - return cut, nil - } - - // Check if prefix overlaps end of string - if idx := suffixOverlap(s, p.prefix); idx != -1 { - // Return everything except overlapping portion - p.sb.Reset() - p.sb.WriteString(s[idx:]) - return s[:idx], errAccumulateMore - } - - // Check if prefix appears in middle of string - if idx := strings.Index(s, p.prefix); idx != -1 { - // Save remainder starting at prefix for next pass - p.sb.Reset() - p.sb.WriteString(strings.TrimSpace(s[idx:])) - // Return everything before prefix - return s[:idx], errAccumulateMore - } - - // No partial prefix found - return s, nil + p.tag = tag + return &p } -// Add processes a string input to parse tool calls and content. -// It handles prefix detection and JSON parsing to extract tool calls. -// -// Returns: -// - tools: Any parsed tool calls -// - content: Non-tool call content -func (p *Parser) Add(s string) (tools []api.ToolCall, content string) { - p.sb.WriteString(s) - s = p.sb.String() - - // Check for prefix pattern in input - s, err := p.checkPrefix(s) - if err != nil { - // Need more input to complete prefix +// Add processes a string input to parse tool calls and content that +// should be sent back to the user. +func (p *Parser) Add(s string) (calls []api.ToolCall, content string) { + if p.state == toolsState_Done { return nil, s } - // Exit if prefix exists in template, greedy parsing is off, and prefix not found - if !p.greedyParseJSON && !p.prefixFound { - p.sb.Reset() - return nil, s + p.buffer = append(p.buffer, s...) + + if p.state == toolsState_LookingForTag { + i, found := p.findTag() + if i == -1 { + content = string(p.buffer) + p.buffer = []byte{} + } else { + content = string(p.buffer[:i]) + p.buffer = p.buffer[i:] + } + + // for models where { or [ are used as tool calling + // tags, we only support parsing tools if the first non- + // whitespace character is { or [ + if p.tag == "{" || p.tag == "[" { + if strings.TrimSpace(content) != "" { + p.state = toolsState_Done + return nil, content + string(p.buffer) + } + } + + if !found { + return nil, content + } + + p.state = toolsState_ToolCalling } - toolCalls, err := parseJSONToolCalls(s, p.name, p.arguments, p.prefix) - if err != nil { - if errors.Is(err, errAccumulateMore) { - return nil, "" + for { + call := p.parseToolCall() + if call == nil { + break } - p.sb.Reset() - // Only do greedy JSON parsing if there is no prefix from template - if p.prefix != "" { - p.greedyParseJSON = false - } - if p.index != 0 && p.prefix == "" { - return nil, "" - } - if p.prefixFound { - // Drop tokens since prefix was found - return nil, "" - } - return nil, s + + calls = append(calls, *call) } - for _, tc := range toolCalls { - tc.Function.Index = p.index - p.index++ + if p.done() { + p.state = toolsState_Done + content = string(p.buffer) + p.buffer = []byte{} } - p.sb.Reset() - return toolCalls, "" + return calls, content } -// NewParser creates a new tool call parser from a template. It extracts the tool call format, -// prefix, and field names from the template to use for parsing tool calls from model output. -// -// Returns an error if the template does not contain valid tool call formatting. -func NewParser(templateToProcess *gotmpl.Template) (*Parser, error) { - parsed, err := template.Parse(templateToProcess.Root.String()) - if err != nil { - return nil, err +// findTag searches the buffer to find and handle a tool calling tag +// returning true if the tag was found and false otherwise, and +// a string content signaling any content that should be sent back to the user +func (p *Parser) findTag() (int, bool) { + // First check for complete substring anywhere in s + if i := bytes.Index(p.buffer, []byte(p.tag)); i > -1 { + return i, true } - tt, err := toolTemplate(parsed) - if err != nil { - return nil, err + // Then check for partial suffix overlap + max := min(len(p.buffer), len(p.tag)) + for i := max; i > 0; i-- { + if bytes.HasSuffix(p.buffer, []byte(p.tag[:i])) { + return len(p.buffer) - i, false + } } - - tp := toolPrefix(templateToProcess) - - name, arguments, err := extractToolArgs(tt) - if err != nil { - return nil, err - } - - return &Parser{ - tmpl: *tt, - sb: strings.Builder{}, - prefix: tp, - greedyParseJSON: true, - name: name, - arguments: arguments, - }, nil + return -1, false +} + +// parseToolCall finds the next complete tool call in the buffer +// incrementing n and advancing the buffer. +func (p *Parser) parseToolCall() *api.ToolCall { + var name string + var args map[string]any + var end int = len(p.buffer) + + // find tool name + var i int + for _, n := range p.names { + if i = bytes.Index(p.buffer, []byte(n)); i != -1 { + if i+len(n) < end { + name = n + end = i + len(n) + } + } + } + + if name == "" { + return nil + } + + if args, i = p.findArguments(); args == nil { + return nil + } + + if i > end { + end = i + } + + tc := &api.ToolCall{ + Function: api.ToolCallFunction{ + Name: name, + Arguments: args, + Index: p.n, + }, + } + + p.n++ + p.buffer = p.buffer[end:] + return tc +} + +// findArguments returns the first object that appears to be +// arguments and the position where the arguments end, returning nil and 0 if +// an invalid JSON object or non-arguments object is found first +func (p *Parser) findArguments() (map[string]any, int) { + if len(p.buffer) == 0 { + return nil, 0 + } + + var braces int + var start int = -1 + var end int + var object []byte + + // find any outer json object + for i, c := range p.buffer { + if c == '{' { + braces++ + if start == -1 { + start = i + } + } + + if c == '}' { + braces-- + if braces == 0 && start != -1 { + end = i + 1 + object = p.buffer[start:end] + break + } + } + } + + if braces > 0 { + return nil, 0 + } + + var data map[string]any + + // not valid json + if err := json.Unmarshal(object, &data); err != nil { + return nil, 0 + } + + var find func(obj any) map[string]any + find = func(obj any) map[string]any { + switch v := obj.(type) { + case map[string]any: + // check if the object keys are valid tool properties + // TODO (jmorganca): check only sets of properties that + // go together instead of the entire set + for _, prop := range p.properties { + if _, exists := v[prop]; exists { + return v + } + } + + for _, value := range v { + if result := find(value); result != nil { + return result + } + } + case []any: + for _, item := range v { + if result := find(item); result != nil { + return result + } + } + } + + return nil + } + + result := find(data) + if result != nil { + return result, end + } + + return nil, 0 +} + +// done checks if the parser is done parsing by looking +// for closing tag. currently only } and ] are supported +// for closing tags as {} or [] pairs may not always +// represent tool calls and we need to send the content back +func (p *Parser) done() bool { + var open, close rune + switch p.tag { + case "{": + open, close = '{', '}' + case "[": + open, close = '[', ']' + default: + return false + } + + var count int + for _, c := range p.buffer { + if c == byte(open) { + count++ + } else if c == byte(close) { + count-- + if count == 0 { + return true + } + } + } + + return false +} + +// Content returns any remaining content that +// should be sent to the user. This should be the empty string +// string unless the tag is { or [ and a tool call was not found +func (p *Parser) Content() string { + if p.n > 0 { + return "" + } + + if p.tag == "{" || p.tag == "[" { + return string(p.buffer) + } + + return "" } diff --git a/tools/tools_test.go b/tools/tools_test.go index 5fee8f57d..678641684 100644 --- a/tools/tools_test.go +++ b/tools/tools_test.go @@ -1,673 +1,805 @@ package tools import ( - "bytes" - "encoding/json" - "fmt" - "os" - "path/filepath" - "strings" "testing" + "text/template" "github.com/google/go-cmp/cmp" - "github.com/ollama/ollama/api" - "github.com/ollama/ollama/template" ) -func readFile(t *testing.T, base, name string) *bytes.Buffer { - t.Helper() - - bts, err := os.ReadFile(filepath.Join(base, name)) +func TestParser(t *testing.T) { + qwen, err := template.New("qwen").Parse(`{{if .ToolCalls}}{{range .ToolCalls}}{"name": "{{.Function.Name}}", "arguments": {{.Function.Arguments}}}{{end}}{{end}}`) if err != nil { - t.Fatal(err) + t.Fatalf("Failed to parse template: %v", err) } - return bytes.NewBuffer(bts) -} + deepseek, err := template.New("deepseek").Parse("{{if .ToolCalls}}<|tool▁calls▁begin|>{{range .ToolCalls}}<|tool▁call▁begin|>function<|tool▁sep|>get_current_weather\n```json\n{\"location\": \"Tokyo\"}\n```<|tool▁call▁end|>{{end}}<|tool▁calls▁end|><|end▁of▁sentence|>{{end}}") + if err != nil { + t.Fatalf("Failed to parse template: %v", err) + } + + json, err := template.New("json").Parse(`{{if .ToolCalls}}{{range .ToolCalls}}{"name": "{{.Function.Name}}", "arguments": {{.Function.Arguments}}}{{end}}{{end}}`) + if err != nil { + t.Fatalf("Failed to parse template: %v", err) + } + + mistral, err := template.New("mistral").Parse(`{{if .ToolCalls}}[TOOL_CALLS] [{{range .ToolCalls}}{"name": "{{.Function.Name}}", "arguments": {{.Function.Arguments}}}{{end}}][/TOOL_CALLS]{{end}}`) + if err != nil { + t.Fatalf("Failed to parse template: %v", err) + } + + list, err := template.New("list").Parse(`{{if .ToolCalls}}[{{range .ToolCalls}}{"name": "{{.Function.Name}}", "arguments": {{.Function.Arguments}}}{{end}}]{{end}}`) + if err != nil { + t.Fatalf("Failed to parse template: %v", err) + } + + tools := []api.Tool{ + { + Type: "function", + Function: api.ToolFunction{ + Name: "get_temperature", + Description: "Retrieve the temperature for a given location", + Parameters: struct { + Type string `json:"type"` + Defs any `json:"$defs,omitempty"` + Items any `json:"items,omitempty"` + Required []string `json:"required"` + Properties map[string]struct { + Type api.PropertyType `json:"type"` + Items any `json:"items,omitempty"` + Description string `json:"description"` + Enum []any `json:"enum,omitempty"` + } `json:"properties"` + }{ + Type: "object", + Properties: map[string]struct { + Type api.PropertyType `json:"type"` + Items any `json:"items,omitempty"` + Description string `json:"description"` + Enum []any `json:"enum,omitempty"` + }{ + "format": { + Type: api.PropertyType{"string"}, + Description: "The format to return the temperature in", + Enum: []any{"fahrenheit", "celsius"}, + }, + "city": { + Type: api.PropertyType{"string"}, + Description: "The city to get the temperature for", + }, + }, + }, + }, + }, + { + Type: "function", + Function: api.ToolFunction{ + Name: "get_conditions", + Description: "Retrieve the current weather conditions for a given location", + Parameters: struct { + Type string `json:"type"` + Defs any `json:"$defs,omitempty"` + Items any `json:"items,omitempty"` + Required []string `json:"required"` + Properties map[string]struct { + Type api.PropertyType `json:"type"` + Items any `json:"items,omitempty"` + Description string `json:"description"` + Enum []any `json:"enum,omitempty"` + } `json:"properties"` + }{ + Type: "object", + Properties: map[string]struct { + Type api.PropertyType `json:"type"` + Items any `json:"items,omitempty"` + Description string `json:"description"` + Enum []any `json:"enum,omitempty"` + }{ + "location": { + Type: api.PropertyType{"string"}, + Description: "The location to get the weather conditions for", + }, + }, + }, + }, + }, + } -func TestParseJSONToolCalls(t *testing.T) { tests := []struct { - name string - input string - nameField string - argsField string - wantToolCalls []api.ToolCall - wantErr error - prefix string + name string + inputs []string + tmpl *template.Template + content string + calls []api.ToolCall }{ { - name: "valid single tool call", - input: `{"name": "test_tool", "arguments": {"arg1": "value1"}}`, - nameField: "name", - argsField: "arguments", - wantToolCalls: []api.ToolCall{ + name: "no tool calls - just text", + inputs: []string{"Hello, how can I help you today?"}, + content: "Hello, how can I help you today?", + tmpl: qwen, + calls: nil, + }, + { + name: "empty input", + inputs: []string{""}, + content: "", + tmpl: qwen, + calls: nil, + }, + { + name: "tool call", + inputs: []string{`{"name": "get_conditions", "arguments": {"location": "San Francisco"}}`}, + content: "", + tmpl: qwen, + calls: []api.ToolCall{ { Function: api.ToolCallFunction{ - Name: "test_tool", - Arguments: map[string]any{ - "arg1": "value1", + Index: 0, + Name: "get_conditions", + Arguments: api.ToolCallFunctionArguments{ + "location": "San Francisco", }, }, }, }, - wantErr: nil, - prefix: "", }, { - name: "incomplete JSON", - input: `{"name": "test_tool", "arguments": {"arg1": `, - nameField: "name", - argsField: "arguments", - wantToolCalls: nil, - wantErr: errAccumulateMore, - prefix: "", - }, - { - name: "invalid JSON", - input: `not json at all`, - nameField: "name", - argsField: "arguments", - wantToolCalls: nil, - wantErr: errInvalidToolCall, - prefix: "", - }, - { - name: "missing required fields", - input: `{"other": "field"}`, - nameField: "name", - argsField: "arguments", - wantToolCalls: nil, - wantErr: errInvalidToolCall, - prefix: "", - }, - { - name: "multiple tool calls in array", - input: `[ - {"name": "tool1", "arguments": {"arg1": 1}}, - {"name": "tool2", "arguments": {"arg2": "value"}} - ]`, - nameField: "name", - argsField: "arguments", - wantToolCalls: []api.ToolCall{ + name: "text before tool call", + inputs: []string{`Let me check the weather. {"name": "get_temperature", "arguments": {"city": "New York"}}`}, + content: "Let me check the weather. ", + tmpl: qwen, + calls: []api.ToolCall{ { Function: api.ToolCallFunction{ - Name: "tool1", - Arguments: map[string]any{ - "arg1": float64(1), - }, - }, - }, - { - Function: api.ToolCallFunction{ - Name: "tool2", - Arguments: map[string]any{ - "arg2": "value", + Index: 0, + Name: "get_temperature", + Arguments: api.ToolCallFunctionArguments{ + "city": "New York", }, }, }, }, - wantErr: nil, - prefix: "", }, { - name: "multiple tool calls without array", - input: ` - {"name": "tool1", "arguments": {"arg1": 1}}, - {"name": "tool2", "arguments": {"arg2": "value"}} - `, - nameField: "name", - argsField: "arguments", - wantToolCalls: []api.ToolCall{ + name: "two tool calls in a list", + inputs: []string{`[TOOL_CALLS] [{"name": "get_temperature", "arguments": {"city": "London", "format": "fahrenheit"}}, {"name": "get_conditions", "arguments": {"location": "Tokyo"}}][/TOOL_CALLS]`}, + content: "", + tmpl: mistral, + calls: []api.ToolCall{ { Function: api.ToolCallFunction{ - Name: "tool1", - Arguments: map[string]any{ - "arg1": float64(1), + Index: 0, + Name: "get_temperature", + Arguments: api.ToolCallFunctionArguments{ + "city": "London", + "format": "fahrenheit", }, }, }, { Function: api.ToolCallFunction{ - Name: "tool2", - Arguments: map[string]any{ - "arg2": "value", + Index: 1, + Name: "get_conditions", + Arguments: api.ToolCallFunctionArguments{ + "location": "Tokyo", }, }, }, }, - wantErr: nil, - prefix: "", }, { - name: "multiple tool calls with text after", - input: ` - {"name": "tool1", "arguments": {"arg1": 1}} text - {"name": "tool2", "arguments": {"arg2": "value"}} text - `, - nameField: "name", - argsField: "arguments", - wantToolCalls: []api.ToolCall{ + name: "two tool calls", + inputs: []string{`Okay, let's call both tools! {"name": "get_temperature", "arguments": {"city": "London", "format": "fahrenheit"}}{"name": "get_conditions", "arguments": {"location": "Tokyo"}}`}, + content: "Okay, let's call both tools! ", + tmpl: qwen, + calls: []api.ToolCall{ { Function: api.ToolCallFunction{ - Name: "tool1", - Arguments: map[string]any{ - "arg1": float64(1), + Index: 0, + Name: "get_temperature", + Arguments: api.ToolCallFunctionArguments{ + "city": "London", + "format": "fahrenheit", }, }, }, { Function: api.ToolCallFunction{ - Name: "tool2", - Arguments: map[string]any{ - "arg2": "value", + Index: 1, + Name: "get_conditions", + Arguments: api.ToolCallFunctionArguments{ + "location": "Tokyo", }, }, }, }, - wantErr: nil, - prefix: "", }, { - name: "second tool call in array", - input: ` - , {"name": "tool2", "arguments": {"arg2": "value"}} - `, - nameField: "name", - argsField: "arguments", - wantToolCalls: []api.ToolCall{ + name: "deepseek", + inputs: []string{"Wait, I need to call a tool<|tool▁calls▁begin|><|tool▁call▁begin|>function<|tool▁sep|>get_temperature\n```json\n{\"city\": \"Tokyo\"}\n```<|tool▁call▁end|><|tool▁calls▁end|><|end▁of▁sentence|>"}, + content: "Wait, I need to call a tool", + tmpl: deepseek, + calls: []api.ToolCall{ { Function: api.ToolCallFunction{ - Name: "tool2", - Arguments: map[string]any{ - "arg2": "value", + Index: 0, + Name: "get_temperature", + Arguments: api.ToolCallFunctionArguments{ + "city": "Tokyo", }, }, }, }, - wantErr: nil, - prefix: "", - }, - // a bad JSON would not return any tool calls or content as it would always accumulate more - { - name: "unbalanced square brackets", - input: `[{"name": "tool1", "arguments": {"arg1": [1, 2}]`, - nameField: "name", - argsField: "arguments", - wantToolCalls: nil, - wantErr: errAccumulateMore, - prefix: "", }, { - name: "incomplete square brackets", - input: `[{"name": "tool1", "arguments": {"arg1": [1, 2, 3`, - nameField: "name", - argsField: "arguments", - wantToolCalls: nil, - wantErr: errAccumulateMore, - prefix: "", - }, - { - name: "nested arrays in arguments", - input: `{"name": "tool1", "arguments": {"arg1": [1, 2, ["nested", "array"]]}}`, - nameField: "name", - argsField: "arguments", - wantToolCalls: []api.ToolCall{ + name: "deepseek incremental", + inputs: []string{ + "Wait", + ", I need", + " to call", + " a tool<|too", + "l▁calls▁begin", + "|>", + "<|tool▁call▁begin|>function<|tool▁sep|>get_temperature\n", + "```json\n", + "{\"city\": \"Tokyo\"}\n", + "```", + "<|tool▁c", "all▁end|>", + "<|tool▁calls▁end|>", + "<|end▁of▁sentence|>", + }, + content: "Wait, I need to call a tool", + tmpl: deepseek, + calls: []api.ToolCall{ { Function: api.ToolCallFunction{ - Name: "tool1", - Arguments: map[string]any{ - "arg1": []any{float64(1), float64(2), []any{"nested", "array"}}, + Index: 0, + Name: "get_temperature", + Arguments: api.ToolCallFunctionArguments{ + "city": "Tokyo", }, }, }, }, - wantErr: nil, - prefix: "", + }, + { + name: "json", + inputs: []string{ + "{", + "\"name\": \"get_temperature\",", + "\"arguments\": {", + "\"city\": \"Tokyo\"", + "}", + "}", + }, + content: "", + tmpl: json, + calls: []api.ToolCall{ + { + Function: api.ToolCallFunction{ + Index: 0, + Name: "get_temperature", + Arguments: api.ToolCallFunctionArguments{ + "city": "Tokyo", + }, + }, + }, + }, + }, + { + name: "json maybe a tool call", + inputs: []string{ + "{", + "\"name\": \"get_temperature\",", + "\"arguments\": {", + }, + content: "", + tmpl: json, + calls: nil, + }, + { + name: "json not a tool call", + inputs: []string{ + "{", + "\"name\": \"search\", ", + "\"arguments\": {", + "\"query\": \"What is the capital of Canada?\"", + "}", + "}", + }, + content: "{\"name\": \"search\", \"arguments\": {\"query\": \"What is the capital of Canada?\"}}", + tmpl: json, + calls: nil, + }, + { + name: "json object followed by tool call", + inputs: []string{ + "{\"name\": \"jeff\"}", + "{\"name\": \"get_conditions\", \"arguments\": {\"location\": \"San Francisco\"}}", + }, + content: "{\"name\": \"jeff\"}{\"name\": \"get_conditions\", \"arguments\": {\"location\": \"San Francisco\"}}", + tmpl: json, + }, + { + name: "json object followed by tool call split", + inputs: []string{ + "{\"name\": \"jeff\"} {", + "\"name\": \"get_conditions\", \"arguments\": {\"location\": \"San Francisco\"}}", + }, + content: "{\"name\": \"jeff\"} {\"name\": \"get_conditions\", \"arguments\": {\"location\": \"San Francisco\"}}", + tmpl: json, + }, + { + name: "json code", + inputs: []string{ + "for { fmt.Println(\"hello\") }", + }, + content: "for { fmt.Println(\"hello\") }", + tmpl: json, + }, + { + name: "list multiple", + inputs: []string{ + "[", + "{", + "\"name\": \"get_temperature\", ", + "\"arguments\": {", + "\"city\": \"London\"", + "}", + "},", + "{", + "\"name\": \"get_conditions\", ", + "\"arguments\": {", + "\"location\": \"Tokyo\"", + "}", + "}]", + }, + content: "", + tmpl: list, + calls: []api.ToolCall{ + { + Function: api.ToolCallFunction{ + Index: 0, + Name: "get_temperature", + Arguments: api.ToolCallFunctionArguments{ + "city": "London", + }, + }, + }, + { + Function: api.ToolCallFunction{ + Index: 1, + Name: "get_conditions", + Arguments: api.ToolCallFunctionArguments{ + "location": "Tokyo", + }, + }, + }, + }, + }, + { + name: "list partial", + inputs: []string{ + "[", + "{", + "\"name\": \"search\", ", + "\"arguments\": {", + "\"query\": \"What is the capital of Canada?\"", + "}", + "}", + }, + content: "", + tmpl: list, + calls: nil, + }, + { + name: "list not a tool call", + inputs: []string{ + "[special", + " del", + "ivery]", + }, + content: "[special delivery]", + tmpl: list, + calls: nil, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - gotCalls, err := parseJSONToolCalls(tt.input, tt.nameField, tt.argsField, tt.prefix) + parser := NewParser(tt.tmpl, tools) - if err != tt.wantErr { - t.Errorf("parseJSONToolCalls() error = %v, want %v", err, tt.wantErr) + var calls []api.ToolCall + var content string + for _, input := range tt.inputs { + tcs, c := parser.Add(input) + calls = append(calls, tcs...) + content += c } - if len(gotCalls) != 0 && tt.wantErr != nil { - t.Errorf("parseJSONToolCalls() valid = %v, want %v", len(gotCalls) == 0, tt.wantErr == nil) + if content != tt.content { + t.Errorf("Expected content %q, got %q", tt.content, content) } - if diff := cmp.Diff(gotCalls, tt.wantToolCalls); diff != "" { - t.Errorf("parseJSONToolCalls() tool calls mismatch (-got +want):\n%s", diff) + if len(calls) != len(tt.calls) { + t.Fatalf("Expected %d tool calls, got %d", len(tt.calls), len(calls)) + } + + for i, want := range tt.calls { + if diff := cmp.Diff(calls[i], want); diff != "" { + t.Errorf("Tool call %d mismatch (-got +want):\n%s", i, diff) + } } }) } } -func TestParseToolCalls(t *testing.T) { - p := filepath.Join("testdata") - t1 := api.ToolCall{ - Function: api.ToolCallFunction{ - Name: "get_current_weather", - Arguments: api.ToolCallFunctionArguments{ - "format": "fahrenheit", - "location": "San Francisco, CA", - }, - }, - } - t2 := api.ToolCall{ - Function: api.ToolCallFunction{ - Name: "get_current_weather", - Arguments: api.ToolCallFunctionArguments{ - "format": "celsius", - "location": "Toronto, Canada", - }, - }, - } - - cases := []struct { - name string - model string - output string - expectedToolCall []api.ToolCall - expectedTokens string +func TestDone(t *testing.T) { + tests := []struct { + name string + tag string + buffer []byte + want bool }{ { - name: "mistral malformed json with tool calls prefix", - model: "mistral", - output: `[TOOL_CALLS] [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_curren}]`, - expectedToolCall: []api.ToolCall{t1}, - expectedTokens: "", + name: "empty", + tag: "", + buffer: []byte{}, + want: false, }, { - name: "mistral multiple tool calls without prefix", - model: "mistral", - output: `[{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}} ]`, - expectedToolCall: []api.ToolCall{t1, t2}, - expectedTokens: "", + name: "empty", + tag: "", + buffer: []byte{}, + want: false, }, { - name: "mistral tool calls with text between no prefix", - model: "mistral", - output: `[{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}] - model outputs more tokens here and then [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]`, - expectedToolCall: []api.ToolCall{t1, t2}, - expectedTokens: `model outputs more tokens here and then [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]`, + name: "json open", + tag: "{", + buffer: []byte("{\"name\": \"get_weather\""), + want: false, }, { - name: "mistral valid json with tool calls prefix", - model: "mistral", - output: `[TOOL_CALLS] [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]`, - expectedToolCall: []api.ToolCall{t1, t2}, - expectedTokens: "", + name: "json closed", + tag: "{", + buffer: []byte("{\"name\": \"get_weather\"}"), + want: true, }, { - name: "mistral multiple tool calls with text between and prefix", - model: "mistral", - output: `[TOOL_CALLS] [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}] - model outputs more tokens here and then [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]`, - expectedToolCall: []api.ToolCall{t1, t2, t1, t2}, - expectedTokens: "", + name: "json empty", + tag: "{", + buffer: []byte("{}"), + want: true, }, { - name: "mistral incomplete json with tool calls prefix", - model: "mistral", - output: `[TOOL_CALLS] [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, `, - expectedToolCall: []api.ToolCall{}, - expectedTokens: "", + name: "list open", + tag: "[", + buffer: []byte("[{\"name\": \"get_weather\""), + want: false, }, { - name: "mistral invalid tool call with explanatory text no prefix", - model: "mistral", - output: `I'm not aware of that information. However, I can suggest searching for the weather using the "get_current_weather" function: - - [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]`, - expectedToolCall: []api.ToolCall{}, - expectedTokens: `I'm not aware of that information. However, I can suggest searching for the weather using the "get_current_weather" function: [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]`, + name: "list closed", + tag: "[", + buffer: []byte("[{\"name\": \"get_weather\"}]"), + want: true, }, { - name: "mistral tool calls without prefix", - model: "mistral", - output: `[{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]`, - expectedToolCall: []api.ToolCall{t1, t2}, - expectedTokens: "", - }, - { - name: "command r plus tool calls with json block format", - model: "command-r-plus", - output: "Action: ```json" + ` - [ - { - "tool_name": "get_current_weather", - "parameters": { - "format": "fahrenheit", - "location": "San Francisco, CA" - } - }, - { - "tool_name": "get_current_weather", - "parameters": { - "format": "celsius", - "location": "Toronto, Canada" - } - } - ] - ` + "```", - expectedToolCall: []api.ToolCall{t1, t2}, - expectedTokens: "", - }, - { - name: "firefunction tool calls with functools prefix", - model: "firefunction", - output: ` functools[{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]`, - expectedToolCall: []api.ToolCall{t1, t2}, - expectedTokens: "", - }, - { - name: "llama3 groq single tool call with xml tags", - model: "llama3-groq-tool-use", - output: ` - {"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}} - `, - expectedToolCall: []api.ToolCall{t1}, - expectedTokens: "", - }, - { - name: "xlam tool calls with wrapper object", - model: "xlam", - output: `{"tool_calls": [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]}`, - expectedToolCall: []api.ToolCall{t1, t2}, - expectedTokens: "", - }, - { - name: "qwen2.5 single tool call with prefix", - model: "qwen2.5", - output: `{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}}`, - expectedToolCall: []api.ToolCall{t1}, - expectedTokens: "", - }, - { - name: "qwen2.5 multiple tool calls with and without prefix", - model: "qwen2.5", - output: `{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}} {"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}} {"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}`, - expectedToolCall: []api.ToolCall{t1, t1, t2}, - expectedTokens: "", - }, - { - name: "qwen2.5 plain text response no tool calls", - model: "qwen2.5", - output: "The weather in San Francisco, CA is 70°F and in Toronto, Canada is 20°C.", - expectedToolCall: []api.ToolCall{}, - expectedTokens: "The weather in San Francisco, CA is 70°F and in Toronto, Canada is 20°C.", - }, - { - name: "qwen2.5 tool calls with trailing text", - model: "qwen2.5", - output: `[{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}}, {"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}] some tokens after call`, - expectedToolCall: []api.ToolCall{t1, t2}, - expectedTokens: "some tokens after call", - }, - { - name: "qwen2.5 tool calls with initial text", - model: "qwen2.5", - output: `some tokens before call [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}}, {"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]`, - expectedToolCall: []api.ToolCall{}, - expectedTokens: `some tokens before call [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}}, {"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]`, - }, - { - name: "qwen2.5 tool calls with prefix and trailing text", - model: "qwen2.5", - output: ` [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}}, {"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}] some tokens after call`, - expectedToolCall: []api.ToolCall{t1, t2}, - expectedTokens: "", - }, - { - name: "qwen2.5 tool calls with prefix and initial text", - model: "qwen2.5", - output: `some tokens before call [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}}, {"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}] `, - expectedToolCall: []api.ToolCall{t1, t2}, - expectedTokens: "some tokens before call", - }, - { - name: "qwen2.5 tool calls without and with prefix", - model: "qwen2.5", - output: `{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}} {"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}`, - expectedToolCall: []api.ToolCall{t1, t2}, - expectedTokens: "", - }, - { - name: "qwen2.5 tool calls without and with prefix and text between", - model: "qwen2.5", - output: `{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}} some tokens between {"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}} some tokens after call`, - expectedToolCall: []api.ToolCall{t1, t2}, - expectedTokens: "some tokens between", - }, - { - name: "qwen2.5 tool calls without prefix and invalid tool call with other tokens", - model: "qwen2.5", - output: `hi [{"options": "foo"}]`, - expectedToolCall: []api.ToolCall{}, - expectedTokens: `hi [{"options": "foo"}]`, - }, - { - name: "qwen2.5 tool calls with prefix and invalid tool call", - model: "qwen2.5", - output: ` [{"options": "foo"}] `, - expectedToolCall: []api.ToolCall{}, - expectedTokens: ``, - }, - { - name: "qwen3 tool call with think prefix and tool prefix (sent as a single token)", - model: "qwen3", - output: `Okay, let me think what tool we should use...{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}}`, - expectedToolCall: []api.ToolCall{t1}, - expectedTokens: "Okay, let me think what tool we should use...", - }, - { - name: "qwen3 tool call with think prefix, tool prefix, and whitespace (sent as separate tokens)", - model: "qwen3", - output: `Okay, let me think what tool we should use... { "name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}} `, - expectedToolCall: []api.ToolCall{t1}, - expectedTokens: "Okay, let me think what tool we should use...", - }, - { - name: "qwen3 empty think prefix without tool prefix and invalid tool call", - model: "qwen3", - output: ` {"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}} `, - expectedToolCall: []api.ToolCall{}, - expectedTokens: ` {"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}} `, - }, - { - name: "qwen3 empty think prefix with tool prefix and valid tool call", - model: "qwen3", - output: `{ "name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}} `, - expectedToolCall: []api.ToolCall{t1}, - expectedTokens: ``, - }, - { - name: "qwen3 invalid tool call with fake tool prefix (single rune suffix match)", - model: "qwen3", - output: `< fakeout {"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}} `, - expectedToolCall: []api.ToolCall{}, - expectedTokens: `< fakeout {"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}} `, - }, - { - name: "qwen3 invalid tool call with partial tool prefix (multiple rune suffix match)", - model: "qwen3", - output: ``, - expectedToolCall: []api.ToolCall{}, - expectedTokens: ``, - }, - { - name: "qwen3 invalid tool call with malformed tool prefix", - model: "qwen3", - output: ``, - expectedToolCall: []api.ToolCall{}, - expectedTokens: ``, - }, - { - name: "model with prefix in template, no prefix in output", - model: "qwen2.5", - output: `[{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}} {"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]`, - expectedToolCall: []api.ToolCall{t1, t2}, - expectedTokens: "", - }, - { - name: "model with prefix in template, prefix in output", - model: "qwen2.5", - output: `[{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}} {"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]`, - expectedToolCall: []api.ToolCall{t1, t2}, - expectedTokens: "", - }, - { - name: "model without prefix in template, no prefix in output", - model: "llama3.2", - output: `[{"name": "get_current_weather", "parameters": {"format":"fahrenheit","location":"San Francisco, CA"}} {"name": "get_current_weather", "parameters": {"format":"celsius","location":"Toronto, Canada"}}]`, - expectedToolCall: []api.ToolCall{t1, t2}, - expectedTokens: "", - }, - { - name: "model without prefix in template, no prefix in output, single tool call", - model: "llama3.2", - output: `{"name": "get_current_weather", "parameters": {"format":"fahrenheit","location":"San Francisco, CA"}}`, - expectedToolCall: []api.ToolCall{t1}, - expectedTokens: "", - }, - { - name: "model without prefix in template, prefix in output, multiple tool calls in list", - model: "llama3.2", - output: ` [{"name": "get_current_weather", "parameters": {"format":"fahrenheit","location":"San Francisco, CA"}} {"name": "get_current_weather", "parameters": {"format":"celsius","location":"Toronto, Canada"}}]`, - expectedToolCall: []api.ToolCall{t1, t2}, - expectedTokens: ``, - }, - { - name: "model without prefix in template, prefix in output, individual tool calls", - model: "llama3.2", - output: ` {"name": "get_current_weather", "parameters": {"format":"fahrenheit","location":"San Francisco, CA"}} {"name": "get_current_weather", "parameters": {"format":"celsius","location":"Toronto, Canada"}}`, - expectedToolCall: []api.ToolCall{t1, t2}, - expectedTokens: ``, - }, - { - name: "model with prefix in template, no prefix in output, tokens before", - model: "qwen2.5", - output: `some tokens before [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}} {"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]`, - expectedToolCall: []api.ToolCall{}, - expectedTokens: `some tokens before [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}} {"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]`, - }, - { - name: "model with prefix in template, prefix in output, tokens after", - model: "qwen2.5", - output: `[{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}} {"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}] some tokens after`, - expectedToolCall: []api.ToolCall{t1, t2}, - expectedTokens: "", - }, - { - name: "model without prefix in template, no prefix in output, tokens after", - model: "llama3.2", - output: `[{"name": "get_current_weather", "parameters": {"format":"fahrenheit","location":"San Francisco, CA"}} {"name": "get_current_weather", "parameters": {"format":"celsius","location":"Toronto, Canada"}}] some tokens after`, - expectedToolCall: []api.ToolCall{t1, t2}, - expectedTokens: "", - }, - { - name: "model without prefix in template, no prefix in output, tokens before", - model: "llama3.2", - output: `some tokens before [{"name": "get_current_weather", "parameters": {"format":"fahrenheit","location":"San Francisco, CA"}} {"name": "get_current_weather", "parameters": {"format":"celsius","location":"Toronto, Canada"}}]`, - expectedToolCall: []api.ToolCall{t1, t2}, - expectedTokens: `some tokens before`, - }, - { - name: "model without prefix in template, prefix in output, tokens after", - model: "llama3.2", - output: ` - [{"name": "get_current_weather", "parameters": {"format":"fahrenheit","location":"San Francisco, CA"}} {"name": "get_current_weather", "parameters": {"format":"celsius","location":"Toronto, Canada"}}] some tokens after`, - expectedToolCall: []api.ToolCall{t1, t2}, - expectedTokens: ``, - }, - { - name: "model without without prefix, match all jsons", - model: "llama3.2", - output: `model outputs some text [{"name": "get_current_weather", "parameters": {"format":"fahrenheit","location":"San Francisco, CA"}} {"name": "get_current_weather", "parameters": {"format":"celsius","location":"Toronto, Canada"}}] some tokens after`, - expectedToolCall: []api.ToolCall{t1, t2}, - expectedTokens: "model outputs some text", - }, - { - name: "model flushes tokens if tool call doesn't match", - model: "llama3.2", - output: `{ "user": {"id": 12345, "name": "Alice", "preferences": {"theme": "dark", "notifications": true}, "stats": {"points": 987, "level": 42}}}`, - expectedToolCall: []api.ToolCall{}, - expectedTokens: `{ "user": {"id": 12345, "name": "Alice", "preferences": {"theme": "dark", "notifications": true}, "stats": {"points": 987, "level": 42}}}`, - }, - { - name: "model flushes tokens if tool call doesn't match array", - model: "llama3.2", - output: `[ { "user": {"id": 12345, "name": "Alice", "preferences": {"theme": "dark", "notifications": true}, "stats": {"points": 987, "level": 42}}}]`, - expectedToolCall: []api.ToolCall{}, - expectedTokens: `[ { "user": {"id": 12345, "name": "Alice", "preferences": {"theme": "dark", "notifications": true}, "stats": {"points": 987, "level": 42}}}]`, + name: "list empty", + tag: "[", + buffer: []byte("[]"), + want: true, }, } - var tools []api.Tool - if err := json.Unmarshal(readFile(t, p, "tools.json").Bytes(), &tools); err != nil { - t.Fatal(err) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + parser := &Parser{ + tag: tt.tag, + buffer: tt.buffer, + } + got := parser.done() + if got != tt.want { + t.Errorf("done() = %t, want %t", got, tt.want) + } + }) + } +} + +func TestContent(t *testing.T) { + tests := []struct { + name string + tag string + content []byte + want string + n int + }{ + { + name: "empty", + content: []byte{}, + tag: "{", + want: "", + n: 0, + }, + { + name: "tag", + tag: "", + content: []byte("{\"name\": \"get_temperature\""), + want: "", + n: 0, + }, + { + name: "json object", + tag: "{", + content: []byte("{\"name\": \"get_temperature\"}"), + want: "{\"name\": \"get_temperature\"}", + n: 0, + }, + { + name: "json object after called", + tag: "{", + content: []byte("{\"hello\": \"world\"}"), + want: "{\"hello\": \"world\"}", + n: 0, + }, + { + name: "json object after called", + tag: "{", + content: []byte("{\"hello\": \"world\"}"), + want: "", + n: 1, + }, + { + name: "list", + tag: "[", + content: []byte("[{\"name\": \"get_temperature\"}]"), + want: "[{\"name\": \"get_temperature\"}]", + n: 0, + }, + { + name: "code", + tag: "{", + content: []byte("{ fmt.Println(\"hello\")"), + want: "{ fmt.Println(\"hello\")", + n: 0, + }, } - var messages []api.Message - if err := json.Unmarshal(readFile(t, p, "messages.json").Bytes(), &messages); err != nil { - t.Fatal(err) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + parser := &Parser{ + tag: tt.tag, + buffer: tt.content, + n: tt.n, + } + got := parser.Content() + if got != tt.want { + t.Errorf("Content() = %q, want %q", got, tt.want) + } + }) + } +} + +func TestFindTag(t *testing.T) { + cases := []struct { + name string + buffer []byte + tag string + i int + found bool + }{ + { + name: "no overlap", + buffer: []byte("hello world"), + tag: "", + i: -1, + found: false, + }, + { + name: "full overlap", + buffer: []byte(""), + tag: "", + i: 0, + found: true, + }, + { + name: "whitespace", + buffer: []byte(" \n {\"name\": \"bob\"}"), + tag: "", + i: 4, + found: true, + }, + { + name: "over", + buffer: []byte("{\"name\""), + tag: "", + i: 0, + found: true, + }, + { + name: "partial overlap", + buffer: []byte("text "), + tag: "", + i: 5, + found: true, + }, + { + name: "overlap with extra", + buffer: []byte(""), + tag: "", + i: 0, + found: true, + }, + { + name: "delimiter longer than string", + buffer: []byte(""), + tag: "", + i: -1, + found: false, + }, + { + name: "empty string", + buffer: []byte{}, + tag: "", + i: -1, + found: false, + }, + { + name: "single char overlap", + buffer: []byte("test<"), + tag: "", + i: 4, + found: false, + }, + { + name: "partial tool call", + buffer: []byte("hello ", + i: 6, + found: false, + }, + { + name: "square bracket", + buffer: []byte("calling tools: ["), + tag: "[", + i: 15, + found: true, + }, + { + name: "bracket", + buffer: []byte("{\"name\": \"bob\""), + tag: "{", + i: 0, + found: true, + }, + { + name: "bracket with whitespace", + buffer: []byte("\n\n{\n\"name\": \"bob\""), + tag: "{", + i: 2, + found: true, + }, } for _, tt := range cases { t.Run(tt.name, func(t *testing.T) { - tmpl, err := template.Parse(readFile(t, p, fmt.Sprintf("%s.gotmpl", tt.model)).String()) - if err != nil { - t.Fatal(err) + parser := &Parser{ + tag: tt.tag, + buffer: tt.buffer, + n: 0, + } + i, found := parser.findTag() + if i != tt.i { + t.Errorf("findTag(%q, %q) = %d; want %d", tt.buffer, tt.tag, i, tt.i) + } + if found != tt.found { + t.Errorf("findTag(%q, %q) = %t; want %t", tt.buffer, tt.tag, found, tt.found) + } + }) + } +} + +func TestFindArguments(t *testing.T) { + tests := []struct { + name string + buffer []byte + want map[string]any + }{ + { + name: "empty string", + buffer: []byte{}, + want: nil, + }, + { + name: "whitespace only", + buffer: []byte(" \n\t "), + want: nil, + }, + { + name: "unbalanced braces - missing closing", + buffer: []byte(`{"format": "fahrenheit", "location": "San Francisco"`), + want: nil, + }, + { + name: "unbalanced braces - extra closing", + buffer: []byte(`{"format": "fahrenheit"}}`), + want: map[string]any{ + "format": "fahrenheit", + }, + }, + { + name: "invalid JSON", + buffer: []byte(`{format: fahrenheit, location: "San Francisco"}`), + want: nil, + }, + { + name: "valid json", + buffer: []byte(`{"name": "get_temperature", "arguments": {"format": "fahrenheit", "location": "San Francisco, CA"}}`), + want: map[string]any{ + "format": "fahrenheit", + "location": "San Francisco, CA", + }, + }, + { + name: "valid arguments with special tokens", + buffer: []byte(`[tool]get_temperature[args]{"format": "fahrenheit", "location": "San Francisco, CA"}[end]`), + want: map[string]any{ + "format": "fahrenheit", + "location": "San Francisco, CA", + }, + }, + { + name: "valid arguments in array", + buffer: []byte(`[{"arguments": {"format": "fahrenheit", "location": "San Francisco, CA"}}`), + want: map[string]any{ + "format": "fahrenheit", + "location": "San Francisco, CA", + }, + }, + { + name: "nested deep", + buffer: []byte(`{"function": {"name": "get_temperature", "arguments": {"format": "fahrenheit", "location": "San Francisco, CA"}}}`), + want: map[string]any{ + "format": "fahrenheit", + "location": "San Francisco, CA", + }, + }, + { + name: "one arg", + buffer: []byte(`get_weather({"location": "San Francisco, CA"})`), + want: map[string]any{ + "location": "San Francisco, CA", + }, + }, + { + name: "two args", + buffer: []byte(`[{"name": "get_weather", "arguments": {"location": "San Francisco, CA", "format": "fahrenheit"}}, {"name": "get_weather", "arguments": {"location": "San Francisco, CA", "format": "fahrenheit"}}]`), + want: map[string]any{ + "location": "San Francisco, CA", + "format": "fahrenheit", + }, + }, + { + name: "deepseek", + buffer: []byte("<|tool▁calls▁begin|><|tool▁call▁begin|>function<|tool▁sep|>get_current_weather\n```json\n{\"location\": \"Tokyo\"}\n```<|tool▁call▁end|><|tool▁calls▁end|><|end▁of▁sentence|>"), + want: map[string]any{ + "location": "Tokyo", + }, + }, + } + + for _, tt := range tests { + parser := &Parser{ + buffer: tt.buffer, + properties: []string{"format", "location"}, + } + + t.Run(tt.name, func(t *testing.T) { + got, _ := parser.findArguments() + + if diff := cmp.Diff(got, tt.want); diff != "" { + t.Errorf("scanArguments() args mismatch (-got +want):\n%s", diff) } - - t.Run("template", func(t *testing.T) { - actual := &bytes.Buffer{} // Create new buffer for each test - if err := tmpl.Execute(actual, template.Values{Tools: tools, Messages: messages}); err != nil { - t.Fatal(err) - } - - if diff := cmp.Diff(actual.String(), readFile(t, p, fmt.Sprintf("%s.out", tt.model)).String()); diff != "" { - t.Errorf("mismatch (-got +want):\n%s", diff) - } - }) - - t.Run("parse", func(t *testing.T) { - tp, err := NewParser(tmpl.Template) - if err != nil { - t.Fatal(err) - } - got := []api.ToolCall{} - var gotTokens strings.Builder - - tokens := strings.Fields(tt.output) - for _, tok := range tokens { - s := " " + tok - - toolCalls, content := tp.Add(s) - if len(content) > 0 { - gotTokens.WriteString(content) - } else if len(toolCalls) > 0 { - got = append(got, toolCalls...) - } - } - - // Compare tool calls if we expect any - if diff := cmp.Diff(got, tt.expectedToolCall); diff != "" { - t.Errorf("tool calls mismatch (-got +want):\n%s", diff) - } - - // Compare tokens if we expect any - stripped := strings.TrimSpace(gotTokens.String()) - if diff := cmp.Diff(stripped, tt.expectedTokens); diff != "" { - t.Log("actualTokens", stripped, "expectedTokens", tt.expectedTokens) - t.Errorf("tokens mismatch (-got +want):\n%s", diff) - } - }) }) } } diff --git a/tools/tools_utils.go b/tools/tools_utils.go deleted file mode 100644 index b6f80729e..000000000 --- a/tools/tools_utils.go +++ /dev/null @@ -1,222 +0,0 @@ -package tools - -import ( - "bytes" - "encoding/json" - "errors" - "log/slog" - "slices" - "strings" - gotmpl "text/template" - "text/template/parse" - - "github.com/ollama/ollama/api" - "github.com/ollama/ollama/template" -) - -// extractToolCallsFormat traverses a template AST to find text that follows a ".ToolCalls" condition. -// It walks the template nodes looking for if-statements containing ".ToolCalls" and extracts any -// immediate text nodes that follow. This is used to identify tool call prefixes and formatting. -// -// Returns: -// - string: The extracted text following the first ".ToolCalls" condition found -// - bool: Whether a ".ToolCalls" condition was found in the template -func extractToolCallsFormat(tmpl *gotmpl.Template) (string, bool) { - if tmpl == nil || tmpl.Tree == nil { - slog.Debug("template or tree is nil") - return "", false - } - - var result string - var found bool - - var walk func(nodes []parse.Node) - walk = func(nodes []parse.Node) { - for _, node := range nodes { - if found { - return - } - - switch n := node.(type) { - case *parse.IfNode: - if isToolCallsNode(n) { - // Collect immediate TextNode(s) at start of IfNode's list - var sb strings.Builder - for _, innerNode := range n.List.Nodes { - if tn, ok := innerNode.(*parse.TextNode); ok { - sb.Write(tn.Text) - } else { - // Stop at first non-text node - break - } - } - result = sb.String() - found = true - return - } - // Recurse into child nodes - walk(n.List.Nodes) - if n.ElseList != nil { - walk(n.ElseList.Nodes) - } - case *parse.ListNode: - walk(n.Nodes) - case *parse.RangeNode: - walk(n.List.Nodes) - if n.ElseList != nil { - walk(n.ElseList.Nodes) - } - case *parse.WithNode: - walk(n.List.Nodes) - if n.ElseList != nil { - walk(n.ElseList.Nodes) - } - default: - // Continue to next node - continue - } - } - } - - walk(tmpl.Tree.Root.Nodes) - return result, found -} - -// isToolCallsNode detects if a node's condition includes ".ToolCalls" -func isToolCallsNode(n *parse.IfNode) bool { - for _, cmd := range n.Pipe.Cmds { - for _, arg := range cmd.Args { - if field, ok := arg.(*parse.FieldNode); ok { - if slices.Contains(field.Ident, "ToolCalls") { - return true - } - } - } - } - return false -} - -func toolPrefix(tmpl *gotmpl.Template) string { - tokenText, ok := extractToolCallsFormat(tmpl) - if !ok { - return "" - } - tokenText = strings.TrimSpace(tokenText) - tokenText = strings.ReplaceAll(tokenText, "\r", "") - tokenText = strings.ReplaceAll(tokenText, "\n", " ") - - return tokenText -} - -// toolTemplate creates a subtree from the node that ranges over .ToolCalls -// -// Returns: -// - *gotmpl.Template: The subtree containing the .ToolCalls range -// - error: Error if parsing failed -func toolTemplate(t *template.Template) (*gotmpl.Template, error) { - tmpl := t.Subtree(func(n parse.Node) bool { - if t, ok := n.(*parse.RangeNode); ok { - return slices.Contains(template.Identifiers(t.Pipe), "ToolCalls") - } - - return false - }) - - if tmpl == nil { - return nil, errors.New("failed to find tool template") - } - - return tmpl, nil -} - -// suffixOverlap returns the index in s where the longest suffix overlap with prefix begins -// -// Returns: -// - int: The starting index in s where the suffix overlap begins -func suffixOverlap(s, prefix string) int { - max := min(len(prefix), len(s)) - for i := max; i > 0; i-- { - if strings.HasSuffix(s, prefix[:i]) { - return len(s) - i - } - } - return -1 -} - -// extractToolArgs executes a template with a known tool call format to extract the name and arguments -// -// Returns: -// - string: The name of the tool call -// - string: The arguments of the tool call -// - error: Error if parsing failed -func extractToolArgs(tmpl *gotmpl.Template) (name, arguments string, err error) { - var b bytes.Buffer - if err := tmpl.Execute(&b, map[string][]api.ToolCall{ - "ToolCalls": { - { - Function: api.ToolCallFunction{ - Name: "@@name@@", - Arguments: api.ToolCallFunctionArguments{ - "@@argument@@": 1, - }, - }, - }, - }, - }); err != nil { - return "", "", err - } - - // Extract JSON object between curly braces - // JSON arrays are also valid as they will not be repeated in the template - output := b.String() - start := strings.Index(output, "{") - end := strings.LastIndex(output, "}") - if start == -1 || end == -1 || start > end { - return "", "", errors.New("no valid JSON object found in template output") - } - jsonStr := output[start : end+1] - - var obj map[string]any - if err := json.Unmarshal([]byte(jsonStr), &obj); err != nil { - return "", "", err - } - - // Find name and arguments fields - for k, v := range obj { - if str, ok := v.(string); ok && str == "@@name@@" { - name = k - } else if _, ok := v.(map[string]any); ok { - arguments = k - } - } - - if name == "" || arguments == "" { - slog.Debug("missing required fields in tool call template", "name", name, "arguments", arguments) - return "", "", errors.New("missing required fields in tool call template") - } - - return name, arguments, nil -} - -// collect recursively traverses an object to collect all nested maps -// -// Returns: -// - []map[string]any: A slice of all nested maps found in the object -func collect(obj any) []map[string]any { - var all []map[string]any - switch o := obj.(type) { - case map[string]any: - all = append(all, o) - for _, v := range o { - all = append(all, collect(v)...) - } - case []any: - for _, v := range o { - all = append(all, collect(v)...) - } - default: - return nil - } - - return all -} diff --git a/tools/tools_utils_test.go b/tools/tools_utils_test.go deleted file mode 100644 index e346117a9..000000000 --- a/tools/tools_utils_test.go +++ /dev/null @@ -1,497 +0,0 @@ -package tools - -import ( - "testing" - gotmpl "text/template" - - "github.com/ollama/ollama/template" -) - -func TestExtractToolCallsFormat(t *testing.T) { - cases := []struct { - name string - template string - want string - found bool - }{ - { - name: "nil template", - template: "", - want: "", - found: false, - }, - { - name: "basic tool call with text", - template: "{{if .ToolCalls}}Hello world{{end}}", - want: "Hello world", - found: true, - }, - { - name: "tool call with json format", - template: "{{if .ToolCalls}}```json\n{{end}}", - want: "```json\n", - found: true, - }, - { - name: "tool call in range", - template: "{{range .ToolCalls}}tool: {{.}}{{end}}", - want: "", - found: false, - }, - { - name: "tool call with multiple text nodes", - template: "{{if .ToolCalls}}First text{{if .Something}}inner{{end}}Second text{{end}}", - want: "First text", - found: true, - }, - { - name: "nested if without tool calls", - template: "{{if .Something}}{{if .OtherThing}}text{{end}}{{end}}", - want: "", - found: false, - }, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - tmpl, err := gotmpl.New("test").Parse(tc.template) - if err != nil && tc.template != "" { - t.Fatalf("failed to parse template: %v", err) - } - - got, found := extractToolCallsFormat(tmpl) - if got != tc.want { - t.Errorf("got text %q, want %q", got, tc.want) - } - if found != tc.found { - t.Errorf("got found %v, want %v", found, tc.found) - } - }) - } -} - -func TestToolPrefix(t *testing.T) { - cases := []struct { - name string - template string - want string - }{ - { - name: "basic tool call with action prefix", - template: "{{if .ToolCalls}}Action: ```json{{end}}", - want: "Action: ```json", - }, - { - name: "incomplete functools bracket", - template: "{{if .ToolCalls}}functools[{{end}}", - want: "functools[", - }, - { - name: "tool call with angle brackets", - template: "{{if .ToolCalls}}Hello, world! {{end}}", - want: "Hello, world! ", - }, - { - name: "multiple tool call formats", - template: "{{if .ToolCalls}}[tool_call] {{end}}", - want: "[tool_call] ", - }, - { - name: "single angle bracket tool call", - template: "{{if .ToolCalls}}{{end}}", - want: "", - }, - { - name: "incomplete angle bracket after tool call", - template: "{{if .ToolCalls}}[tool_call] <{{end}}", - want: "[tool_call] <", - }, - { - name: "angle bracket prefix with tool call", - template: "{{if .ToolCalls}}> {{end}}", - want: "> ", - }, - { - name: "uppercase tool call with incomplete bracket", - template: "{{if .ToolCalls}}[TOOL_CALL] [{{end}}", - want: "[TOOL_CALL] [", - }, - { - name: "uppercase tool call with adjacent bracket", - template: "{{if .ToolCalls}}[TOOL_CALL][{{end}}", - want: "[TOOL_CALL][", - }, - { - name: "tool call with pipe delimiters", - template: "{{if .ToolCalls}}<|tool_call|>{{end}}", - want: "<|tool_call|>", - }, - { - name: "tool with no prefix", - template: "{{if .ToolCalls}}{{end}}", - want: "", - }, - } - - for _, tt := range cases { - t.Run(tt.name, func(t *testing.T) { - tmpl, err := gotmpl.New("test").Parse(tt.template) - if err != nil { - t.Fatalf("failed to parse template: %v", err) - } - got := toolPrefix(tmpl) - if got != tt.want { - t.Errorf("ToolToken(%q) = %q; want %q", tt.template, got, tt.want) - } - }) - } -} - -func TestToolTemplate(t *testing.T) { - cases := []struct { - name string - template string - want bool - }{ - { - name: "basic tool call range", - template: "{{range .ToolCalls}}test{{end}}", - want: true, - }, - { - name: "no tool calls", - template: "{{range .Other}}test{{end}}", - want: false, - }, - { - name: "nested tool calls", - template: "{{range .Outer}}{{range .ToolCalls}}test{{end}}{{end}}", - want: true, - }, - { - name: "empty template", - template: "", - want: false, - }, - { - name: "tool calls in if statement", - template: "{{if .ToolCalls}}test{{end}}", - want: false, - }, - } - - for _, tt := range cases { - t.Run(tt.name, func(t *testing.T) { - tmpl, err := gotmpl.New("test").Parse(tt.template) - if err != nil { - t.Fatalf("failed to parse template: %v", err) - } - - parsed, err := template.Parse(tmpl.Root.String()) - if err != nil { - t.Fatalf("failed to parse template: %v", err) - } - - _, err = toolTemplate(parsed) - if err != nil && tt.want { - t.Errorf("toolTemplate() = %v; want %v", err, tt.want) - } - }) - } -} - -func TestSuffixOverlap(t *testing.T) { - cases := []struct { - name string - s string - d string - want int - }{ - { - name: "no overlap", - s: "hello world", - d: "", - want: -1, - }, - { - name: "full overlap", - s: "", - d: "", - want: 0, - }, - { - name: "partial overlap", - s: "text ", - d: "", - want: 5, - }, - { - name: "delimiter longer than string", - s: "", - d: "", - want: -1, - }, - { - name: "empty string", - s: "", - d: "", - want: -1, - }, - { - name: "empty delimiter", - s: "", - d: "", - want: -1, - }, - { - name: "single char overlap", - s: "test<", - d: "", - want: 4, - }, - { - name: "partial tool call", - s: "hello ", - want: 6, - }, - } - - for _, tt := range cases { - t.Run(tt.name, func(t *testing.T) { - got := suffixOverlap(tt.s, tt.d) - if got != tt.want { - t.Errorf("suffixOverlap(%q, %q) = %d; want %d", tt.s, tt.d, got, tt.want) - } - }) - } -} - -func TestExtractToolArgs(t *testing.T) { - cases := []struct { - name string - template string - wantName string - wantArgs string - wantErr bool - }{ - { - name: "basic tool call", - template: `{{ range .ToolCalls }} -{"name": "{{ .Function.Name }}", "parameters": {{ .Function.Arguments }}}{{ end }}`, - wantName: "name", - wantArgs: "parameters", - wantErr: false, - }, - { - name: "tool call with whitespace", - template: `{{range .ToolCalls}} - {"name": "{{.Function.Name}}", "parameters": {{.Function.Arguments}}} -{{end}}`, - wantName: "name", - wantArgs: "parameters", - wantErr: false, - }, - { - name: "tool call with extra content", - template: `Before {{range .ToolCalls}} -{"name": "{{.Function.Name}}", "arguments": {{.Function.Arguments}}}{{end}} After`, - wantName: "name", - wantArgs: "arguments", - wantErr: false, - }, - { - name: "no tool calls", - template: `{{if .Something}}no tools here{{end}}`, - wantName: "", - wantArgs: "", - wantErr: true, - }, - { - name: "empty template", - template: ``, - wantName: "", - wantArgs: "", - wantErr: true, - }, - { - name: "prefix within tool call", - template: `{{- if .ToolCalls }} -{{ range .ToolCalls }} - -{"name": "{{ .Function.Name }}", "arguments": {{ .Function.Arguments }}} -{{ end }}{{- end }}`, - wantName: "name", - wantArgs: "arguments", - wantErr: false, - }, - { - name: "JSON array", - template: `{{ range .ToolCalls }} -[{"name": "{{ .Function.Name }}", "arguments": {{ .Function.Arguments }}}]{{ end }}`, - wantName: "name", - wantArgs: "arguments", - wantErr: false, - }, - { - name: "invalid JSON", - template: `{{ range .ToolCalls }} -{"name": "{{ .Function.Name }}", "arguments": {{ .Function.Arguments }}, invalid}{{ end }}`, - wantName: "", - wantArgs: "", - wantErr: true, - }, - { - name: "missing name field", - template: `{{ range .ToolCalls }} -{"parameters": {{ .Function.Arguments }}}{{ end }}`, - wantName: "", - wantArgs: "", - wantErr: true, - }, - { - name: "missing arguments field", - template: `{{ range .ToolCalls }} -{"name": "{{ .Function.Name }}"}{{ end }}`, - wantName: "", - wantArgs: "", - wantErr: true, - }, - { - name: "malformed JSON", - template: `{{ range .ToolCalls }} -{"name": {{ .Function.Name }}, "arguments": {{ .Function.Arguments }}{{ end }}`, - wantName: "", - wantArgs: "", - wantErr: true, - }, - } - - for _, tt := range cases { - t.Run(tt.name, func(t *testing.T) { - tmpl, err := gotmpl.New("test").Parse(tt.template) - if err != nil { - t.Fatalf("failed to parse template: %v", err) - } - - gotName, gotArgs, err := extractToolArgs(tmpl) - if (err != nil) != tt.wantErr { - t.Errorf("extractToolArgs() error = %v, wantErr %v", err, tt.wantErr) - return - } - if err != nil { - return - } - - if gotName != tt.wantName { - t.Errorf("extractToolArgs() gotName = %q, want %q", gotName, tt.wantName) - } - if gotArgs != tt.wantArgs { - t.Errorf("extractToolArgs() gotArgs = %q, want %q", gotArgs, tt.wantArgs) - } - }) - } -} - -func TestCollect(t *testing.T) { - cases := []struct { - name string - obj any - want []map[string]any - }{ - { - name: "simple map", - obj: map[string]any{ - "key": "value", - }, - want: []map[string]any{ - {"key": "value"}, - }, - }, - { - name: "nested map", - obj: map[string]any{ - "outer": map[string]any{ - "inner": "value", - }, - }, - want: []map[string]any{ - {"outer": map[string]any{"inner": "value"}}, - {"inner": "value"}, - }, - }, - { - name: "array of maps", - obj: []any{ - map[string]any{"key1": "val1"}, - map[string]any{"key2": "val2"}, - }, - want: []map[string]any{ - {"key1": "val1"}, - {"key2": "val2"}, - }, - }, - { - name: "deeply nested", - obj: map[string]any{ - "l1": map[string]any{ - "l2": map[string]any{ - "l3": "value", - }, - }, - }, - want: []map[string]any{ - {"l1": map[string]any{"l2": map[string]any{"l3": "value"}}}, - {"l2": map[string]any{"l3": "value"}}, - {"l3": "value"}, - }, - }, - { - name: "non-map value", - obj: "string", - want: nil, - }, - } - - for _, tt := range cases { - t.Run(tt.name, func(t *testing.T) { - got := collect(tt.obj) - if len(got) != len(tt.want) { - t.Errorf("collect() got %d maps, want %d", len(got), len(tt.want)) - return - } - - // Compare each map in the result - for i := range tt.want { - if !mapsEqual(got[i], tt.want[i]) { - t.Errorf("collect() map[%d] = %v, want %v", i, got[i], tt.want[i]) - } - } - }) - } -} - -// mapsEqual compares two maps for deep equality -func mapsEqual(m1, m2 map[string]any) bool { - if len(m1) != len(m2) { - return false - } - for k, v1 := range m1 { - v2, ok := m2[k] - if !ok { - return false - } - switch val1 := v1.(type) { - case map[string]any: - val2, ok := v2.(map[string]any) - if !ok || !mapsEqual(val1, val2) { - return false - } - default: - if v1 != v2 { - return false - } - } - } - return true -} From 5a8eb0e1510a5a35b80649f2b88e9231716b6850 Mon Sep 17 00:00:00 2001 From: Phil Date: Sat, 14 Jun 2025 17:54:03 +0200 Subject: [PATCH 44/54] readme: add GPTranslate to community integrations (#11071) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index ffaec6281..a216589af 100644 --- a/README.md +++ b/README.md @@ -407,6 +407,7 @@ See the [API documentation](./docs/api.md) for all endpoints. - [Lumina](https://github.com/cushydigit/lumina.git) (A lightweight, minimal React.js frontend for interacting with Ollama servers) - [Tiny Notepad](https://pypi.org/project/tiny-notepad) (A lightweight, notepad-like interface to chat with ollama available on PyPI) - [macLlama (macOS native)](https://github.com/hellotunamayo/macLlama) (A native macOS GUI application for interacting with Ollama models, featuring a chat interface.) +- [GPTranslate](https://github.com/philberndt/GPTranslate) (A fast and lightweight, AI powered desktop translation application written with Rust and Tauri. Features real-time translation with OpenAI/Azure/Ollama.) ### Cloud From 502028968ddca04bd19c0859a73fb4e0cbeac3e1 Mon Sep 17 00:00:00 2001 From: NGC13009 Date: Mon, 16 Jun 2025 12:27:49 +0800 Subject: [PATCH 45/54] readme: add ollama-launcher to community integrations (#11080) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index a216589af..e148f9af3 100644 --- a/README.md +++ b/README.md @@ -408,6 +408,7 @@ See the [API documentation](./docs/api.md) for all endpoints. - [Tiny Notepad](https://pypi.org/project/tiny-notepad) (A lightweight, notepad-like interface to chat with ollama available on PyPI) - [macLlama (macOS native)](https://github.com/hellotunamayo/macLlama) (A native macOS GUI application for interacting with Ollama models, featuring a chat interface.) - [GPTranslate](https://github.com/philberndt/GPTranslate) (A fast and lightweight, AI powered desktop translation application written with Rust and Tauri. Features real-time translation with OpenAI/Azure/Ollama.) +- [ollama launcher](https://github.com/NGC13009/ollama-launcher) (A launcher for Ollama, aiming to provide users with convenient functions such as ollama server launching, management, or configuration.) ### Cloud From a6fbfc880c3de9b57e341db374907e2fedda9fa6 Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Mon, 16 Jun 2025 10:42:32 -0700 Subject: [PATCH 46/54] gguf: fix write order (#11068) * ggml: test write gguf order * ggml: fix write tensor order --- fs/ggml/gguf.go | 12 ++--- fs/ggml/gguf_test.go | 110 +++++++++++++++++++++++++------------------ 2 files changed, 68 insertions(+), 54 deletions(-) diff --git a/fs/ggml/gguf.go b/fs/ggml/gguf.go index 8e75625e0..33b596ccc 100644 --- a/fs/ggml/gguf.go +++ b/fs/ggml/gguf.go @@ -527,23 +527,17 @@ func WriteGGUF(f *os.File, kv KV, ts []*Tensor) error { return err } - keys := slices.Collect(maps.Keys(kv)) - slices.Sort(keys) - - for _, key := range keys { + for _, key := range slices.Sorted(maps.Keys(kv)) { if err := ggufWriteKV(f, key, kv[key]); err != nil { return err } } slices.SortStableFunc(ts, func(a, b *Tensor) int { - if i, j := a.block(), b.block(); i < 0 && j > 0 { - return 1 - } else if i > 0 && j < 0 { - return -1 - } else { + if i, j := a.block(), b.block(); i > 0 && j > 0 { return cmp.Compare(i, j) } + return cmp.Compare(a.Name, b.Name) }) var s uint64 diff --git a/fs/ggml/gguf_test.go b/fs/ggml/gguf_test.go index 0e0718008..bf7679182 100644 --- a/fs/ggml/gguf_test.go +++ b/fs/ggml/gguf_test.go @@ -2,62 +2,82 @@ package ggml import ( "bytes" + "math/rand/v2" "os" - "slices" + "strings" "testing" "github.com/google/go-cmp/cmp" ) func TestWriteGGUF(t *testing.T) { - w, err := os.CreateTemp(t.TempDir(), "*.bin") - if err != nil { - t.Fatal(err) - } - defer w.Close() + r := rand.New(rand.NewPCG(0, 0)) + for range 8 { + t.Run("shuffle", func(t *testing.T) { + t.Parallel() - if err := WriteGGUF(w, KV{ - "general.alignment": uint32(16), - }, []*Tensor{ - {Name: "test.0", Shape: []uint64{2, 3}, WriterTo: bytes.NewBuffer(slices.Repeat([]byte{0}, 2*3*4))}, - {Name: "test.1", Shape: []uint64{2, 3}, WriterTo: bytes.NewBuffer(slices.Repeat([]byte{0}, 2*3*4))}, - {Name: "test.2", Shape: []uint64{2, 3}, WriterTo: bytes.NewBuffer(slices.Repeat([]byte{0}, 2*3*4))}, - {Name: "test.3", Shape: []uint64{2, 3}, WriterTo: bytes.NewBuffer(slices.Repeat([]byte{0}, 2*3*4))}, - {Name: "test.4", Shape: []uint64{2, 3}, WriterTo: bytes.NewBuffer(slices.Repeat([]byte{0}, 2*3*4))}, - {Name: "test.5", Shape: []uint64{2, 3}, WriterTo: bytes.NewBuffer(slices.Repeat([]byte{0}, 2*3*4))}, - }); err != nil { - t.Fatal(err) - } + ts := []*Tensor{ + {Name: "token_embd.weight", Shape: []uint64{2, 3}, WriterTo: bytes.NewBuffer(make([]byte, 2*3))}, + {Name: "blk.0.attn_norm.weight", Shape: []uint64{2, 3}, WriterTo: bytes.NewBuffer(make([]byte, 2*3))}, + {Name: "blk.1.attn_norm.weight", Shape: []uint64{2, 3}, WriterTo: bytes.NewBuffer(make([]byte, 2*3))}, + {Name: "blk.2.attn_norm.weight", Shape: []uint64{2, 3}, WriterTo: bytes.NewBuffer(make([]byte, 2*3))}, + {Name: "blk.3.attn_norm.weight", Shape: []uint64{2, 3}, WriterTo: bytes.NewBuffer(make([]byte, 2*3))}, + {Name: "blk.4.attn_norm.weight", Shape: []uint64{2, 3}, WriterTo: bytes.NewBuffer(make([]byte, 2*3))}, + {Name: "blk.5.attn_norm.weight", Shape: []uint64{2, 3}, WriterTo: bytes.NewBuffer(make([]byte, 2*3))}, + {Name: "output_norm.weight", Shape: []uint64{3, 2}, WriterTo: bytes.NewBuffer(make([]byte, 3*2))}, + {Name: "output.weight", Shape: []uint64{3, 2}, WriterTo: bytes.NewBuffer(make([]byte, 3*2))}, + } - r, err := os.Open(w.Name()) - if err != nil { - t.Fatal(err) - } - defer r.Close() + r.Shuffle(len(ts), func(i, j int) { + ts[i], ts[j] = ts[j], ts[i] + }) - ff, err := Decode(r, 0) - if err != nil { - t.Fatal(err) - } + w, err := os.CreateTemp(t.TempDir(), strings.ReplaceAll(t.Name(), "/", "_")+"*.bin") + if err != nil { + t.Fatal(err) + } + defer w.Close() - if diff := cmp.Diff(ff.KV(), KV{ - "general.alignment": uint32(16), - "general.parameter_count": uint64(36), - }); diff != "" { - t.Errorf("Mismatch (-want +got):\n%s", diff) - } + if err := WriteGGUF(w, KV{ + "general.alignment": uint32(16), + }, ts); err != nil { + t.Fatal(err) + } - if diff := cmp.Diff(ff.Tensors(), Tensors{ - Offset: 336, - items: []*Tensor{ - {Name: "test.0", Offset: 0, Shape: []uint64{2, 3}}, - {Name: "test.1", Offset: 32, Shape: []uint64{2, 3}}, - {Name: "test.2", Offset: 64, Shape: []uint64{2, 3}}, - {Name: "test.3", Offset: 96, Shape: []uint64{2, 3}}, - {Name: "test.4", Offset: 128, Shape: []uint64{2, 3}}, - {Name: "test.5", Offset: 160, Shape: []uint64{2, 3}}, - }, - }, cmp.AllowUnexported(Tensors{})); diff != "" { - t.Errorf("Mismatch (-want +got):\n%s", diff) + r, err := os.Open(w.Name()) + if err != nil { + t.Fatal(err) + } + defer r.Close() + + ff, err := Decode(r, 0) + if err != nil { + t.Fatal(err) + } + + if diff := cmp.Diff(KV{ + "general.alignment": uint32(16), + "general.parameter_count": uint64(54), + }, ff.KV()); diff != "" { + t.Errorf("Mismatch (-want +got):\n%s", diff) + } + + if diff := cmp.Diff(Tensors{ + Offset: 608, + items: []*Tensor{ + {Name: "blk.0.attn_norm.weight", Offset: 0, Shape: []uint64{2, 3}}, + {Name: "blk.1.attn_norm.weight", Offset: 32, Shape: []uint64{2, 3}}, + {Name: "blk.2.attn_norm.weight", Offset: 64, Shape: []uint64{2, 3}}, + {Name: "blk.3.attn_norm.weight", Offset: 96, Shape: []uint64{2, 3}}, + {Name: "blk.4.attn_norm.weight", Offset: 128, Shape: []uint64{2, 3}}, + {Name: "blk.5.attn_norm.weight", Offset: 160, Shape: []uint64{2, 3}}, + {Name: "output.weight", Offset: 192, Shape: []uint64{3, 2}}, + {Name: "output_norm.weight", Offset: 224, Shape: []uint64{3, 2}}, + {Name: "token_embd.weight", Offset: 256, Shape: []uint64{2, 3}}, + }, + }, ff.Tensors(), cmp.AllowUnexported(Tensors{})); diff != "" { + t.Errorf("Mismatch (-want +got):\n%s", diff) + } + }) } } From 9e125d884cf995dfae7fcd74690d525e4326a517 Mon Sep 17 00:00:00 2001 From: Jeffrey Morgan Date: Mon, 16 Jun 2025 16:03:16 -0700 Subject: [PATCH 47/54] model: treat 'user defined' tokens as special tokens (#11077) --- model/vocabulary.go | 2 +- model/vocabulary_test.go | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 model/vocabulary_test.go diff --git a/model/vocabulary.go b/model/vocabulary.go index 24adbaca3..a86de58df 100644 --- a/model/vocabulary.go +++ b/model/vocabulary.go @@ -87,7 +87,7 @@ func (v *Vocabulary) Decode(id int32) string { func (v *Vocabulary) SpecialVocabulary() []string { v.specialOnce.Do(func() { for i := range v.Values { - if v.Types[i] == TOKEN_TYPE_CONTROL { + if v.Types[i] == TOKEN_TYPE_CONTROL || v.Types[i] == TOKEN_TYPE_USER_DEFINED { v.special = append(v.special, v.Values[i]) } } diff --git a/model/vocabulary_test.go b/model/vocabulary_test.go new file mode 100644 index 000000000..46f0ead23 --- /dev/null +++ b/model/vocabulary_test.go @@ -0,0 +1,16 @@ +package model + +import "testing" + +func TestVocabulary_SpecialVocabulary(t *testing.T) { + vocab := &Vocabulary{ + Values: []string{"<|startoftext|>", "<|endoftext|>", "<|tool_call_start|>", "<|tool_call_end|>", "hi"}, + Types: []int32{TOKEN_TYPE_CONTROL, TOKEN_TYPE_CONTROL, TOKEN_TYPE_USER_DEFINED, TOKEN_TYPE_USER_DEFINED, TOKEN_TYPE_NORMAL}, + } + + specialVocab := vocab.SpecialVocabulary() + + if len(specialVocab) != 4 { + t.Errorf("expected 4 special tokens, got %d", len(specialVocab)) + } +} From 6bda1d24798e40fc9ea1419c6ce22c6cdcc9dfe2 Mon Sep 17 00:00:00 2001 From: Jeffrey Morgan Date: Tue, 17 Jun 2025 10:51:43 -0700 Subject: [PATCH 48/54] tools: fix parsing tool calls without any parameters (#11101) Fixes issue where tool calls that don't expect any parameters were not being parsed. This also fixes two additional issues: one where 2+ tool calls would not be correctly parsed, and cases where tool calls with invalid parameters would still get parsed --- tools/tools.go | 85 ++++++++------- tools/tools_test.go | 258 ++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 297 insertions(+), 46 deletions(-) diff --git a/tools/tools.go b/tools/tools.go index efeaeee0c..a86163baa 100644 --- a/tools/tools.go +++ b/tools/tools.go @@ -18,9 +18,8 @@ const ( ) type Parser struct { - tag string - names []string - properties []string + tag string + tools []api.Tool state toolsState buffer []byte @@ -34,15 +33,10 @@ func NewParser(tmpl *template.Template, tools []api.Tool) *Parser { } func NewParserWithTag(tools []api.Tool, tag string) *Parser { - var p Parser - for _, t := range tools { - p.names = append(p.names, t.Function.Name) - for r := range t.Function.Parameters.Properties { - p.properties = append(p.properties, r) - } + return &Parser{ + tag: tag, + tools: tools, } - p.tag = tag - return &p } // Add processes a string input to parse tool calls and content that @@ -121,36 +115,40 @@ func (p *Parser) findTag() (int, bool) { // parseToolCall finds the next complete tool call in the buffer // incrementing n and advancing the buffer. func (p *Parser) parseToolCall() *api.ToolCall { - var name string var args map[string]any + var tool *api.Tool var end int = len(p.buffer) - // find tool name var i int - for _, n := range p.names { + // find tool name + for _, t := range p.tools { + n := t.Function.Name if i = bytes.Index(p.buffer, []byte(n)); i != -1 { if i+len(n) < end { - name = n + tool = &t end = i + len(n) } } } - if name == "" { + if tool == nil { return nil } - if args, i = p.findArguments(); args == nil { - return nil - } + // only look for arguments if the tool has parameters + if len(tool.Function.Parameters.Properties) > 0 { + if args, i = p.findArguments(*tool); args == nil { + return nil + } - if i > end { - end = i + if i > end { + end = i + } } tc := &api.ToolCall{ Function: api.ToolCallFunction{ - Name: name, + Name: tool.Function.Name, Arguments: args, Index: p.n, }, @@ -162,13 +160,17 @@ func (p *Parser) parseToolCall() *api.ToolCall { } // findArguments returns the first object that appears to be -// arguments and the position where the arguments end, returning nil and 0 if -// an invalid JSON object or non-arguments object is found first -func (p *Parser) findArguments() (map[string]any, int) { +// arguments for the provided tool, returning nil +func (p *Parser) findArguments(tool api.Tool) (map[string]any, int) { if len(p.buffer) == 0 { return nil, 0 } + // no arguments to parse + if len(tool.Function.Parameters.Properties) == 0 { + return nil, 0 + } + var braces int var start int = -1 var end int @@ -184,11 +186,13 @@ func (p *Parser) findArguments() (map[string]any, int) { } if c == '}' { - braces-- - if braces == 0 && start != -1 { - end = i + 1 - object = p.buffer[start:end] - break + if start != -1 { + braces-- + if braces == 0 { + end = i + 1 + object = p.buffer[start:end] + break + } } } } @@ -206,24 +210,27 @@ func (p *Parser) findArguments() (map[string]any, int) { var find func(obj any) map[string]any find = func(obj any) map[string]any { - switch v := obj.(type) { + switch obj := obj.(type) { case map[string]any: - // check if the object keys are valid tool properties - // TODO (jmorganca): check only sets of properties that - // go together instead of the entire set - for _, prop := range p.properties { - if _, exists := v[prop]; exists { - return v + found := true + for key := range obj { + if _, exists := tool.Function.Parameters.Properties[key]; !exists { + found = false + break } } - for _, value := range v { + if found { + return obj + } + + for _, value := range obj { if result := find(value); result != nil { return result } } case []any: - for _, item := range v { + for _, item := range obj { if result := find(item); result != nil { return result } diff --git a/tools/tools_test.go b/tools/tools_test.go index 678641684..ebf4ad7dc 100644 --- a/tools/tools_test.go +++ b/tools/tools_test.go @@ -104,6 +104,13 @@ func TestParser(t *testing.T) { }, }, }, + { + Type: "function", + Function: api.ToolFunction{ + Name: "say_hello", + Description: "Say hello", + }, + }, } tests := []struct { @@ -144,6 +151,20 @@ func TestParser(t *testing.T) { }, }, }, + { + name: "invalid arguments", + inputs: []string{`{"name": "get_conditions", "arguments": {"city": "San Francisco"}}`}, + content: "", + tmpl: qwen, + calls: nil, + }, + { + name: "missing args", + inputs: []string{`{"name": "get_conditions"}`}, + content: "", + tmpl: qwen, + calls: nil, + }, { name: "text before tool call", inputs: []string{`Let me check the weather. {"name": "get_temperature", "arguments": {"city": "New York"}}`}, @@ -161,6 +182,27 @@ func TestParser(t *testing.T) { }, }, }, + { + name: "qwen no args tool call", + inputs: []string{`Let me say hello to the user. I'll use the say_hello tool {"name": "say_hello"}`}, + content: "Let me say hello to the user. I'll use the say_hello tool ", + tmpl: qwen, + calls: []api.ToolCall{ + { + Function: api.ToolCallFunction{ + Index: 0, + Name: "say_hello", + }, + }, + }, + }, + { + name: "qwen no args with text", + inputs: []string{"Let me say hello to the user. I'll use the say_hello tool. "}, + content: "Let me say hello to the user. I'll use the say_hello tool. ", + tmpl: qwen, + calls: nil, + }, { name: "two tool calls in a list", inputs: []string{`[TOOL_CALLS] [{"name": "get_temperature", "arguments": {"city": "London", "format": "fahrenheit"}}, {"name": "get_conditions", "arguments": {"location": "Tokyo"}}][/TOOL_CALLS]`}, @@ -189,7 +231,7 @@ func TestParser(t *testing.T) { }, }, { - name: "two tool calls", + name: "qwen two tool calls", inputs: []string{`Okay, let's call both tools! {"name": "get_temperature", "arguments": {"city": "London", "format": "fahrenheit"}}{"name": "get_conditions", "arguments": {"location": "Tokyo"}}`}, content: "Okay, let's call both tools! ", tmpl: qwen, @@ -215,6 +257,29 @@ func TestParser(t *testing.T) { }, }, }, + { + name: "qwen two tool calls one with no args", + inputs: []string{`Let me check the weather. {"name": "say_hello"}{"name": "get_conditions", "arguments": {"location": "Tokyo"}}`}, + content: "Let me check the weather. ", + tmpl: qwen, + calls: []api.ToolCall{ + { + Function: api.ToolCallFunction{ + Index: 0, + Name: "say_hello", + }, + }, + { + Function: api.ToolCallFunction{ + Index: 1, + Name: "get_conditions", + Arguments: api.ToolCallFunctionArguments{ + "location": "Tokyo", + }, + }, + }, + }, + }, { name: "deepseek", inputs: []string{"Wait, I need to call a tool<|tool▁calls▁begin|><|tool▁call▁begin|>function<|tool▁sep|>get_temperature\n```json\n{\"city\": \"Tokyo\"}\n```<|tool▁call▁end|><|tool▁calls▁end|><|end▁of▁sentence|>"}, @@ -338,6 +403,50 @@ func TestParser(t *testing.T) { content: "for { fmt.Println(\"hello\") }", tmpl: json, }, + { + name: "json no args tool call", + inputs: []string{ + "{\"name\": \"say_hello\"}", + }, + content: "", + tmpl: json, + calls: []api.ToolCall{ + { + Function: api.ToolCallFunction{ + Index: 0, + Name: "say_hello", + }, + }, + }, + }, + { + name: "json no args no tool call", + inputs: []string{ + "I'll use the say_hello tool to say hello to the user.", + }, + content: "I'll use the say_hello tool to say hello to the user.", + tmpl: json, + calls: nil, + }, + + // TODO (jmorganca): this is a false positive, we should + // not be parsing this as a tool call + { + name: "json no args false positive", + inputs: []string{ + `{say_hello!!!}`, + }, + content: "", + tmpl: json, + calls: []api.ToolCall{ + { + Function: api.ToolCallFunction{ + Index: 0, + Name: "say_hello", + }, + }, + }, + }, { name: "list multiple", inputs: []string{ @@ -380,6 +489,30 @@ func TestParser(t *testing.T) { }, { name: "list partial", + inputs: []string{ + "[{", + "\"name\": \"get_conditions\", ", + "\"arguments\": {", + "\"location\": \"Tokyo\"", + "}", + "}", + }, + content: "", + tmpl: list, + calls: []api.ToolCall{ + { + Function: api.ToolCallFunction{ + Index: 0, + Name: "get_conditions", + Arguments: api.ToolCallFunctionArguments{ + "location": "Tokyo", + }, + }, + }, + }, + }, + { + name: "list invalid", inputs: []string{ "[", "{", @@ -393,6 +526,33 @@ func TestParser(t *testing.T) { tmpl: list, calls: nil, }, + { + name: "list trailing ]", + inputs: []string{ + "[", + "{", + "\"name\": \"get_conditions\", ", + "\"arguments\": {", + "\"location\": \"Tokyo\"", + "}", + "}", + "]", + "]", + }, + content: "", + tmpl: list, + calls: []api.ToolCall{ + { + Function: api.ToolCallFunction{ + Index: 0, + Name: "get_conditions", + Arguments: api.ToolCallFunctionArguments{ + "location": "Tokyo", + }, + }, + }, + }, + }, { name: "list not a tool call", inputs: []string{ @@ -404,6 +564,25 @@ func TestParser(t *testing.T) { tmpl: list, calls: nil, }, + { + name: "list with no arguments", + inputs: []string{ + "[", + "{", + "\"name\": \"say_hello\"", + "}", + }, + content: "", + tmpl: list, + calls: []api.ToolCall{ + { + Function: api.ToolCallFunction{ + Index: 0, + Name: "say_hello", + }, + }, + }, + }, } for _, tt := range tests { @@ -700,25 +879,75 @@ func TestFindTag(t *testing.T) { } func TestFindArguments(t *testing.T) { + tool := api.Tool{ + Type: "function", + Function: api.ToolFunction{ + Name: "get_temperature", + Description: "Retrieve the temperature for a given location", + Parameters: struct { + Type string `json:"type"` + Defs any `json:"$defs,omitempty"` + Items any `json:"items,omitempty"` + Required []string `json:"required"` + Properties map[string]struct { + Type api.PropertyType `json:"type"` + Items any `json:"items,omitempty"` + Description string `json:"description"` + Enum []any `json:"enum,omitempty"` + } `json:"properties"` + }{ + Type: "object", + Properties: map[string]struct { + Type api.PropertyType `json:"type"` + Items any `json:"items,omitempty"` + Description string `json:"description"` + Enum []any `json:"enum,omitempty"` + }{ + "format": { + Type: api.PropertyType{"string"}, + Description: "The format to return the temperature in", + Enum: []any{"fahrenheit", "celsius"}, + }, + "location": { + Type: api.PropertyType{"string"}, + Description: "The location to get the temperature for", + }, + }, + }, + }, + } + + tool2 := api.Tool{ + Type: "function", + Function: api.ToolFunction{ + Name: "say_hello", + Description: "Say hello to the user", + }, + } + tests := []struct { name string buffer []byte want map[string]any + tool api.Tool }{ { name: "empty string", buffer: []byte{}, want: nil, + tool: tool, }, { name: "whitespace only", buffer: []byte(" \n\t "), want: nil, + tool: tool, }, { name: "unbalanced braces - missing closing", buffer: []byte(`{"format": "fahrenheit", "location": "San Francisco"`), want: nil, + tool: tool, }, { name: "unbalanced braces - extra closing", @@ -726,11 +955,13 @@ func TestFindArguments(t *testing.T) { want: map[string]any{ "format": "fahrenheit", }, + tool: tool, }, { name: "invalid JSON", buffer: []byte(`{format: fahrenheit, location: "San Francisco"}`), want: nil, + tool: tool, }, { name: "valid json", @@ -739,6 +970,7 @@ func TestFindArguments(t *testing.T) { "format": "fahrenheit", "location": "San Francisco, CA", }, + tool: tool, }, { name: "valid arguments with special tokens", @@ -747,6 +979,7 @@ func TestFindArguments(t *testing.T) { "format": "fahrenheit", "location": "San Francisco, CA", }, + tool: tool, }, { name: "valid arguments in array", @@ -755,6 +988,7 @@ func TestFindArguments(t *testing.T) { "format": "fahrenheit", "location": "San Francisco, CA", }, + tool: tool, }, { name: "nested deep", @@ -763,39 +997,49 @@ func TestFindArguments(t *testing.T) { "format": "fahrenheit", "location": "San Francisco, CA", }, + tool: tool, }, { name: "one arg", - buffer: []byte(`get_weather({"location": "San Francisco, CA"})`), + buffer: []byte(`get_temperature({"location": "San Francisco, CA"})`), want: map[string]any{ "location": "San Francisco, CA", }, + tool: tool, }, { name: "two args", - buffer: []byte(`[{"name": "get_weather", "arguments": {"location": "San Francisco, CA", "format": "fahrenheit"}}, {"name": "get_weather", "arguments": {"location": "San Francisco, CA", "format": "fahrenheit"}}]`), + buffer: []byte(`[{"name": "get_temperature", "arguments": {"location": "San Francisco, CA", "format": "fahrenheit"}}, {"name": "get_weather", "arguments": {"location": "San Francisco, CA", "format": "fahrenheit"}}]`), want: map[string]any{ "location": "San Francisco, CA", "format": "fahrenheit", }, + tool: tool, + }, + { + name: "no args", + buffer: []byte(`{"name": "say_hello"}`), + want: nil, + tool: tool2, }, { name: "deepseek", - buffer: []byte("<|tool▁calls▁begin|><|tool▁call▁begin|>function<|tool▁sep|>get_current_weather\n```json\n{\"location\": \"Tokyo\"}\n```<|tool▁call▁end|><|tool▁calls▁end|><|end▁of▁sentence|>"), + buffer: []byte("<|tool▁calls▁begin|><|tool▁call▁begin|>function<|tool▁sep|>get_temperature\n```json\n{\"location\": \"Tokyo\"}\n```<|tool▁call▁end|><|tool▁calls▁end|><|end▁of▁sentence|>"), want: map[string]any{ "location": "Tokyo", }, + tool: tool, }, } for _, tt := range tests { parser := &Parser{ - buffer: tt.buffer, - properties: []string{"format", "location"}, + buffer: tt.buffer, + tools: []api.Tool{tool, tool2}, } t.Run(tt.name, func(t *testing.T) { - got, _ := parser.findArguments() + got, _ := parser.findArguments(tool) if diff := cmp.Diff(got, tt.want); diff != "" { t.Errorf("scanArguments() args mismatch (-got +want):\n%s", diff) From 55bbf3b4a1766fc99e32ebd39f0dd5749152b55d Mon Sep 17 00:00:00 2001 From: Jeffrey Morgan Date: Wed, 18 Jun 2025 05:20:43 -0700 Subject: [PATCH 49/54] tools: return empty arguments object instead of null (#11113) --- tools/tools.go | 4 ++-- tools/tools_test.go | 25 +++++++++++++++---------- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/tools/tools.go b/tools/tools.go index a86163baa..8a983e19f 100644 --- a/tools/tools.go +++ b/tools/tools.go @@ -115,11 +115,10 @@ func (p *Parser) findTag() (int, bool) { // parseToolCall finds the next complete tool call in the buffer // incrementing n and advancing the buffer. func (p *Parser) parseToolCall() *api.ToolCall { - var args map[string]any var tool *api.Tool var end int = len(p.buffer) - var i int + // find tool name for _, t := range p.tools { n := t.Function.Name @@ -136,6 +135,7 @@ func (p *Parser) parseToolCall() *api.ToolCall { } // only look for arguments if the tool has parameters + args := map[string]any{} if len(tool.Function.Parameters.Properties) > 0 { if args, i = p.findArguments(*tool); args == nil { return nil diff --git a/tools/tools_test.go b/tools/tools_test.go index ebf4ad7dc..35b583438 100644 --- a/tools/tools_test.go +++ b/tools/tools_test.go @@ -190,8 +190,9 @@ func TestParser(t *testing.T) { calls: []api.ToolCall{ { Function: api.ToolCallFunction{ - Index: 0, - Name: "say_hello", + Index: 0, + Name: "say_hello", + Arguments: api.ToolCallFunctionArguments{}, }, }, }, @@ -265,8 +266,9 @@ func TestParser(t *testing.T) { calls: []api.ToolCall{ { Function: api.ToolCallFunction{ - Index: 0, - Name: "say_hello", + Index: 0, + Name: "say_hello", + Arguments: api.ToolCallFunctionArguments{}, }, }, { @@ -413,8 +415,9 @@ func TestParser(t *testing.T) { calls: []api.ToolCall{ { Function: api.ToolCallFunction{ - Index: 0, - Name: "say_hello", + Index: 0, + Name: "say_hello", + Arguments: api.ToolCallFunctionArguments{}, }, }, }, @@ -441,8 +444,9 @@ func TestParser(t *testing.T) { calls: []api.ToolCall{ { Function: api.ToolCallFunction{ - Index: 0, - Name: "say_hello", + Index: 0, + Name: "say_hello", + Arguments: api.ToolCallFunctionArguments{}, }, }, }, @@ -577,8 +581,9 @@ func TestParser(t *testing.T) { calls: []api.ToolCall{ { Function: api.ToolCallFunction{ - Index: 0, - Name: "say_hello", + Index: 0, + Name: "say_hello", + Arguments: api.ToolCallFunctionArguments{}, }, }, }, From 60cfa2a203201e8513b7da26035f74495e498bc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9B=B9=E5=AE=B6=E5=B7=A7?= Date: Wed, 18 Jun 2025 20:21:45 +0800 Subject: [PATCH 50/54] cache: fix comment function name in cache.go (#11110) --- server/internal/cache/blob/cache.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/internal/cache/blob/cache.go b/server/internal/cache/blob/cache.go index a13515388..2f02ca955 100644 --- a/server/internal/cache/blob/cache.go +++ b/server/internal/cache/blob/cache.go @@ -59,7 +59,7 @@ type DiskCache struct { testHookBeforeFinalWrite func(f *os.File) } -// PutString is a convenience function for c.Put(d, strings.NewReader(s), int64(len(s))). +// PutBytes is a convenience function for c.Put(d, strings.NewReader(s), int64(len(s))). func PutBytes[S string | []byte](c *DiskCache, d Digest, data S) error { return c.Put(d, bytes.NewReader([]byte(data)), int64(len(data))) } From a6e64fbdf28f0d6cb97cc7f022ca493b905fe895 Mon Sep 17 00:00:00 2001 From: Jeffrey Morgan Date: Wed, 18 Jun 2025 05:42:44 -0700 Subject: [PATCH 51/54] Revert "feat: incremental gguf parser (#10822)" (#11114) This reverts commit 6b04cad7e816d1a119559e092d59f4fbaa6c3a0b. --- fs/gguf/gguf.go | 347 ------------------------------------ fs/gguf/gguf_test.go | 249 -------------------------- fs/gguf/keyvalue.go | 90 ---------- fs/gguf/keyvalue_test.go | 208 --------------------- fs/gguf/lazy.go | 89 --------- fs/gguf/reader.go | 23 --- fs/gguf/tensor.go | 288 ------------------------------ go.mod | 2 +- go.sum | 4 +- server/images.go | 24 +-- server/images_test.go | 165 ++++++++++++----- server/quantization_test.go | 12 +- server/sched_test.go | 20 ++- 13 files changed, 164 insertions(+), 1357 deletions(-) delete mode 100644 fs/gguf/gguf.go delete mode 100644 fs/gguf/gguf_test.go delete mode 100644 fs/gguf/keyvalue.go delete mode 100644 fs/gguf/keyvalue_test.go delete mode 100644 fs/gguf/lazy.go delete mode 100644 fs/gguf/reader.go delete mode 100644 fs/gguf/tensor.go diff --git a/fs/gguf/gguf.go b/fs/gguf/gguf.go deleted file mode 100644 index ebb9286f3..000000000 --- a/fs/gguf/gguf.go +++ /dev/null @@ -1,347 +0,0 @@ -package gguf - -import ( - "bytes" - "cmp" - "encoding/binary" - "errors" - "fmt" - "io" - "iter" - "os" - "slices" - "strings" -) - -const ( - typeUint8 uint32 = iota - typeInt8 - typeUint16 - typeInt16 - typeUint32 - typeInt32 - typeFloat32 - typeBool - typeString - typeArray - typeUint64 - typeInt64 - typeFloat64 -) - -var ErrUnsupported = errors.New("unsupported") - -type File struct { - Magic [4]byte - Version uint32 - - keyValues *lazy[KeyValue] - tensors *lazy[TensorInfo] - offset int64 - - file *os.File - reader *bufferedReader - bts []byte -} - -func Open(path string) (f *File, err error) { - f = &File{bts: make([]byte, 4096)} - f.file, err = os.Open(path) - if err != nil { - return nil, err - } - - f.reader = newBufferedReader(f.file, 32<<10) - - if err := binary.Read(f.reader, binary.LittleEndian, &f.Magic); err != nil { - return nil, err - } - - if bytes.Equal(f.Magic[:], []byte("gguf")) { - return nil, fmt.Errorf("%w file type %v", ErrUnsupported, f.Magic) - } - - if err := binary.Read(f.reader, binary.LittleEndian, &f.Version); err != nil { - return nil, err - } - - if f.Version != 3 { - return nil, fmt.Errorf("%w version %v", ErrUnsupported, f.Version) - } - - f.tensors, err = newLazy(f, f.readTensor) - if err != nil { - return nil, err - } - - f.tensors.successFunc = func() error { - offset := f.reader.offset - - alignment := cmp.Or(f.KeyValue("general.alignment").Int(), 32) - f.offset = offset + (alignment-offset%alignment)%alignment - return nil - } - - f.keyValues, err = newLazy(f, f.readKeyValue) - if err != nil { - return nil, err - } - - return f, nil -} - -func (f *File) readTensor() (TensorInfo, error) { - name, err := readString(f) - if err != nil { - return TensorInfo{}, err - } - - dims, err := read[uint32](f) - if err != nil { - return TensorInfo{}, err - } - - shape := make([]uint64, dims) - for i := range dims { - shape[i], err = read[uint64](f) - if err != nil { - return TensorInfo{}, err - } - } - - type_, err := read[uint32](f) - if err != nil { - return TensorInfo{}, err - } - - offset, err := read[uint64](f) - if err != nil { - return TensorInfo{}, err - } - - return TensorInfo{ - Name: name, - Offset: offset, - Shape: shape, - Type: TensorType(type_), - }, nil -} - -func (f *File) readKeyValue() (KeyValue, error) { - key, err := readString(f) - if err != nil { - return KeyValue{}, err - } - - t, err := read[uint32](f) - if err != nil { - return KeyValue{}, err - } - - value, err := func() (any, error) { - switch t { - case typeUint8: - return read[uint8](f) - case typeInt8: - return read[int8](f) - case typeUint16: - return read[uint16](f) - case typeInt16: - return read[int16](f) - case typeUint32: - return read[uint32](f) - case typeInt32: - return read[int32](f) - case typeUint64: - return read[uint64](f) - case typeInt64: - return read[int64](f) - case typeFloat32: - return read[float32](f) - case typeFloat64: - return read[float64](f) - case typeBool: - return read[bool](f) - case typeString: - return readString(f) - case typeArray: - return readArray(f) - default: - return nil, fmt.Errorf("%w type %d", ErrUnsupported, t) - } - }() - if err != nil { - return KeyValue{}, err - } - - return KeyValue{ - Key: key, - Value: Value{value}, - }, nil -} - -func read[T any](f *File) (t T, err error) { - err = binary.Read(f.reader, binary.LittleEndian, &t) - return t, err -} - -func readString(f *File) (string, error) { - n, err := read[uint64](f) - if err != nil { - return "", err - } - - if int(n) > len(f.bts) { - f.bts = make([]byte, n) - } - - bts := f.bts[:n] - if _, err := io.ReadFull(f.reader, bts); err != nil { - return "", err - } - defer clear(bts) - - return string(bts), nil -} - -func readArray(f *File) (any, error) { - t, err := read[uint32](f) - if err != nil { - return nil, err - } - - n, err := read[uint64](f) - if err != nil { - return nil, err - } - - switch t { - case typeUint8: - return readArrayData[uint8](f, n) - case typeInt8: - return readArrayData[int8](f, n) - case typeUint16: - return readArrayData[uint16](f, n) - case typeInt16: - return readArrayData[int16](f, n) - case typeUint32: - return readArrayData[uint32](f, n) - case typeInt32: - return readArrayData[int32](f, n) - case typeUint64: - return readArrayData[uint64](f, n) - case typeInt64: - return readArrayData[int64](f, n) - case typeFloat32: - return readArrayData[float32](f, n) - case typeFloat64: - return readArrayData[float64](f, n) - case typeBool: - return readArrayData[bool](f, n) - case typeString: - return readArrayString(f, n) - default: - return nil, fmt.Errorf("%w type %d", ErrUnsupported, t) - } -} - -func readArrayData[T any](f *File, n uint64) (s []T, err error) { - s = make([]T, n) - for i := range n { - e, err := read[T](f) - if err != nil { - return nil, err - } - - s[i] = e - } - - return s, nil -} - -func readArrayString(f *File, n uint64) (s []string, err error) { - s = make([]string, n) - for i := range n { - e, err := readString(f) - if err != nil { - return nil, err - } - - s[i] = e - } - - return s, nil -} - -func (f *File) Close() error { - f.keyValues.stop() - f.tensors.stop() - return f.file.Close() -} - -func (f *File) KeyValue(key string) KeyValue { - if !strings.HasPrefix(key, "general.") && !strings.HasPrefix(key, "tokenizer.") { - key = f.KeyValue("general.architecture").String() + "." + key - } - - if index := slices.IndexFunc(f.keyValues.values, func(kv KeyValue) bool { - return kv.Key == key - }); index >= 0 { - return f.keyValues.values[index] - } - - for keyValue, ok := f.keyValues.next(); ok; keyValue, ok = f.keyValues.next() { - if keyValue.Key == key { - return keyValue - } - } - - return KeyValue{} -} - -func (f *File) NumKeyValues() int { - return int(f.keyValues.count) -} - -func (f *File) KeyValues() iter.Seq2[int, KeyValue] { - return f.keyValues.All() -} - -func (f *File) TensorInfo(name string) TensorInfo { - if index := slices.IndexFunc(f.tensors.values, func(t TensorInfo) bool { - return t.Name == name - }); index >= 0 { - return f.tensors.values[index] - } - - // fast-forward through key values if we haven't already - _ = f.keyValues.rest() - for tensor, ok := f.tensors.next(); ok; tensor, ok = f.tensors.next() { - if tensor.Name == name { - return tensor - } - } - - return TensorInfo{} -} - -func (f *File) NumTensors() int { - return int(f.tensors.count) -} - -func (f *File) TensorInfos() iter.Seq2[int, TensorInfo] { - // fast forward through key values if we haven't already - f.keyValues.rest() - return f.tensors.All() -} - -func (f *File) TensorReader(name string) (TensorInfo, io.Reader, error) { - t := f.TensorInfo(name) - if t.NumBytes() == 0 { - return TensorInfo{}, nil, fmt.Errorf("tensor %s not found", name) - } - - // fast forward through tensor info if we haven't already - _ = f.tensors.rest() - return t, io.NewSectionReader(f.file, f.offset+int64(t.Offset), t.NumBytes()), nil -} diff --git a/fs/gguf/gguf_test.go b/fs/gguf/gguf_test.go deleted file mode 100644 index eea28a480..000000000 --- a/fs/gguf/gguf_test.go +++ /dev/null @@ -1,249 +0,0 @@ -package gguf_test - -import ( - "bytes" - "os" - "strconv" - "strings" - "testing" - - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" - "github.com/ollama/ollama/fs/ggml" - "github.com/ollama/ollama/fs/gguf" -) - -func createBinFile(tb testing.TB) string { - tb.Helper() - f, err := os.CreateTemp(tb.TempDir(), "") - if err != nil { - tb.Fatal(err) - } - defer f.Close() - - kv := ggml.KV{ - "general.architecture": "llama", - "llama.block_count": uint32(8), - "llama.embedding_length": uint32(3), - "llama.attention.head_count": uint32(2), - "llama.attention.head_count_kv": uint32(2), - "llama.attention.key_length": uint32(3), - "llama.rope.dimension_count": uint32(4), - "llama.rope.freq_base": float32(10000.0), - "llama.rope.freq_scale": float32(1.0), - "llama.attention.layer_norm_rms_epsilon": float32(1e-6), - "tokenizer.ggml.eos_token_id": uint32(0), - "tokenizer.ggml.eos_token_ids": []int32{1, 2, 3}, - "tokenizer.ggml.tokens": []string{"hello", "world"}, - "tokenizer.ggml.scores": []float32{0, 1}, - } - - tensors := []*ggml.Tensor{ - { - Name: "token_embd.weight", - Kind: 0, - Shape: []uint64{2, 3}, - WriterTo: bytes.NewBuffer(make([]byte, 4*2*3)), - }, - { - Name: "output.weight", - Kind: 0, - Shape: []uint64{3, 2}, - WriterTo: bytes.NewBuffer(make([]byte, 4*3*2)), - }, - } - - for i := range 8 { - tensors = append(tensors, &ggml.Tensor{ - Name: "blk." + strconv.Itoa(i) + ".attn_q.weight", - Kind: 0, - Shape: []uint64{3, 3}, - WriterTo: bytes.NewBuffer(make([]byte, 4*3*3)), - }, &ggml.Tensor{ - Name: "blk." + strconv.Itoa(i) + ".attn_k.weight", - Kind: 0, - Shape: []uint64{3, 3}, - WriterTo: bytes.NewBuffer(make([]byte, 4*3*3)), - }, &ggml.Tensor{ - Name: "blk." + strconv.Itoa(i) + ".attn_v.weight", - Kind: 0, - Shape: []uint64{3, 3}, - WriterTo: bytes.NewBuffer(make([]byte, 4*3*3)), - }, &ggml.Tensor{ - Name: "blk." + strconv.Itoa(i) + ".attn_output.weight", - Kind: 0, - Shape: []uint64{3, 3}, - WriterTo: bytes.NewBuffer(make([]byte, 4*3*3)), - }) - } - - if err := ggml.WriteGGUF(f, kv, tensors); err != nil { - tb.Fatal(err) - } - - return f.Name() -} - -func TestRead(t *testing.T) { - f, err := gguf.Open(createBinFile(t)) - if err != nil { - t.Fatal(err) - } - defer f.Close() - - if got := f.KeyValue("does.not.exist").Valid(); got { - t.Errorf(`KeyValue("does.not.exist").Exists() = %v, want false`, got) - } - - if got := f.KeyValue("general.architecture").String(); got != "llama" { - t.Errorf(`KeyValue("general.architecture").String() = %q, want %q`, got, "llama") - } - - if got := f.TensorInfo("token_embd.weight"); got.Name != "token_embd.weight" { - t.Errorf(`TensorInfo("token_embd.weight").Name = %q, want %q`, got.Name, "token_embd.weight") - } else if diff := cmp.Diff(got.Shape, []uint64{2, 3}); diff != "" { - t.Errorf(`TensorInfo("token_embd.weight").Shape mismatch (-got +want):\n%s`, diff) - } else if got.Type != gguf.TensorTypeF32 { - t.Errorf(`TensorInfo("token_embd.weight").Type = %d, want %d`, got.Type, gguf.TensorTypeF32) - } - - if got := f.KeyValue("block_count").Uint(); got != 8 { - t.Errorf(`KeyValue("block_count").Uint() = %d, want %d`, got, 8) - } - - if diff := cmp.Diff(f.KeyValue("tokenizer.ggml.tokens").Strings(), []string{"hello", "world"}); diff != "" { - t.Errorf("KeyValue(\"tokenizer.ggml.tokens\").Strings() mismatch (-got +want):\n%s", diff) - } - - if diff := cmp.Diff(f.KeyValue("tokenizer.ggml.scores").Floats(), []float64{0, 1}); diff != "" { - t.Errorf("KeyValue(\"tokenizer.ggml.scores\").Ints() mismatch (-got +want):\n%s", diff) - } - - var kvs []string - for _, kv := range f.KeyValues() { - if !kv.Valid() { - t.Error("found invalid key-value pair:", kv) - } - - kvs = append(kvs, kv.Key) - } - - if len(kvs) != f.NumKeyValues() { - t.Errorf("iterated key count = %d, want %d", len(kvs), f.NumKeyValues()) - } - - if diff := cmp.Diff(kvs, []string{ - "general.architecture", - "llama.block_count", - "llama.embedding_length", - "llama.attention.head_count", - "llama.attention.head_count_kv", - "llama.attention.key_length", - "llama.rope.dimension_count", - "llama.rope.freq_base", - "llama.rope.freq_scale", - "llama.attention.layer_norm_rms_epsilon", - "tokenizer.ggml.eos_token_id", - "tokenizer.ggml.eos_token_ids", - "tokenizer.ggml.tokens", - "tokenizer.ggml.scores", - }, cmpopts.SortSlices(strings.Compare)); diff != "" { - t.Errorf("KeyValues() mismatch (-got +want):\n%s", diff) - } - - var tis []string - for _, ti := range f.TensorInfos() { - if !ti.Valid() { - t.Error("found invalid tensor info:", ti) - } - - tis = append(tis, ti.Name) - } - - if len(tis) != f.NumTensors() { - t.Errorf("iterated tensor count = %d, want %d", len(tis), f.NumTensors()) - } - - if diff := cmp.Diff(tis, []string{ - "token_embd.weight", - "output.weight", - "blk.0.attn_q.weight", - "blk.0.attn_k.weight", - "blk.0.attn_v.weight", - "blk.0.attn_output.weight", - "blk.1.attn_q.weight", - "blk.1.attn_k.weight", - "blk.1.attn_v.weight", - "blk.1.attn_output.weight", - "blk.2.attn_q.weight", - "blk.2.attn_k.weight", - "blk.2.attn_v.weight", - "blk.2.attn_output.weight", - "blk.3.attn_q.weight", - "blk.3.attn_k.weight", - "blk.3.attn_v.weight", - "blk.3.attn_output.weight", - "blk.4.attn_q.weight", - "blk.4.attn_k.weight", - "blk.4.attn_v.weight", - "blk.4.attn_output.weight", - "blk.5.attn_q.weight", - "blk.5.attn_k.weight", - "blk.5.attn_v.weight", - "blk.5.attn_output.weight", - "blk.6.attn_q.weight", - "blk.6.attn_k.weight", - "blk.6.attn_v.weight", - "blk.6.attn_output.weight", - "blk.7.attn_q.weight", - "blk.7.attn_k.weight", - "blk.7.attn_v.weight", - "blk.7.attn_output.weight", - }, cmpopts.SortSlices(strings.Compare)); diff != "" { - t.Errorf("TensorInfos() mismatch (-got +want):\n%s", diff) - } - - ti, r, err := f.TensorReader("output.weight") - if err != nil { - t.Fatalf(`TensorReader("output.weight") error: %v`, err) - } - - if ti.Name != "output.weight" { - t.Errorf(`TensorReader("output.weight").Name = %q, want %q`, ti.Name, "output.weight") - } else if diff := cmp.Diff(ti.Shape, []uint64{3, 2}); diff != "" { - t.Errorf(`TensorReader("output.weight").Shape mismatch (-got +want):\n%s`, diff) - } else if ti.Type != gguf.TensorTypeF32 { - t.Errorf(`TensorReader("output.weight").Type = %d, want %d`, ti.Type, gguf.TensorTypeF32) - } - - var b bytes.Buffer - if _, err := b.ReadFrom(r); err != nil { - t.Fatalf(`ReadFrom TensorReader("output.weight") error: %v`, err) - } - - if b.Len() != int(ti.NumBytes()) { - t.Errorf(`ReadFrom TensorReader("output.weight") length = %d, want %d`, b.Len(), ti.NumBytes()) - } -} - -func BenchmarkRead(b *testing.B) { - b.ReportAllocs() - - p := createBinFile(b) - for b.Loop() { - f, err := gguf.Open(p) - if err != nil { - b.Fatal(err) - } - - if got := f.KeyValue("general.architecture").String(); got != "llama" { - b.Errorf("got = %q, want %q", got, "llama") - } - - // Iterate through some tensors - for range f.TensorInfos() { - } - - f.Close() - } -} diff --git a/fs/gguf/keyvalue.go b/fs/gguf/keyvalue.go deleted file mode 100644 index 5843326c1..000000000 --- a/fs/gguf/keyvalue.go +++ /dev/null @@ -1,90 +0,0 @@ -package gguf - -import ( - "reflect" - "slices" -) - -type KeyValue struct { - Key string - Value -} - -func (kv KeyValue) Valid() bool { - return kv.Key != "" && kv.Value.value != nil -} - -type Value struct { - value any -} - -func value[T any](v Value, kinds ...reflect.Kind) (t T) { - vv := reflect.ValueOf(v.value) - if slices.Contains(kinds, vv.Kind()) { - t = vv.Convert(reflect.TypeOf(t)).Interface().(T) - } - return -} - -func values[T any](v Value, kinds ...reflect.Kind) (ts []T) { - switch vv := reflect.ValueOf(v.value); vv.Kind() { - case reflect.Slice: - if slices.Contains(kinds, vv.Type().Elem().Kind()) { - ts = make([]T, vv.Len()) - for i := range vv.Len() { - ts[i] = vv.Index(i).Convert(reflect.TypeOf(ts[i])).Interface().(T) - } - } - } - return -} - -// Int returns Value as a signed integer. If it is not a signed integer, it returns 0. -func (v Value) Int() int64 { - return value[int64](v, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64) -} - -// Ints returns Value as a signed integer slice. If it is not a signed integer slice, it returns nil. -func (v Value) Ints() (i64s []int64) { - return values[int64](v, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64) -} - -// Uint converts an unsigned integer value to uint64. If the value is not a unsigned integer, it returns 0. -func (v Value) Uint() uint64 { - return value[uint64](v, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64) -} - -// Uints returns Value as a unsigned integer slice. If it is not a unsigned integer slice, it returns nil. -func (v Value) Uints() (u64s []uint64) { - return values[uint64](v, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64) -} - -// Float returns Value as a float. If it is not a float, it returns 0. -func (v Value) Float() float64 { - return value[float64](v, reflect.Float32, reflect.Float64) -} - -// Floats returns Value as a float slice. If it is not a float slice, it returns nil. -func (v Value) Floats() (f64s []float64) { - return values[float64](v, reflect.Float32, reflect.Float64) -} - -// Bool returns Value as a boolean. If it is not a boolean, it returns false. -func (v Value) Bool() bool { - return value[bool](v, reflect.Bool) -} - -// Bools returns Value as a boolean slice. If it is not a boolean slice, it returns nil. -func (v Value) Bools() (bools []bool) { - return values[bool](v, reflect.Bool) -} - -// String returns Value as a string. If it is not a string, it returns an empty string. -func (v Value) String() string { - return value[string](v, reflect.String) -} - -// Strings returns Value as a string slice. If it is not a string slice, it returns nil. -func (v Value) Strings() (strings []string) { - return values[string](v, reflect.String) -} diff --git a/fs/gguf/keyvalue_test.go b/fs/gguf/keyvalue_test.go deleted file mode 100644 index 2caacc538..000000000 --- a/fs/gguf/keyvalue_test.go +++ /dev/null @@ -1,208 +0,0 @@ -package gguf - -import ( - "testing" - - "github.com/google/go-cmp/cmp" -) - -func split(name string, values map[string][]any) (matched []any, unmatched []any) { - for key, value := range values { - if key == name { - matched = value - } else { - unmatched = append(unmatched, value...) - } - } - return -} - -func TestValue(t *testing.T) { - values := map[string][]any{ - "int64": {int(42), int8(42), int16(42), int32(42), int64(42)}, - "uint64": {uint(42), uint8(42), uint16(42), uint32(42), uint64(42)}, - "float64": {float32(42), float64(42)}, - "string": {"42", "hello"}, - "bool": {true, false}, - } - - t.Run("int64", func(t *testing.T) { - matched, unmatched := split("int64", values) - for _, v := range matched { - kv := KeyValue{"key", Value{v}} - if i64 := kv.Int(); i64 != 42 { - t.Errorf("expected 42, got %d", i64) - } - } - - for _, v := range unmatched { - kv := KeyValue{"key", Value{v}} - if i64 := kv.Int(); i64 != 0 { - t.Errorf("expected 42, got %d", i64) - } - } - }) - - t.Run("uint64", func(t *testing.T) { - matched, unmatched := split("uint64", values) - for _, v := range matched { - kv := KeyValue{"key", Value{v}} - if u64 := kv.Uint(); u64 != 42 { - t.Errorf("expected 42, got %d", u64) - } - } - - for _, v := range unmatched { - kv := KeyValue{"key", Value{v}} - if u64 := kv.Uint(); u64 != 0 { - t.Errorf("expected 42, got %d", u64) - } - } - }) - - t.Run("float64", func(t *testing.T) { - matched, unmatched := split("float64", values) - for _, v := range matched { - kv := KeyValue{"key", Value{v}} - if f64 := kv.Float(); f64 != 42 { - t.Errorf("expected 42, got %f", f64) - } - } - - for _, v := range unmatched { - kv := KeyValue{"key", Value{v}} - if f64 := kv.Float(); f64 != 0 { - t.Errorf("expected 42, got %f", f64) - } - } - }) - - t.Run("string", func(t *testing.T) { - matched, unmatched := split("string", values) - for _, v := range matched { - kv := KeyValue{"key", Value{v}} - if s := kv.String(); s != v { - t.Errorf("expected 42, got %s", s) - } - } - - for _, v := range unmatched { - kv := KeyValue{"key", Value{v}} - if s := kv.String(); s != "" { - t.Errorf("expected 42, got %s", s) - } - } - }) - - t.Run("bool", func(t *testing.T) { - matched, unmatched := split("bool", values) - for _, v := range matched { - kv := KeyValue{"key", Value{v}} - if b := kv.Bool(); b != v { - t.Errorf("expected true, got %v", b) - } - } - - for _, v := range unmatched { - kv := KeyValue{"key", Value{v}} - if b := kv.Bool(); b != false { - t.Errorf("expected false, got %v", b) - } - } - }) -} - -func TestValues(t *testing.T) { - values := map[string][]any{ - "int64s": {[]int{42}, []int8{42}, []int16{42}, []int32{42}, []int64{42}}, - "uint64s": {[]uint{42}, []uint8{42}, []uint16{42}, []uint32{42}, []uint64{42}}, - "float64s": {[]float32{42}, []float64{42}}, - "strings": {[]string{"42"}, []string{"hello"}}, - "bools": {[]bool{true}, []bool{false}}, - } - - t.Run("int64s", func(t *testing.T) { - matched, unmatched := split("int64s", values) - for _, v := range matched { - kv := KeyValue{"key", Value{v}} - if diff := cmp.Diff(kv.Ints(), []int64{42}); diff != "" { - t.Errorf("diff: %s", diff) - } - } - - for _, v := range unmatched { - kv := KeyValue{"key", Value{v}} - if i64s := kv.Ints(); i64s != nil { - t.Errorf("expected nil, got %v", i64s) - } - } - }) - - t.Run("uint64s", func(t *testing.T) { - matched, unmatched := split("uint64s", values) - for _, v := range matched { - kv := KeyValue{"key", Value{v}} - if diff := cmp.Diff(kv.Uints(), []uint64{42}); diff != "" { - t.Errorf("diff: %s", diff) - } - } - - for _, v := range unmatched { - kv := KeyValue{"key", Value{v}} - if u64s := kv.Uints(); u64s != nil { - t.Errorf("expected nil, got %v", u64s) - } - } - }) - - t.Run("float64s", func(t *testing.T) { - matched, unmatched := split("float64s", values) - for _, v := range matched { - kv := KeyValue{"key", Value{v}} - if diff := cmp.Diff(kv.Floats(), []float64{42}); diff != "" { - t.Errorf("diff: %s", diff) - } - } - - for _, v := range unmatched { - kv := KeyValue{"key", Value{v}} - if f64s := kv.Floats(); f64s != nil { - t.Errorf("expected nil, got %v", f64s) - } - } - }) - - t.Run("strings", func(t *testing.T) { - matched, unmatched := split("strings", values) - for _, v := range matched { - kv := KeyValue{"key", Value{v}} - if diff := cmp.Diff(kv.Strings(), v); diff != "" { - t.Errorf("diff: %s", diff) - } - } - - for _, v := range unmatched { - kv := KeyValue{"key", Value{v}} - if s := kv.Strings(); s != nil { - t.Errorf("expected nil, got %v", s) - } - } - }) - - t.Run("bools", func(t *testing.T) { - matched, unmatched := split("bools", values) - for _, v := range matched { - kv := KeyValue{"key", Value{v}} - if diff := cmp.Diff(kv.Bools(), v); diff != "" { - t.Errorf("diff: %s", diff) - } - } - - for _, v := range unmatched { - kv := KeyValue{"key", Value{v}} - if b := kv.Bools(); b != nil { - t.Errorf("expected nil, got %v", b) - } - } - }) -} diff --git a/fs/gguf/lazy.go b/fs/gguf/lazy.go deleted file mode 100644 index 16ab99093..000000000 --- a/fs/gguf/lazy.go +++ /dev/null @@ -1,89 +0,0 @@ -package gguf - -import ( - "encoding/binary" - "iter" - "log/slog" -) - -type lazy[T any] struct { - count uint64 - next func() (T, bool) - stop func() - values []T - - // successFunc is called when all values have been successfully read. - successFunc func() error -} - -func newLazy[T any](f *File, fn func() (T, error)) (*lazy[T], error) { - it := lazy[T]{} - if err := binary.Read(f.reader, binary.LittleEndian, &it.count); err != nil { - return nil, err - } - - it.values = make([]T, 0) - it.next, it.stop = iter.Pull(func(yield func(T) bool) { - for i := range it.count { - t, err := fn() - if err != nil { - slog.Error("error reading tensor", "index", i, "error", err) - return - } - - it.values = append(it.values, t) - if !yield(t) { - break - } - } - - if it.successFunc != nil { - it.successFunc() - } - }) - - return &it, nil -} - -func (g *lazy[T]) Values() iter.Seq[T] { - return func(yield func(T) bool) { - for _, v := range g.All() { - if !yield(v) { - break - } - } - } -} - -func (g *lazy[T]) All() iter.Seq2[int, T] { - return func(yield func(int, T) bool) { - for i := range int(g.count) { - if i < len(g.values) { - if !yield(i, g.values[i]) { - break - } - } else { - t, ok := g.next() - if !ok { - break - } - - if !yield(i, t) { - break - } - } - } - } -} - -func (g *lazy[T]) rest() (collected bool) { - for { - _, ok := g.next() - collected = collected || ok - if !ok { - break - } - } - - return collected -} diff --git a/fs/gguf/reader.go b/fs/gguf/reader.go deleted file mode 100644 index 0bd761840..000000000 --- a/fs/gguf/reader.go +++ /dev/null @@ -1,23 +0,0 @@ -package gguf - -import ( - "bufio" - "io" -) - -type bufferedReader struct { - offset int64 - *bufio.Reader -} - -func newBufferedReader(rs io.ReadSeeker, size int) *bufferedReader { - return &bufferedReader{ - Reader: bufio.NewReaderSize(rs, size), - } -} - -func (rs *bufferedReader) Read(p []byte) (n int, err error) { - n, err = rs.Reader.Read(p) - rs.offset += int64(n) - return n, err -} diff --git a/fs/gguf/tensor.go b/fs/gguf/tensor.go deleted file mode 100644 index 194c1d739..000000000 --- a/fs/gguf/tensor.go +++ /dev/null @@ -1,288 +0,0 @@ -package gguf - -import ( - "log/slog" - "strings" -) - -type TensorInfo struct { - Name string - Offset uint64 - Shape []uint64 - Type TensorType -} - -func (ti TensorInfo) Valid() bool { - return ti.Name != "" && ti.NumBytes() > 0 -} - -func (ti TensorInfo) NumValues() int64 { - var numItems int64 = 1 - for _, dim := range ti.Shape { - numItems *= int64(dim) - } - return numItems -} - -// NumBytes returns the number of bytes in the tensor. -func (ti TensorInfo) NumBytes() int64 { - return int64(float64(ti.NumValues()) * ti.Type.NumBytes()) -} - -func (ti TensorInfo) LogValue() slog.Value { - return slog.GroupValue( - slog.String("name", ti.Name), - slog.Int64("offset", int64(ti.Offset)), - slog.Any("shape", ti.Shape), - slog.Int64("num_values", ti.NumValues()), - slog.Int64("num_bytes", ti.NumBytes()), - slog.Any("type", ti.Type), - ) -} - -type TensorType uint32 - -const ( - TensorTypeF32 TensorType = iota - TensorTypeF16 - TensorTypeQ4_0 - TensorTypeQ4_1 - - // unexported // unused in gguf - tensorTypeQ4_2 - tensorTypeQ4_3 - - TensorTypeQ5_0 - TensorTypeQ5_1 - TensorTypeQ8_0 - TensorTypeQ8_1 - TensorTypeQ2_K - TensorTypeQ3_K - TensorTypeQ4_K - TensorTypeQ5_K - TensorTypeQ6_K - TensorTypeQ8_K - - // unexported // unquantizable by ollama - tensorTypeIQ2_XXS - tensorTypeIQ2_XS - tensorTypeIQ3_XXS - tensorTypeIQ1_S - tensorTypeIQ4_NL - tensorTypeIQ3_S - tensorTypeIQ2_S - tensorTypeIQ4_XS - - TensorTypeI8 - TensorTypeI16 - TensorTypeI32 - TensorTypeI64 - TensorTypeF64 - - // unexported // unquantizable by ollama - tensorTypeIQ1_M - - TensorTypeBF16 - - // unexported // unused in gguf - tensorTypeQ4_0_4_4 - tensorTypeQ4_0_4_8 - tensorTypeQ4_0_8_8 - - // unexported // unquantizable by ollama - tensorTypeTQ1_0 - tensorTypeTQ2_0 - - // unexported // unused in gguf - tensorTypeIQ4_NL_4_4 - tensorTypeIQ4_NL_4_8 - tensorTypeIQ4_NL_8_8 -) - -func (tt TensorType) NumBytes() float64 { - return float64(tt.typeSize()) / float64(tt.blockSize()) -} - -func (tt TensorType) typeSize() int64 { - switch tt { - case TensorTypeF32: - return 4 - case TensorTypeF16: - return 2 - case TensorTypeQ4_0: - return 2 + tt.blockSize()/2 - case TensorTypeQ4_1: - return 2 + 2 + tt.blockSize()/2 - case TensorTypeQ5_0: - return 2 + 4 + tt.blockSize()/2 - case TensorTypeQ5_1: - return 2 + 2 + 4 + tt.blockSize()/2 - case TensorTypeQ8_0: - return 2 + tt.blockSize() - case TensorTypeQ8_1: - return 2 + 2 + tt.blockSize() - case TensorTypeQ2_K: - return tt.blockSize()/16 + tt.blockSize()/4 + 2 + 2 - case TensorTypeQ3_K: - return tt.blockSize()/8 + tt.blockSize()/4 + 12 + 2 - case TensorTypeQ4_K: - return 2 + 2 + 12 + tt.blockSize()/2 - case TensorTypeQ5_K: - return 2 + 2 + 12 + tt.blockSize()/8 + tt.blockSize()/2 - case TensorTypeQ6_K: - return tt.blockSize()/2 + tt.blockSize()/4 + tt.blockSize()/16 + 2 - case TensorTypeQ8_K: - return 4 + tt.blockSize() + 2*tt.blockSize()/16 - case tensorTypeIQ2_XXS: - return 2 + 2*tt.blockSize()/8 - case tensorTypeIQ2_XS: - return 2 + 2*tt.blockSize()/8 + tt.blockSize()/32 - case tensorTypeIQ3_XXS: - return 2 + tt.blockSize()/4 + tt.blockSize()/8 - case tensorTypeIQ1_S: - return 2 + tt.blockSize()/8 + tt.blockSize()/16 - case tensorTypeIQ4_NL: - return 2 + tt.blockSize()/2 - case tensorTypeIQ3_S: - return 2 + tt.blockSize()/4 + tt.blockSize()/8 + tt.blockSize()/32 + 4 - case tensorTypeIQ2_S: - return 2 + tt.blockSize()/4 + tt.blockSize()/16 - case tensorTypeIQ4_XS: - return 2 + 2 + tt.blockSize()/2 + tt.blockSize()/64 - case TensorTypeI8: - return 1 - case TensorTypeI16: - return 2 - case TensorTypeI32: - return 4 - case TensorTypeI64: - return 8 - case TensorTypeF64: - return 8 - case tensorTypeIQ1_M: - return tt.blockSize()/8 + tt.blockSize()/16 + tt.blockSize()/32 - case TensorTypeBF16: - return 2 - default: - return 0 - } -} - -func (tt TensorType) blockSize() int64 { - switch tt { - case TensorTypeF32, - TensorTypeF16, - TensorTypeI8, - TensorTypeI16, - TensorTypeI32, - TensorTypeI64, - TensorTypeF64, - TensorTypeBF16: - return 1 - case TensorTypeQ4_0, - TensorTypeQ4_1, - TensorTypeQ5_0, - TensorTypeQ5_1, - TensorTypeQ8_0, - TensorTypeQ8_1, - tensorTypeIQ4_NL: - return 32 - default: - return 256 - } -} - -func (tt TensorType) String() string { - switch tt { - case TensorTypeF32: - return "f32" - case TensorTypeF16: - return "f16" - case TensorTypeQ4_0: - return "q4_0" - case TensorTypeQ4_1: - return "q4_1" - case tensorTypeQ4_2: - return "q4_2" - case tensorTypeQ4_3: - return "q4_3" - case TensorTypeQ5_0: - return "q5_0" - case TensorTypeQ5_1: - return "q5_1" - case TensorTypeQ8_0: - return "q8_0" - case TensorTypeQ8_1: - return "q8_1" - case TensorTypeQ2_K: - return "q2_k" - case TensorTypeQ3_K: - return "q3_k" - case TensorTypeQ4_K: - return "q4_k" - case TensorTypeQ5_K: - return "q5_k" - case TensorTypeQ6_K: - return "q6_k" - case TensorTypeQ8_K: - return "q8_k" - case tensorTypeIQ2_XXS: - return "iq2_xxs" - case tensorTypeIQ2_XS: - return "iq2_xs" - case tensorTypeIQ3_XXS: - return "iq3_xxs" - case tensorTypeIQ1_S: - return "iq1_s" - case tensorTypeIQ4_NL: - return "iq4_nl" - case tensorTypeIQ3_S: - return "iq3_s" - case tensorTypeIQ2_S: - return "iq2_s" - case tensorTypeIQ4_XS: - return "iq4_xs" - case TensorTypeI8: - return "i8" - case TensorTypeI16: - return "i16" - case TensorTypeI32: - return "i32" - case TensorTypeI64: - return "i64" - case TensorTypeF64: - return "f64" - case tensorTypeIQ1_M: - return "iq1_m" - case TensorTypeBF16: - return "bf16" - case tensorTypeQ4_0_4_4: - return "q4_0_4_4" - case tensorTypeQ4_0_4_8: - return "q4_0_4_8" - case tensorTypeQ4_0_8_8: - return "q4_0_8_8" - case tensorTypeTQ1_0: - return "tq1_0" - case tensorTypeTQ2_0: - return "tq2_0" - case tensorTypeIQ4_NL_4_4: - return "iq4_nl_4_4" - case tensorTypeIQ4_NL_4_8: - return "iq4_nl_4_8" - case tensorTypeIQ4_NL_8_8: - return "iq4_nl_8_8" - default: - return "unknown" - } -} - -func (tt TensorType) LogValue() slog.Value { - return slog.GroupValue( - slog.Uint64("value", uint64(tt)), - slog.String("name", strings.ToUpper(tt.String())), - slog.Int64("size", tt.typeSize()), - slog.Int64("block_size", tt.blockSize()), - slog.Float64("num_bytes", tt.NumBytes()), - ) -} diff --git a/go.mod b/go.mod index 6de5959be..283286b7d 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/d4l3k/go-bfloat16 v0.0.0-20211005043715-690c3bdd05f1 github.com/dlclark/regexp2 v1.11.4 github.com/emirpasic/gods/v2 v2.0.0-alpha - github.com/google/go-cmp v0.7.0 + github.com/google/go-cmp v0.6.0 github.com/mattn/go-runewidth v0.0.14 github.com/nlpodyssey/gopickle v0.3.0 github.com/pdevine/tensor v0.0.0-20240510204454-f88f4562727c diff --git a/go.sum b/go.sum index c0ab53aab..5755616f6 100644 --- a/go.sum +++ b/go.sum @@ -112,8 +112,8 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= -github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= diff --git a/server/images.go b/server/images.go index 38505cc51..d6cceff4c 100644 --- a/server/images.go +++ b/server/images.go @@ -23,7 +23,7 @@ import ( "github.com/ollama/ollama/api" "github.com/ollama/ollama/envconfig" - "github.com/ollama/ollama/fs/gguf" + "github.com/ollama/ollama/fs/ggml" "github.com/ollama/ollama/parser" "github.com/ollama/ollama/template" "github.com/ollama/ollama/thinking" @@ -73,18 +73,22 @@ func (m *Model) Capabilities() []model.Capability { capabilities := []model.Capability{} // Check for completion capability - f, err := gguf.Open(m.ModelPath) + r, err := os.Open(m.ModelPath) if err == nil { - defer f.Close() + defer r.Close() - if f.KeyValue("pooling_type").Valid() { - capabilities = append(capabilities, model.CapabilityEmbedding) + f, err := ggml.Decode(r, 1024) + if err == nil { + if _, ok := f.KV()[fmt.Sprintf("%s.pooling_type", f.KV().Architecture())]; ok { + capabilities = append(capabilities, model.CapabilityEmbedding) + } else { + capabilities = append(capabilities, model.CapabilityCompletion) + } + if _, ok := f.KV()[fmt.Sprintf("%s.vision.block_count", f.KV().Architecture())]; ok { + capabilities = append(capabilities, model.CapabilityVision) + } } else { - // If no embedding is specified, we assume the model supports completion - capabilities = append(capabilities, model.CapabilityCompletion) - } - if f.KeyValue("vision.block_count").Valid() { - capabilities = append(capabilities, model.CapabilityVision) + slog.Error("couldn't decode ggml", "error", err) } } else { slog.Error("couldn't open model file", "error", err) diff --git a/server/images_test.go b/server/images_test.go index a2fba8d98..363b298e1 100644 --- a/server/images_test.go +++ b/server/images_test.go @@ -1,42 +1,123 @@ package server import ( + "bytes" + "encoding/binary" + "errors" + "os" + "path/filepath" "strings" "testing" - "github.com/ollama/ollama/fs/ggml" "github.com/ollama/ollama/template" "github.com/ollama/ollama/types/model" ) +// Constants for GGUF magic bytes and version +var ( + ggufMagic = []byte{0x47, 0x47, 0x55, 0x46} // "GGUF" + ggufVer = uint32(3) // Version 3 +) + +// Helper function to create mock GGUF data +func createMockGGUFData(architecture string, vision bool) []byte { + var buf bytes.Buffer + + // Write GGUF header + buf.Write(ggufMagic) + binary.Write(&buf, binary.LittleEndian, ggufVer) + + // Write tensor count (0 for our test) + var numTensors uint64 = 0 + binary.Write(&buf, binary.LittleEndian, numTensors) + + // Calculate number of metadata entries + numMetaEntries := uint64(1) // architecture entry + if vision { + numMetaEntries++ + } + // Add embedding entry if architecture is "bert" + if architecture == "bert" { + numMetaEntries++ + } + binary.Write(&buf, binary.LittleEndian, numMetaEntries) + + // Write architecture metadata + archKey := "general.architecture" + keyLen := uint64(len(archKey)) + binary.Write(&buf, binary.LittleEndian, keyLen) + buf.WriteString(archKey) + + // String type (8) + var strType uint32 = 8 + binary.Write(&buf, binary.LittleEndian, strType) + + // String length + strLen := uint64(len(architecture)) + binary.Write(&buf, binary.LittleEndian, strLen) + buf.WriteString(architecture) + + if vision { + visionKey := architecture + ".vision.block_count" + keyLen = uint64(len(visionKey)) + binary.Write(&buf, binary.LittleEndian, keyLen) + buf.WriteString(visionKey) + + // uint32 type (4) + var uint32Type uint32 = 4 + binary.Write(&buf, binary.LittleEndian, uint32Type) + + // uint32 value (1) + var countVal uint32 = 1 + binary.Write(&buf, binary.LittleEndian, countVal) + } + // Write embedding metadata if architecture is "bert" + if architecture == "bert" { + poolKey := architecture + ".pooling_type" + keyLen = uint64(len(poolKey)) + binary.Write(&buf, binary.LittleEndian, keyLen) + buf.WriteString(poolKey) + + // uint32 type (4) + var uint32Type uint32 = 4 + binary.Write(&buf, binary.LittleEndian, uint32Type) + + // uint32 value (1) + var poolingVal uint32 = 1 + binary.Write(&buf, binary.LittleEndian, poolingVal) + } + + return buf.Bytes() +} + func TestModelCapabilities(t *testing.T) { - // Create completion model (llama architecture without vision) - completionModelPath, _ := createBinFile(t, ggml.KV{ - "general.architecture": "llama", - }, []*ggml.Tensor{}) + // Create a temporary directory for test files + tempDir := t.TempDir() - // Create vision model (llama architecture with vision block count) - visionModelPath, _ := createBinFile(t, ggml.KV{ - "general.architecture": "llama", - "llama.vision.block_count": uint32(1), - }, []*ggml.Tensor{}) + // Create different types of mock model files + completionModelPath := filepath.Join(tempDir, "model.bin") + visionModelPath := filepath.Join(tempDir, "vision_model.bin") + embeddingModelPath := filepath.Join(tempDir, "embedding_model.bin") + // Create a simple model file for tests that don't depend on GGUF content + simpleModelPath := filepath.Join(tempDir, "simple_model.bin") - // Create embedding model (bert architecture with pooling type) - embeddingModelPath, _ := createBinFile(t, ggml.KV{ - "general.architecture": "bert", - "bert.pooling_type": uint32(1), - }, []*ggml.Tensor{}) + if err := errors.Join( + os.WriteFile(completionModelPath, createMockGGUFData("llama", false), 0o644), + os.WriteFile(visionModelPath, createMockGGUFData("llama", true), 0o644), + os.WriteFile(embeddingModelPath, createMockGGUFData("bert", false), 0o644), + os.WriteFile(simpleModelPath, []byte("dummy model data"), 0o644), + ); err != nil { + t.Fatalf("Failed to create model files: %v", err) + } toolsInsertTemplate, err := template.Parse("{{ .prompt }}{{ if .tools }}{{ .tools }}{{ end }}{{ if .suffix }}{{ .suffix }}{{ end }}") if err != nil { t.Fatalf("Failed to parse template: %v", err) } - chatTemplate, err := template.Parse("{{ .prompt }}") if err != nil { t.Fatalf("Failed to parse template: %v", err) } - toolsTemplate, err := template.Parse("{{ .prompt }}{{ if .tools }}{{ .tools }}{{ end }}") if err != nil { t.Fatalf("Failed to parse template: %v", err) @@ -64,13 +145,21 @@ func TestModelCapabilities(t *testing.T) { }, expectedCaps: []model.Capability{model.CapabilityCompletion, model.CapabilityTools, model.CapabilityInsert}, }, + { + name: "model with tools and insert capability", + model: Model{ + ModelPath: simpleModelPath, + Template: toolsInsertTemplate, + }, + expectedCaps: []model.Capability{model.CapabilityTools, model.CapabilityInsert}, + }, { name: "model with tools capability", model: Model{ - ModelPath: completionModelPath, + ModelPath: simpleModelPath, Template: toolsTemplate, }, - expectedCaps: []model.Capability{model.CapabilityCompletion, model.CapabilityTools}, + expectedCaps: []model.Capability{model.CapabilityTools}, }, { name: "model with vision capability", @@ -135,33 +224,29 @@ func TestModelCapabilities(t *testing.T) { } func TestModelCheckCapabilities(t *testing.T) { - // Create simple model file for tests that don't depend on GGUF content - completionModelPath, _ := createBinFile(t, ggml.KV{ - "general.architecture": "llama", - }, []*ggml.Tensor{}) + // Create a temporary directory for test files + tempDir := t.TempDir() - // Create vision model (llama architecture with vision block count) - visionModelPath, _ := createBinFile(t, ggml.KV{ - "general.architecture": "llama", - "llama.vision.block_count": uint32(1), - }, []*ggml.Tensor{}) + visionModelPath := filepath.Join(tempDir, "vision_model.bin") + simpleModelPath := filepath.Join(tempDir, "model.bin") + embeddingModelPath := filepath.Join(tempDir, "embedding_model.bin") - // Create embedding model (bert architecture with pooling type) - embeddingModelPath, _ := createBinFile(t, ggml.KV{ - "general.architecture": "bert", - "bert.pooling_type": uint32(1), - }, []*ggml.Tensor{}) + if err := errors.Join( + os.WriteFile(simpleModelPath, []byte("dummy model data"), 0o644), + os.WriteFile(visionModelPath, createMockGGUFData("llama", true), 0o644), + os.WriteFile(embeddingModelPath, createMockGGUFData("bert", false), 0o644), + ); err != nil { + t.Fatalf("Failed to create model files: %v", err) + } toolsInsertTemplate, err := template.Parse("{{ .prompt }}{{ if .tools }}{{ .tools }}{{ end }}{{ if .suffix }}{{ .suffix }}{{ end }}") if err != nil { t.Fatalf("Failed to parse template: %v", err) } - chatTemplate, err := template.Parse("{{ .prompt }}") if err != nil { t.Fatalf("Failed to parse template: %v", err) } - toolsTemplate, err := template.Parse("{{ .prompt }}{{ if .tools }}{{ .tools }}{{ end }}") if err != nil { t.Fatalf("Failed to parse template: %v", err) @@ -176,7 +261,7 @@ func TestModelCheckCapabilities(t *testing.T) { { name: "completion model without tools capability", model: Model{ - ModelPath: completionModelPath, + ModelPath: simpleModelPath, Template: chatTemplate, }, checkCaps: []model.Capability{model.CapabilityTools}, @@ -185,7 +270,7 @@ func TestModelCheckCapabilities(t *testing.T) { { name: "model with all needed capabilities", model: Model{ - ModelPath: completionModelPath, + ModelPath: simpleModelPath, Template: toolsInsertTemplate, }, checkCaps: []model.Capability{model.CapabilityTools, model.CapabilityInsert}, @@ -193,7 +278,7 @@ func TestModelCheckCapabilities(t *testing.T) { { name: "model missing insert capability", model: Model{ - ModelPath: completionModelPath, + ModelPath: simpleModelPath, Template: toolsTemplate, }, checkCaps: []model.Capability{model.CapabilityInsert}, @@ -202,7 +287,7 @@ func TestModelCheckCapabilities(t *testing.T) { { name: "model missing vision capability", model: Model{ - ModelPath: completionModelPath, + ModelPath: simpleModelPath, Template: toolsTemplate, }, checkCaps: []model.Capability{model.CapabilityVision}, @@ -227,7 +312,7 @@ func TestModelCheckCapabilities(t *testing.T) { { name: "unknown capability", model: Model{ - ModelPath: completionModelPath, + ModelPath: simpleModelPath, Template: chatTemplate, }, checkCaps: []model.Capability{"unknown"}, diff --git a/server/quantization_test.go b/server/quantization_test.go index 8b726c836..4f717c2c2 100644 --- a/server/quantization_test.go +++ b/server/quantization_test.go @@ -257,8 +257,16 @@ func TestQuantizeModel(t *testing.T) { for _, tt := range cases { t.Run(tt.name, func(t *testing.T) { - p, _ := createBinFile(t, tt.kv, tt.tensors) - fp, err := os.Open(p) + f, err := os.CreateTemp(t.TempDir(), tt.name) + if err != nil { + t.Fatal(err.Error()) + } + defer f.Close() + err = fsggml.WriteGGUF(f, tt.kv, tt.tensors) + if err != nil { + t.Fatalf("failed to create initial model: %s", err) + } + fp, err := os.Open(f.Name()) if err != nil { t.Fatal(err.Error()) } diff --git a/server/sched_test.go b/server/sched_test.go index 3892fbbab..01fb9a703 100644 --- a/server/sched_test.go +++ b/server/sched_test.go @@ -112,7 +112,11 @@ func newScenarioRequest(t *testing.T, ctx context.Context, modelName string, est b.ctx, b.ctxDone = context.WithCancel(ctx) t.Helper() - p, _ := createBinFile(t, ggml.KV{ + f, err := os.CreateTemp(t.TempDir(), modelName) + require.NoError(t, err) + defer f.Close() + + require.NoError(t, ggml.WriteGGUF(f, ggml.KV{ "general.architecture": "llama", "llama.context_length": uint32(32), "llama.embedding_length": uint32(4096), @@ -125,14 +129,14 @@ func newScenarioRequest(t *testing.T, ctx context.Context, modelName string, est }, []*ggml.Tensor{ {Name: "blk.0.attn.weight", Kind: uint32(0), Offset: uint64(0), Shape: []uint64{1, 1, 1, 1}, WriterTo: bytes.NewReader(make([]byte, 32))}, {Name: "output.weight", Kind: uint32(0), Offset: uint64(0), Shape: []uint64{1, 1, 1, 1}, WriterTo: bytes.NewReader(make([]byte, 32))}, - }) + })) + require.NoError(t, err) + + fname := f.Name() + model := &Model{Name: modelName, ModelPath: fname} + b.f, err = llm.LoadModel(model.ModelPath, 0) + require.NoError(t, err) - model := &Model{Name: modelName, ModelPath: p} - f, err := llm.LoadModel(model.ModelPath, 0) - if err != nil { - t.Fatal(err) - } - b.f = f if duration == nil { duration = &api.Duration{Duration: 5 * time.Millisecond} } From ed567ef43b5822423bd165f5f57fb6bad5fce1b3 Mon Sep 17 00:00:00 2001 From: Jeffrey Morgan Date: Wed, 18 Jun 2025 05:45:00 -0700 Subject: [PATCH 52/54] Revert "ggml: Export GPU UUIDs" (#11115) This reverts commit aaa7818000c42a82fc030212c35ef83f9799efd7. --- .../patches/0017-ggml-Export-GPU-UUIDs.patch | 102 ------------------ ml/backend.go | 8 -- ml/backend/ggml/ggml.go | 6 -- ml/backend/ggml/ggml/include/ggml-backend.h | 1 - .../ggml/ggml/src/ggml-cuda/ggml-cuda.cu | 33 ------ .../ggml/ggml/src/ggml-metal/ggml-metal.m | 1 - 6 files changed, 151 deletions(-) delete mode 100644 llama/patches/0017-ggml-Export-GPU-UUIDs.patch diff --git a/llama/patches/0017-ggml-Export-GPU-UUIDs.patch b/llama/patches/0017-ggml-Export-GPU-UUIDs.patch deleted file mode 100644 index a2539034c..000000000 --- a/llama/patches/0017-ggml-Export-GPU-UUIDs.patch +++ /dev/null @@ -1,102 +0,0 @@ -From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 -From: Jesse Gross -Date: Thu, 24 Apr 2025 14:48:51 -0700 -Subject: [PATCH] ggml: Export GPU UUIDs - -This enables matching up devices and information reported by the backend -with tools (e.g. nvidia-smi) and system management libraries (e.g. nvml). ---- - ggml/include/ggml-backend.h | 1 + - ggml/src/ggml-cuda/ggml-cuda.cu | 33 ++++++++++++++++++++++++++++++++ - ggml/src/ggml-metal/ggml-metal.m | 1 + - 3 files changed, 35 insertions(+) - -diff --git a/ggml/include/ggml-backend.h b/ggml/include/ggml-backend.h -index 74e46716..a880df33 100644 ---- a/ggml/include/ggml-backend.h -+++ b/ggml/include/ggml-backend.h -@@ -152,6 +152,7 @@ extern "C" { - struct ggml_backend_dev_props { - const char * name; - const char * description; -+ const char * uuid; - size_t memory_free; - size_t memory_total; - enum ggml_backend_dev_type type; -diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu -index cb0d8528..4c829153 100644 ---- a/ggml/src/ggml-cuda/ggml-cuda.cu -+++ b/ggml/src/ggml-cuda/ggml-cuda.cu -@@ -2884,6 +2884,7 @@ struct ggml_backend_cuda_device_context { - int device; - std::string name; - std::string description; -+ std::string uuid; - }; - - static const char * ggml_backend_cuda_device_get_name(ggml_backend_dev_t dev) { -@@ -2896,6 +2897,11 @@ static const char * ggml_backend_cuda_device_get_description(ggml_backend_dev_t - return ctx->description.c_str(); - } - -+static const char * ggml_backend_cuda_device_get_uuid(ggml_backend_dev_t dev) { -+ ggml_backend_cuda_device_context * ctx = (ggml_backend_cuda_device_context *)dev->context; -+ return ctx->uuid.c_str(); -+} -+ - static void ggml_backend_cuda_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) { - ggml_backend_cuda_device_context * ctx = (ggml_backend_cuda_device_context *)dev->context; - ggml_cuda_set_device(ctx->device); -@@ -2910,6 +2916,7 @@ static enum ggml_backend_dev_type ggml_backend_cuda_device_get_type(ggml_backend - static void ggml_backend_cuda_device_get_props(ggml_backend_dev_t dev, ggml_backend_dev_props * props) { - props->name = ggml_backend_cuda_device_get_name(dev); - props->description = ggml_backend_cuda_device_get_description(dev); -+ props->uuid = ggml_backend_cuda_device_get_uuid(dev); - props->type = ggml_backend_cuda_device_get_type(dev); - ggml_backend_cuda_device_get_memory(dev, &props->memory_free, &props->memory_total); - -@@ -3458,6 +3465,32 @@ ggml_backend_reg_t ggml_backend_cuda_reg() { - CUDA_CHECK(cudaGetDeviceProperties(&prop, i)); - dev_ctx->description = prop.name; - -+ #if !defined(GGML_USE_HIP) -+ char uuid[64]; -+ snprintf(uuid, sizeof(uuid), -+ "GPU-%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x", -+ (unsigned char)prop.uuid.bytes[0], -+ (unsigned char)prop.uuid.bytes[1], -+ (unsigned char)prop.uuid.bytes[2], -+ (unsigned char)prop.uuid.bytes[3], -+ (unsigned char)prop.uuid.bytes[4], -+ (unsigned char)prop.uuid.bytes[5], -+ (unsigned char)prop.uuid.bytes[6], -+ (unsigned char)prop.uuid.bytes[7], -+ (unsigned char)prop.uuid.bytes[8], -+ (unsigned char)prop.uuid.bytes[9], -+ (unsigned char)prop.uuid.bytes[10], -+ (unsigned char)prop.uuid.bytes[11], -+ (unsigned char)prop.uuid.bytes[12], -+ (unsigned char)prop.uuid.bytes[13], -+ (unsigned char)prop.uuid.bytes[14], -+ (unsigned char)prop.uuid.bytes[15] -+ ); -+ dev_ctx->uuid = uuid; -+ #else -+ dev_ctx->uuid = "GPU-" + std::string(prop.uuid.bytes, 16); -+ #endif -+ - ggml_backend_dev_t dev = new ggml_backend_device { - /* .iface = */ ggml_backend_cuda_device_interface, - /* .reg = */ ®, -diff --git a/ggml/src/ggml-metal/ggml-metal.m b/ggml/src/ggml-metal/ggml-metal.m -index 1b56f858..ee4f2dcb 100644 ---- a/ggml/src/ggml-metal/ggml-metal.m -+++ b/ggml/src/ggml-metal/ggml-metal.m -@@ -5703,6 +5703,7 @@ static enum ggml_backend_dev_type ggml_backend_metal_device_get_type(ggml_backen - static void ggml_backend_metal_device_get_props(ggml_backend_dev_t dev, struct ggml_backend_dev_props * props) { - props->name = ggml_backend_metal_device_get_name(dev); - props->description = ggml_backend_metal_device_get_description(dev); -+ props->uuid = "0"; - props->type = ggml_backend_metal_device_get_type(dev); - ggml_backend_metal_device_get_memory(dev, &props->memory_free, &props->memory_total); - props->caps = (struct ggml_backend_dev_caps) { diff --git a/ml/backend.go b/ml/backend.go index 2df6c8923..65f169486 100644 --- a/ml/backend.go +++ b/ml/backend.go @@ -124,10 +124,6 @@ type DeviceMemory struct { // may not be persistent across instances of the runner. Name string - // UUID is a unique persistent identifier for the device for matching - // with system management libraries - UUID string - // Weights is the per-layer memory needed for the model weights. Weights []Memory @@ -156,10 +152,6 @@ func (m DeviceMemory) LogValue() slog.Value { attrs = append(attrs, slog.Any("Graph", m.Graph)) } - if len(attrs) > 0 && m.UUID != "" { - attrs = append([]slog.Attr{slog.String("UUID", m.UUID)}, attrs...) - } - return slog.GroupValue(attrs...) } diff --git a/ml/backend/ggml/ggml.go b/ml/backend/ggml/ggml.go index 5a9fe67e5..76172ae1a 100644 --- a/ml/backend/ggml/ggml.go +++ b/ml/backend/ggml/ggml.go @@ -136,9 +136,6 @@ func New(modelPath string, params ml.BackendParams) (ml.Backend, error) { } requiredMemory.CPU.Name = C.GoString(C.ggml_backend_dev_name(cpuDeviceBufferType.d)) - var props C.struct_ggml_backend_dev_props - C.ggml_backend_dev_get_props(cpuDeviceBufferType.d, &props) - requiredMemory.CPU.UUID = C.GoString(props.uuid) requiredMemory.CPU.Weights = make([]ml.Memory, blocks+1) requiredMemory.CPU.Cache = make([]ml.Memory, blocks+1) @@ -153,9 +150,6 @@ func New(modelPath string, params ml.BackendParams) (ml.Backend, error) { }) btDeviceMemory[bt] = &requiredMemory.GPUs[i] requiredMemory.GPUs[i].Name = C.GoString(C.ggml_backend_dev_name(d)) - var props C.struct_ggml_backend_dev_props - C.ggml_backend_dev_get_props(d, &props) - requiredMemory.GPUs[i].UUID = C.GoString(props.uuid) requiredMemory.GPUs[i].Weights = make([]ml.Memory, blocks+1) requiredMemory.GPUs[i].Cache = make([]ml.Memory, blocks+1) } diff --git a/ml/backend/ggml/ggml/include/ggml-backend.h b/ml/backend/ggml/ggml/include/ggml-backend.h index a880df33e..74e467163 100644 --- a/ml/backend/ggml/ggml/include/ggml-backend.h +++ b/ml/backend/ggml/ggml/include/ggml-backend.h @@ -152,7 +152,6 @@ extern "C" { struct ggml_backend_dev_props { const char * name; const char * description; - const char * uuid; size_t memory_free; size_t memory_total; enum ggml_backend_dev_type type; diff --git a/ml/backend/ggml/ggml/src/ggml-cuda/ggml-cuda.cu b/ml/backend/ggml/ggml/src/ggml-cuda/ggml-cuda.cu index 4c8291532..cb0d8528d 100644 --- a/ml/backend/ggml/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ml/backend/ggml/ggml/src/ggml-cuda/ggml-cuda.cu @@ -2884,7 +2884,6 @@ struct ggml_backend_cuda_device_context { int device; std::string name; std::string description; - std::string uuid; }; static const char * ggml_backend_cuda_device_get_name(ggml_backend_dev_t dev) { @@ -2897,11 +2896,6 @@ static const char * ggml_backend_cuda_device_get_description(ggml_backend_dev_t return ctx->description.c_str(); } -static const char * ggml_backend_cuda_device_get_uuid(ggml_backend_dev_t dev) { - ggml_backend_cuda_device_context * ctx = (ggml_backend_cuda_device_context *)dev->context; - return ctx->uuid.c_str(); -} - static void ggml_backend_cuda_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) { ggml_backend_cuda_device_context * ctx = (ggml_backend_cuda_device_context *)dev->context; ggml_cuda_set_device(ctx->device); @@ -2916,7 +2910,6 @@ static enum ggml_backend_dev_type ggml_backend_cuda_device_get_type(ggml_backend static void ggml_backend_cuda_device_get_props(ggml_backend_dev_t dev, ggml_backend_dev_props * props) { props->name = ggml_backend_cuda_device_get_name(dev); props->description = ggml_backend_cuda_device_get_description(dev); - props->uuid = ggml_backend_cuda_device_get_uuid(dev); props->type = ggml_backend_cuda_device_get_type(dev); ggml_backend_cuda_device_get_memory(dev, &props->memory_free, &props->memory_total); @@ -3465,32 +3458,6 @@ ggml_backend_reg_t ggml_backend_cuda_reg() { CUDA_CHECK(cudaGetDeviceProperties(&prop, i)); dev_ctx->description = prop.name; - #if !defined(GGML_USE_HIP) - char uuid[64]; - snprintf(uuid, sizeof(uuid), - "GPU-%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x", - (unsigned char)prop.uuid.bytes[0], - (unsigned char)prop.uuid.bytes[1], - (unsigned char)prop.uuid.bytes[2], - (unsigned char)prop.uuid.bytes[3], - (unsigned char)prop.uuid.bytes[4], - (unsigned char)prop.uuid.bytes[5], - (unsigned char)prop.uuid.bytes[6], - (unsigned char)prop.uuid.bytes[7], - (unsigned char)prop.uuid.bytes[8], - (unsigned char)prop.uuid.bytes[9], - (unsigned char)prop.uuid.bytes[10], - (unsigned char)prop.uuid.bytes[11], - (unsigned char)prop.uuid.bytes[12], - (unsigned char)prop.uuid.bytes[13], - (unsigned char)prop.uuid.bytes[14], - (unsigned char)prop.uuid.bytes[15] - ); - dev_ctx->uuid = uuid; - #else - dev_ctx->uuid = "GPU-" + std::string(prop.uuid.bytes, 16); - #endif - ggml_backend_dev_t dev = new ggml_backend_device { /* .iface = */ ggml_backend_cuda_device_interface, /* .reg = */ ®, diff --git a/ml/backend/ggml/ggml/src/ggml-metal/ggml-metal.m b/ml/backend/ggml/ggml/src/ggml-metal/ggml-metal.m index ee4f2dcb0..1b56f858c 100644 --- a/ml/backend/ggml/ggml/src/ggml-metal/ggml-metal.m +++ b/ml/backend/ggml/ggml/src/ggml-metal/ggml-metal.m @@ -5703,7 +5703,6 @@ static enum ggml_backend_dev_type ggml_backend_metal_device_get_type(ggml_backen static void ggml_backend_metal_device_get_props(ggml_backend_dev_t dev, struct ggml_backend_dev_props * props) { props->name = ggml_backend_metal_device_get_name(dev); props->description = ggml_backend_metal_device_get_description(dev); - props->uuid = "0"; props->type = ggml_backend_metal_device_get_type(dev); ggml_backend_metal_device_get_memory(dev, &props->memory_free, &props->memory_total); props->caps = (struct ggml_backend_dev_caps) { From 6baf1e31e2e5b28c4ce6d145f4524448c9747204 Mon Sep 17 00:00:00 2001 From: Jeffrey Morgan Date: Wed, 18 Jun 2025 07:30:49 -0700 Subject: [PATCH 53/54] Revert "Revert "ggml: Export GPU UUIDs" (#11115)" (#11117) Reverts PR #11115. The original change was mistakingly reverted instead of #10822 --- .../patches/0017-ggml-Export-GPU-UUIDs.patch | 102 ++++++++++++++++++ ml/backend.go | 8 ++ ml/backend/ggml/ggml.go | 6 ++ ml/backend/ggml/ggml/include/ggml-backend.h | 1 + .../ggml/ggml/src/ggml-cuda/ggml-cuda.cu | 33 ++++++ .../ggml/ggml/src/ggml-metal/ggml-metal.m | 1 + 6 files changed, 151 insertions(+) create mode 100644 llama/patches/0017-ggml-Export-GPU-UUIDs.patch diff --git a/llama/patches/0017-ggml-Export-GPU-UUIDs.patch b/llama/patches/0017-ggml-Export-GPU-UUIDs.patch new file mode 100644 index 000000000..a2539034c --- /dev/null +++ b/llama/patches/0017-ggml-Export-GPU-UUIDs.patch @@ -0,0 +1,102 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Jesse Gross +Date: Thu, 24 Apr 2025 14:48:51 -0700 +Subject: [PATCH] ggml: Export GPU UUIDs + +This enables matching up devices and information reported by the backend +with tools (e.g. nvidia-smi) and system management libraries (e.g. nvml). +--- + ggml/include/ggml-backend.h | 1 + + ggml/src/ggml-cuda/ggml-cuda.cu | 33 ++++++++++++++++++++++++++++++++ + ggml/src/ggml-metal/ggml-metal.m | 1 + + 3 files changed, 35 insertions(+) + +diff --git a/ggml/include/ggml-backend.h b/ggml/include/ggml-backend.h +index 74e46716..a880df33 100644 +--- a/ggml/include/ggml-backend.h ++++ b/ggml/include/ggml-backend.h +@@ -152,6 +152,7 @@ extern "C" { + struct ggml_backend_dev_props { + const char * name; + const char * description; ++ const char * uuid; + size_t memory_free; + size_t memory_total; + enum ggml_backend_dev_type type; +diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu +index cb0d8528..4c829153 100644 +--- a/ggml/src/ggml-cuda/ggml-cuda.cu ++++ b/ggml/src/ggml-cuda/ggml-cuda.cu +@@ -2884,6 +2884,7 @@ struct ggml_backend_cuda_device_context { + int device; + std::string name; + std::string description; ++ std::string uuid; + }; + + static const char * ggml_backend_cuda_device_get_name(ggml_backend_dev_t dev) { +@@ -2896,6 +2897,11 @@ static const char * ggml_backend_cuda_device_get_description(ggml_backend_dev_t + return ctx->description.c_str(); + } + ++static const char * ggml_backend_cuda_device_get_uuid(ggml_backend_dev_t dev) { ++ ggml_backend_cuda_device_context * ctx = (ggml_backend_cuda_device_context *)dev->context; ++ return ctx->uuid.c_str(); ++} ++ + static void ggml_backend_cuda_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) { + ggml_backend_cuda_device_context * ctx = (ggml_backend_cuda_device_context *)dev->context; + ggml_cuda_set_device(ctx->device); +@@ -2910,6 +2916,7 @@ static enum ggml_backend_dev_type ggml_backend_cuda_device_get_type(ggml_backend + static void ggml_backend_cuda_device_get_props(ggml_backend_dev_t dev, ggml_backend_dev_props * props) { + props->name = ggml_backend_cuda_device_get_name(dev); + props->description = ggml_backend_cuda_device_get_description(dev); ++ props->uuid = ggml_backend_cuda_device_get_uuid(dev); + props->type = ggml_backend_cuda_device_get_type(dev); + ggml_backend_cuda_device_get_memory(dev, &props->memory_free, &props->memory_total); + +@@ -3458,6 +3465,32 @@ ggml_backend_reg_t ggml_backend_cuda_reg() { + CUDA_CHECK(cudaGetDeviceProperties(&prop, i)); + dev_ctx->description = prop.name; + ++ #if !defined(GGML_USE_HIP) ++ char uuid[64]; ++ snprintf(uuid, sizeof(uuid), ++ "GPU-%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x", ++ (unsigned char)prop.uuid.bytes[0], ++ (unsigned char)prop.uuid.bytes[1], ++ (unsigned char)prop.uuid.bytes[2], ++ (unsigned char)prop.uuid.bytes[3], ++ (unsigned char)prop.uuid.bytes[4], ++ (unsigned char)prop.uuid.bytes[5], ++ (unsigned char)prop.uuid.bytes[6], ++ (unsigned char)prop.uuid.bytes[7], ++ (unsigned char)prop.uuid.bytes[8], ++ (unsigned char)prop.uuid.bytes[9], ++ (unsigned char)prop.uuid.bytes[10], ++ (unsigned char)prop.uuid.bytes[11], ++ (unsigned char)prop.uuid.bytes[12], ++ (unsigned char)prop.uuid.bytes[13], ++ (unsigned char)prop.uuid.bytes[14], ++ (unsigned char)prop.uuid.bytes[15] ++ ); ++ dev_ctx->uuid = uuid; ++ #else ++ dev_ctx->uuid = "GPU-" + std::string(prop.uuid.bytes, 16); ++ #endif ++ + ggml_backend_dev_t dev = new ggml_backend_device { + /* .iface = */ ggml_backend_cuda_device_interface, + /* .reg = */ ®, +diff --git a/ggml/src/ggml-metal/ggml-metal.m b/ggml/src/ggml-metal/ggml-metal.m +index 1b56f858..ee4f2dcb 100644 +--- a/ggml/src/ggml-metal/ggml-metal.m ++++ b/ggml/src/ggml-metal/ggml-metal.m +@@ -5703,6 +5703,7 @@ static enum ggml_backend_dev_type ggml_backend_metal_device_get_type(ggml_backen + static void ggml_backend_metal_device_get_props(ggml_backend_dev_t dev, struct ggml_backend_dev_props * props) { + props->name = ggml_backend_metal_device_get_name(dev); + props->description = ggml_backend_metal_device_get_description(dev); ++ props->uuid = "0"; + props->type = ggml_backend_metal_device_get_type(dev); + ggml_backend_metal_device_get_memory(dev, &props->memory_free, &props->memory_total); + props->caps = (struct ggml_backend_dev_caps) { diff --git a/ml/backend.go b/ml/backend.go index 65f169486..2df6c8923 100644 --- a/ml/backend.go +++ b/ml/backend.go @@ -124,6 +124,10 @@ type DeviceMemory struct { // may not be persistent across instances of the runner. Name string + // UUID is a unique persistent identifier for the device for matching + // with system management libraries + UUID string + // Weights is the per-layer memory needed for the model weights. Weights []Memory @@ -152,6 +156,10 @@ func (m DeviceMemory) LogValue() slog.Value { attrs = append(attrs, slog.Any("Graph", m.Graph)) } + if len(attrs) > 0 && m.UUID != "" { + attrs = append([]slog.Attr{slog.String("UUID", m.UUID)}, attrs...) + } + return slog.GroupValue(attrs...) } diff --git a/ml/backend/ggml/ggml.go b/ml/backend/ggml/ggml.go index 76172ae1a..5a9fe67e5 100644 --- a/ml/backend/ggml/ggml.go +++ b/ml/backend/ggml/ggml.go @@ -136,6 +136,9 @@ func New(modelPath string, params ml.BackendParams) (ml.Backend, error) { } requiredMemory.CPU.Name = C.GoString(C.ggml_backend_dev_name(cpuDeviceBufferType.d)) + var props C.struct_ggml_backend_dev_props + C.ggml_backend_dev_get_props(cpuDeviceBufferType.d, &props) + requiredMemory.CPU.UUID = C.GoString(props.uuid) requiredMemory.CPU.Weights = make([]ml.Memory, blocks+1) requiredMemory.CPU.Cache = make([]ml.Memory, blocks+1) @@ -150,6 +153,9 @@ func New(modelPath string, params ml.BackendParams) (ml.Backend, error) { }) btDeviceMemory[bt] = &requiredMemory.GPUs[i] requiredMemory.GPUs[i].Name = C.GoString(C.ggml_backend_dev_name(d)) + var props C.struct_ggml_backend_dev_props + C.ggml_backend_dev_get_props(d, &props) + requiredMemory.GPUs[i].UUID = C.GoString(props.uuid) requiredMemory.GPUs[i].Weights = make([]ml.Memory, blocks+1) requiredMemory.GPUs[i].Cache = make([]ml.Memory, blocks+1) } diff --git a/ml/backend/ggml/ggml/include/ggml-backend.h b/ml/backend/ggml/ggml/include/ggml-backend.h index 74e467163..a880df33e 100644 --- a/ml/backend/ggml/ggml/include/ggml-backend.h +++ b/ml/backend/ggml/ggml/include/ggml-backend.h @@ -152,6 +152,7 @@ extern "C" { struct ggml_backend_dev_props { const char * name; const char * description; + const char * uuid; size_t memory_free; size_t memory_total; enum ggml_backend_dev_type type; diff --git a/ml/backend/ggml/ggml/src/ggml-cuda/ggml-cuda.cu b/ml/backend/ggml/ggml/src/ggml-cuda/ggml-cuda.cu index cb0d8528d..4c8291532 100644 --- a/ml/backend/ggml/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ml/backend/ggml/ggml/src/ggml-cuda/ggml-cuda.cu @@ -2884,6 +2884,7 @@ struct ggml_backend_cuda_device_context { int device; std::string name; std::string description; + std::string uuid; }; static const char * ggml_backend_cuda_device_get_name(ggml_backend_dev_t dev) { @@ -2896,6 +2897,11 @@ static const char * ggml_backend_cuda_device_get_description(ggml_backend_dev_t return ctx->description.c_str(); } +static const char * ggml_backend_cuda_device_get_uuid(ggml_backend_dev_t dev) { + ggml_backend_cuda_device_context * ctx = (ggml_backend_cuda_device_context *)dev->context; + return ctx->uuid.c_str(); +} + static void ggml_backend_cuda_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) { ggml_backend_cuda_device_context * ctx = (ggml_backend_cuda_device_context *)dev->context; ggml_cuda_set_device(ctx->device); @@ -2910,6 +2916,7 @@ static enum ggml_backend_dev_type ggml_backend_cuda_device_get_type(ggml_backend static void ggml_backend_cuda_device_get_props(ggml_backend_dev_t dev, ggml_backend_dev_props * props) { props->name = ggml_backend_cuda_device_get_name(dev); props->description = ggml_backend_cuda_device_get_description(dev); + props->uuid = ggml_backend_cuda_device_get_uuid(dev); props->type = ggml_backend_cuda_device_get_type(dev); ggml_backend_cuda_device_get_memory(dev, &props->memory_free, &props->memory_total); @@ -3458,6 +3465,32 @@ ggml_backend_reg_t ggml_backend_cuda_reg() { CUDA_CHECK(cudaGetDeviceProperties(&prop, i)); dev_ctx->description = prop.name; + #if !defined(GGML_USE_HIP) + char uuid[64]; + snprintf(uuid, sizeof(uuid), + "GPU-%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x", + (unsigned char)prop.uuid.bytes[0], + (unsigned char)prop.uuid.bytes[1], + (unsigned char)prop.uuid.bytes[2], + (unsigned char)prop.uuid.bytes[3], + (unsigned char)prop.uuid.bytes[4], + (unsigned char)prop.uuid.bytes[5], + (unsigned char)prop.uuid.bytes[6], + (unsigned char)prop.uuid.bytes[7], + (unsigned char)prop.uuid.bytes[8], + (unsigned char)prop.uuid.bytes[9], + (unsigned char)prop.uuid.bytes[10], + (unsigned char)prop.uuid.bytes[11], + (unsigned char)prop.uuid.bytes[12], + (unsigned char)prop.uuid.bytes[13], + (unsigned char)prop.uuid.bytes[14], + (unsigned char)prop.uuid.bytes[15] + ); + dev_ctx->uuid = uuid; + #else + dev_ctx->uuid = "GPU-" + std::string(prop.uuid.bytes, 16); + #endif + ggml_backend_dev_t dev = new ggml_backend_device { /* .iface = */ ggml_backend_cuda_device_interface, /* .reg = */ ®, diff --git a/ml/backend/ggml/ggml/src/ggml-metal/ggml-metal.m b/ml/backend/ggml/ggml/src/ggml-metal/ggml-metal.m index 1b56f858c..ee4f2dcb0 100644 --- a/ml/backend/ggml/ggml/src/ggml-metal/ggml-metal.m +++ b/ml/backend/ggml/ggml/src/ggml-metal/ggml-metal.m @@ -5703,6 +5703,7 @@ static enum ggml_backend_dev_type ggml_backend_metal_device_get_type(ggml_backen static void ggml_backend_metal_device_get_props(ggml_backend_dev_t dev, struct ggml_backend_dev_props * props) { props->name = ggml_backend_metal_device_get_name(dev); props->description = ggml_backend_metal_device_get_description(dev); + props->uuid = "0"; props->type = ggml_backend_metal_device_get_type(dev); ggml_backend_metal_device_get_memory(dev, &props->memory_free, &props->memory_total); props->caps = (struct ggml_backend_dev_caps) { From 8bcb3125c1b416b43aa431b2b3b105d933eca697 Mon Sep 17 00:00:00 2001 From: Jeffrey Morgan Date: Wed, 18 Jun 2025 12:58:50 -0700 Subject: [PATCH 54/54] benchmark: remove unused benchmark test (#11120) Removes a test under benchmark/ that is unused --- benchmark/server_benchmark_test.go | 178 ----------------------------- docs/benchmark.md | 59 ---------- 2 files changed, 237 deletions(-) delete mode 100644 benchmark/server_benchmark_test.go delete mode 100644 docs/benchmark.md diff --git a/benchmark/server_benchmark_test.go b/benchmark/server_benchmark_test.go deleted file mode 100644 index 4a3c46cda..000000000 --- a/benchmark/server_benchmark_test.go +++ /dev/null @@ -1,178 +0,0 @@ -package benchmark - -import ( - "context" - "flag" - "fmt" - "testing" - "time" - - "github.com/ollama/ollama/api" -) - -// Command line flags -var modelFlag string - -func init() { - flag.StringVar(&modelFlag, "m", "", "Name of the model to benchmark") - flag.Lookup("m").DefValue = "model" -} - -// modelName returns the model name from flags, failing the test if not set -func modelName(b *testing.B) string { - if modelFlag == "" { - b.Fatal("Error: -m flag is required for benchmark tests") - } - return modelFlag -} - -type TestCase struct { - name string - prompt string - maxTokens int -} - -// runGenerateBenchmark contains the common generate and metrics logic -func runGenerateBenchmark(b *testing.B, ctx context.Context, client *api.Client, req *api.GenerateRequest) { - start := time.Now() - var ttft time.Duration - var metrics api.Metrics - - err := client.Generate(ctx, req, func(resp api.GenerateResponse) error { - if ttft == 0 && resp.Response != "" { - ttft = time.Since(start) - } - if resp.Done { - metrics = resp.Metrics - } - return nil - }) - - // Report custom metrics as part of the benchmark results - b.ReportMetric(float64(ttft.Milliseconds()), "ttft_ms") - b.ReportMetric(float64(metrics.LoadDuration.Milliseconds()), "load_ms") - - // Token throughput metrics - promptThroughput := float64(metrics.PromptEvalCount) / metrics.PromptEvalDuration.Seconds() - genThroughput := float64(metrics.EvalCount) / metrics.EvalDuration.Seconds() - b.ReportMetric(promptThroughput, "prompt_tok/s") - b.ReportMetric(genThroughput, "gen_tok/s") - - // Token counts - b.ReportMetric(float64(metrics.PromptEvalCount), "prompt_tokens") - b.ReportMetric(float64(metrics.EvalCount), "gen_tokens") - if err != nil { - b.Fatal(err) - } -} - -// BenchmarkColdStart runs benchmarks with model loading from cold state -func BenchmarkColdStart(b *testing.B) { - client := setup(b) - tests := []TestCase{ - {"short_prompt", "Write a long story", 100}, - {"medium_prompt", "Write a detailed economic analysis", 500}, - {"long_prompt", "Write a comprehensive AI research paper", 1000}, - } - m := modelName(b) - - for _, tt := range tests { - b.Run(fmt.Sprintf("%s/cold/%s", m, tt.name), func(b *testing.B) { - ctx := b.Context() - - // Set number of tokens as our throughput metric - b.SetBytes(int64(tt.maxTokens)) - - for b.Loop() { - b.StopTimer() - // Ensure model is unloaded before each iteration - unload(client, m, b) - b.StartTimer() - - req := &api.GenerateRequest{ - Model: m, - Prompt: tt.prompt, - Options: map[string]any{"num_predict": tt.maxTokens, "temperature": 0.1}, - } - - runGenerateBenchmark(b, ctx, client, req) - } - }) - } -} - -// BenchmarkWarmStart runs benchmarks with pre-loaded model -func BenchmarkWarmStart(b *testing.B) { - client := setup(b) - tests := []TestCase{ - {"short_prompt", "Write a long story", 100}, - {"medium_prompt", "Write a detailed economic analysis", 500}, - {"long_prompt", "Write a comprehensive AI research paper", 1000}, - } - m := modelName(b) - - for _, tt := range tests { - b.Run(fmt.Sprintf("%s/warm/%s", m, tt.name), func(b *testing.B) { - ctx := b.Context() - - // Pre-warm the model - warmup(client, m, tt.prompt, b) - - // Set number of tokens as our throughput metric - b.SetBytes(int64(tt.maxTokens)) - - for b.Loop() { - req := &api.GenerateRequest{ - Model: m, - Prompt: tt.prompt, - Options: map[string]any{"num_predict": tt.maxTokens, "temperature": 0.1}, - } - - runGenerateBenchmark(b, ctx, client, req) - } - }) - } -} - -// setup verifies server and model availability -func setup(b *testing.B) *api.Client { - client, err := api.ClientFromEnvironment() - if err != nil { - b.Fatal(err) - } - if _, err := client.Show(b.Context(), &api.ShowRequest{Model: modelName(b)}); err != nil { - b.Fatalf("Model unavailable: %v", err) - } - - return client -} - -// warmup ensures the model is loaded and warmed up -func warmup(client *api.Client, model string, prompt string, b *testing.B) { - for range 3 { - err := client.Generate( - context.Background(), - &api.GenerateRequest{ - Model: model, - Prompt: prompt, - Options: map[string]any{"num_predict": 50, "temperature": 0.1}, - }, - func(api.GenerateResponse) error { return nil }, - ) - if err != nil { - b.Logf("Error during model warm-up: %v", err) - } - } -} - -// unload forces model unloading using KeepAlive: 0 parameter -func unload(client *api.Client, model string, b *testing.B) { - req := &api.GenerateRequest{ - Model: model, - KeepAlive: &api.Duration{Duration: 0}, - } - if err := client.Generate(context.Background(), req, func(api.GenerateResponse) error { return nil }); err != nil { - b.Logf("Unload error: %v", err) - } - time.Sleep(1 * time.Second) -} diff --git a/docs/benchmark.md b/docs/benchmark.md deleted file mode 100644 index a7bed8083..000000000 --- a/docs/benchmark.md +++ /dev/null @@ -1,59 +0,0 @@ -# Benchmark - -Go benchmark tests that measure end-to-end performance of a running Ollama server. Run these tests to evaluate model inference performance on your hardware and measure the impact of code changes. - -## When to use - -Run these benchmarks when: -- Making changes to the model inference engine -- Modifying model loading/unloading logic -- Changing prompt processing or token generation code -- Implementing a new model architecture -- Testing performance across different hardware setups - -## Prerequisites -- Ollama server running locally with `ollama serve` on `127.0.0.1:11434` -## Usage and Examples - ->[!NOTE] ->All commands must be run from the root directory of the Ollama project. - -Basic syntax: -```bash -go test -bench=. ./benchmark/... -m $MODEL_NAME -``` - -Required flags: -- `-bench=.`: Run all benchmarks -- `-m`: Model name to benchmark - -Optional flags: -- `-count N`: Number of times to run the benchmark (useful for statistical analysis) -- `-timeout T`: Maximum time for the benchmark to run (e.g. "10m" for 10 minutes) - -Common usage patterns: - -Single benchmark run with a model specified: -```bash -go test -bench=. ./benchmark/... -m llama3.3 -``` - -## Output metrics - -The benchmark reports several key metrics: - -- `gen_tok/s`: Generated tokens per second -- `prompt_tok/s`: Prompt processing tokens per second -- `ttft_ms`: Time to first token in milliseconds -- `load_ms`: Model load time in milliseconds -- `gen_tokens`: Total tokens generated -- `prompt_tokens`: Total prompt tokens processed - -Each benchmark runs two scenarios: -- Cold start: Model is loaded from disk for each test -- Warm start: Model is pre-loaded in memory - -Three prompt lengths are tested for each scenario: -- Short prompt (100 tokens) -- Medium prompt (500 tokens) -- Long prompt (1000 tokens)