diff --git a/transformers/llm/engine/src/sampler.cpp b/transformers/llm/engine/src/sampler.cpp
index e1ad3e8a..ef0ac2a1 100644
--- a/transformers/llm/engine/src/sampler.cpp
+++ b/transformers/llm/engine/src/sampler.cpp
@@ -232,6 +232,17 @@ void Sampler::SamplerConfig::configMixed(std::shared_ptr<LlmConfig> llmConfig) {
         this->configSampler(samplerName, llmConfig);
         // std::cout << samplerName << " " << std::flush;
     }
+    for (int i=1; i<mixedSamplers.size(); ++i) {
+        // "penalty" can only locate at the first position
+        if (mixedSamplers[i]=="penalty") {
+            mixedSamplers.erase(mixedSamplers.begin()+i); 
+            i--;
+            if (mixedSamplers[0]!="penalty") {
+                mixedSamplers.insert(mixedSamplers.begin(), "penalty");
+                i++;
+            }
+        }
+    }
     // std::cout << std::endl;
     // set select type
     // the final sampler select the token