diff --git a/core/src/main/java/org/springframework/security/core/userdetails/User.java b/core/src/main/java/org/springframework/security/core/userdetails/User.java
index ce8e318c24..ebbf61132c 100644
--- a/core/src/main/java/org/springframework/security/core/userdetails/User.java
+++ b/core/src/main/java/org/springframework/security/core/userdetails/User.java
@@ -323,8 +323,7 @@ public class User implements UserDetails, CredentialsContainer {
 		 */
 		public UserBuilder password(String password) {
 			Assert.notNull(password, "password cannot be null");
-			String encodedPassword = this.passwordEncoder.apply(password);
-			this.password = encodedPassword;
+			this.password = password;
 			return this;
 		}
 
@@ -339,7 +338,7 @@ public class User implements UserDetails, CredentialsContainer {
 		public UserBuilder passwordEncoder(Function<String,String> encoder) {
 			Assert.notNull(encoder, "encoder cannot be null");
 			this.passwordEncoder = encoder;
-			return this.password == null ? this : password(this.password);
+			return this;
 		}
 
 		/**
@@ -467,7 +466,8 @@ public class User implements UserDetails, CredentialsContainer {
 		}
 
 		public UserDetails build() {
-			return new User(username, password, !disabled, !accountExpired,
+			String encodedPassword = this.passwordEncoder.apply(password);
+			return new User(username, encodedPassword, !disabled, !accountExpired,
 					!credentialsExpired, !accountLocked, authorities);
 		}
 	}
diff --git a/core/src/test/java/org/springframework/security/core/userdetails/UserTests.java b/core/src/test/java/org/springframework/security/core/userdetails/UserTests.java
index e763c61f7c..95825bdd8f 100644
--- a/core/src/test/java/org/springframework/security/core/userdetails/UserTests.java
+++ b/core/src/test/java/org/springframework/security/core/userdetails/UserTests.java
@@ -23,6 +23,7 @@ import java.io.ObjectOutputStream;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
+import java.util.function.Function;
 
 import org.junit.Test;
 import org.springframework.security.core.GrantedAuthority;
@@ -210,4 +211,17 @@ public class UserTests {
 
 		assertThat(withEncodedPassword.getPassword()).isEqualTo("passwordencoded");
 	}
+
+	@Test
+	public void withUsernameWhenPasswordAndPasswordEncoderTwiceThenEncodesOnce() {
+		Function<String, String> encoder = p -> p + "encoded";
+		UserDetails withEncodedPassword = User.withUsername("user")
+			.passwordEncoder(encoder)
+			.password("password")
+			.passwordEncoder(encoder)
+			.roles("USER")
+			.build();
+
+		assertThat(withEncodedPassword.getPassword()).isEqualTo("passwordencoded");
+	}
 }