diff --git a/mysql-test/r/ctype_ucs.result b/mysql-test/r/ctype_ucs.result
index c38a03f76a802..7a93c5524ef35 100644
--- a/mysql-test/r/ctype_ucs.result
+++ b/mysql-test/r/ctype_ucs.result
@@ -4357,5 +4357,15 @@ SELECT CHAR_LENGTH(TRIM(BOTH 0x00 FROM _ucs2 0x0061));
 CHAR_LENGTH(TRIM(BOTH 0x00 FROM _ucs2 0x0061))
 1
 #
+# MDEV-11685: sql_mode can't be set with non-ascii connection charset
+#
+SET character_set_connection=ucs2;
+SET sql_mode='NO_ENGINE_SUBSTITUTION';
+SELECT @@sql_mode;
+@@sql_mode
+NO_ENGINE_SUBSTITUTION
+SET sql_mode=DEFAULT;
+SET NAMES utf8;
+#
 # End of 5.5 tests
 #
diff --git a/mysql-test/r/ctype_utf16.result b/mysql-test/r/ctype_utf16.result
index bfb2d6a498cff..0846811e2e639 100644
--- a/mysql-test/r/ctype_utf16.result
+++ b/mysql-test/r/ctype_utf16.result
@@ -1207,5 +1207,15 @@ DO LPAD(_utf16 0x0061 COLLATE utf16_unicode_ci, 10000, 0x0061DE989999);
 Warnings:
 Warning	1300	Invalid utf16 character string: 'DE9899'
 #
+# MDEV-11685: sql_mode can't be set with non-ascii connection charset
+#
+SET character_set_connection=utf16;
+SET sql_mode='NO_ENGINE_SUBSTITUTION';
+SELECT @@sql_mode;
+@@sql_mode
+NO_ENGINE_SUBSTITUTION
+SET sql_mode=DEFAULT;
+SET NAMES utf8;
+#
 # End of 5.5 tests
 #
diff --git a/mysql-test/r/ctype_utf32.result b/mysql-test/r/ctype_utf32.result
index ae55f2c101ed9..9b062f9480f07 100644
--- a/mysql-test/r/ctype_utf32.result
+++ b/mysql-test/r/ctype_utf32.result
@@ -1273,5 +1273,15 @@ select hex(lower(cast(0xffff0000 as char character set utf32))) as c;
 c
 FFFF0000
 #
+# MDEV-11685: sql_mode can't be set with non-ascii connection charset
+#
+SET character_set_connection=utf32;
+SET sql_mode='NO_ENGINE_SUBSTITUTION';
+SELECT @@sql_mode;
+@@sql_mode
+NO_ENGINE_SUBSTITUTION
+SET sql_mode=DEFAULT;
+SET NAMES utf8;
+#
 # End of 5.5 tests
 #
diff --git a/mysql-test/t/ctype_ucs.test b/mysql-test/t/ctype_ucs.test
index 7fd3768aa5f64..d94c9ae62ac6e 100644
--- a/mysql-test/t/ctype_ucs.test
+++ b/mysql-test/t/ctype_ucs.test
@@ -853,6 +853,14 @@ SELECT CHAR_LENGTH(TRIM(BOTH 0x0001 FROM _ucs2 0x0061));
 SELECT CHAR_LENGTH(TRIM(BOTH 0x61 FROM _ucs2 0x0061));
 SELECT CHAR_LENGTH(TRIM(BOTH 0x00 FROM _ucs2 0x0061));
 
+--echo #
+--echo # MDEV-11685: sql_mode can't be set with non-ascii connection charset
+--echo #
+SET character_set_connection=ucs2;
+SET sql_mode='NO_ENGINE_SUBSTITUTION';
+SELECT @@sql_mode;
+SET sql_mode=DEFAULT;
+SET NAMES utf8;
 
 --echo #
 --echo # End of 5.5 tests
diff --git a/mysql-test/t/ctype_utf16.test b/mysql-test/t/ctype_utf16.test
index 3f9e5eece2170..ab87c7da0d8ba 100644
--- a/mysql-test/t/ctype_utf16.test
+++ b/mysql-test/t/ctype_utf16.test
@@ -792,6 +792,15 @@ SELECT 'a' AS id, REPEAT('bla bla', 100) AS body) t1;
 DO RPAD(_utf16 0x0061 COLLATE utf16_unicode_ci, 10000, 0x0061DE989999);
 DO LPAD(_utf16 0x0061 COLLATE utf16_unicode_ci, 10000, 0x0061DE989999);
 
+--echo #
+--echo # MDEV-11685: sql_mode can't be set with non-ascii connection charset
+--echo #
+SET character_set_connection=utf16;
+SET sql_mode='NO_ENGINE_SUBSTITUTION';
+SELECT @@sql_mode;
+SET sql_mode=DEFAULT;
+SET NAMES utf8;
+
 --echo #
 --echo # End of 5.5 tests
 --echo #
diff --git a/mysql-test/t/ctype_utf32.test b/mysql-test/t/ctype_utf32.test
index 8cbb8e2e55e19..2b3d3b3bdc59d 100644
--- a/mysql-test/t/ctype_utf32.test
+++ b/mysql-test/t/ctype_utf32.test
@@ -881,6 +881,15 @@ SELECT CHAR_LENGTH(TRIM(BOTH 0x00 FROM _utf32 0x00000061));
 #
 select hex(lower(cast(0xffff0000 as char character set utf32))) as c;
 
+--echo #
+--echo # MDEV-11685: sql_mode can't be set with non-ascii connection charset
+--echo #
+SET character_set_connection=utf32;
+SET sql_mode='NO_ENGINE_SUBSTITUTION';
+SELECT @@sql_mode;
+SET sql_mode=DEFAULT;
+SET NAMES utf8;
+
 --echo #
 --echo # End of 5.5 tests
 --echo #
diff --git a/sql/sys_vars.h b/sql/sys_vars.h
index 3cbd24f1c89d8..dbe27ab107eca 100644
--- a/sql/sys_vars.h
+++ b/sql/sys_vars.h
@@ -1141,7 +1141,7 @@ class Sys_var_set: public Sys_var_typelib
 
     if (var->value->result_type() == STRING_RESULT)
     {
-      if (!(res=var->value->val_str(&str)))
+      if (!(res=var->value->val_str_ascii(&str)))
         return true;
       else
       {