patch 8.0.0250: virtcol() does not work well for multi-byte characters

Problem:    When virtcol() gets a column that is not the first byte of a
            multi-byte character the result is unpredictable. (Christian
            Ludwig)
Solution:   Correct the column to the first byte of a multi-byte character.
            Change the utf-8 test to new style.

.travis.yml

@@ -18,13 +18,16 @@ env:
     # Mac OSX build
   - BUILD=yes TEST=test COVERAGE=no FEATURES=huge SHADOWOPT= SRCDIR=./src CHECK_AUTOCONF=no
     "CONFOPT='--enable-perlinterp --enable-pythoninterp --enable-rubyinterp --enable-luainterp'"
+    # ASAN build
+  - BUILD=yes TEST=test SANITIZER_CFLAGS="-g -O1 -fsanitize=address -fno-omit-frame-pointer" FEATURES=huge SRCDIR=./src CHECK_AUTOCONF=no
+    "CONFOPT='--enable-perlinterp --enable-pythoninterp --enable-rubyinterp --enable-luainterp'"
 
 sudo: false
 
 git:
-  depth: 1
+  depth: 10
 
-# instead of a 6*2*2 matrix (2*os + 2*compiler + 6*env),
+# instead of a 2*2*8 matrix (2*os + 2*compiler + 8*env),
 # exclude some builds on mac os x and linux
 # linux: 2*compiler + 5*env + mac: 2*compiler + 2*env
 matrix:
@@ -38,6 +41,13 @@ matrix:
     - os: osx
       env: BUILD=yes TEST=scripttests COVERAGE=yes CFLAGS=--coverage LDFLAGS=--coverage FEATURES=huge SHADOWOPT= SRCDIR=./src CHECK_AUTOCONF=no
             "CONFOPT='--enable-perlinterp --enable-pythoninterp --enable-python3interp --enable-rubyinterp --enable-luainterp'"
+    - os: osx
+      env: BUILD=yes TEST=test SANITIZER_CFLAGS="-g -O1 -fsanitize=address -fno-omit-frame-pointer" FEATURES=huge SRCDIR=./src CHECK_AUTOCONF=no
+            "CONFOPT='--enable-perlinterp --enable-pythoninterp --enable-rubyinterp --enable-luainterp'"
+    - os: linux
+      compiler: gcc
+      env: BUILD=yes TEST=test SANITIZER_CFLAGS="-g -O1 -fsanitize=address -fno-omit-frame-pointer" FEATURES=huge SRCDIR=./src CHECK_AUTOCONF=no
+            "CONFOPT='--enable-perlinterp --enable-pythoninterp --enable-rubyinterp --enable-luainterp'"
     - os: linux
       compiler: clang
       env: BUILD=no TEST=unittests COVERAGE=yes CFLAGS=--coverage LDFLAGS=--coverage FEATURES=huge SHADOWOPT= SRCDIR=./src CHECK_AUTOCONF=yes

src/Makefile

@@ -2058,7 +2058,6 @@ test1 \
 	test_listlbr \
 	test_listlbr_utf8 \
 	test_search_mbyte \
-	test_utf8 \
 	test_wordcount \
 	test3 test4 test5 test6 test7 test8 test9 \
 	test11 test12 test14 test15 test17 test18 test19 \
@@ -2183,6 +2182,7 @@ test_arglist \
 	test_undo \
 	test_unlet \
 	test_usercommands \
+	test_utf8 \
 	test_viminfo \
 	test_viml \
 	test_visual \

src/charset.c

@@ -1296,7 +1296,14 @@ getvcol(
     if (pos->col == MAXCOL)
 	posptr = NULL;  /* continue until the NUL */
     else
+    {
 	posptr = ptr + pos->col;
+#ifdef FEAT_MBYTE
+	if (has_mbyte)
+	    /* always start on the first byte */
+	    posptr -= (*mb_head_off)(line, posptr);
+#endif
+    }
 
     /*
      * This function is used very often, do some speed optimizations.

src/edit.c

@@ -2821,6 +2821,7 @@ set_completion(colnr_T startcol, list_T *list)
     if (ctrl_x_mode != 0)
 	ins_compl_prep(' ');
     ins_compl_clear();
+    ins_compl_free();
 
     compl_direction = FORWARD;
     if (startcol > curwin->w_cursor.col)
@@ -9463,12 +9464,17 @@ bracketed_paste(paste_mode_T mode, int drop, garray_T *gap)
     char_u	*end = find_termcode((char_u *)"PE");
     int		ret_char = -1;
     int		save_allow_keys = allow_keys;
+    int		save_paste = p_paste;
+    int		save_ai = curbuf->b_p_ai;
 
     /* If the end code is too long we can't detect it, read everything. */
     if (STRLEN(end) >= NUMBUFLEN)
 	end = NULL;
     ++no_mapping;
     allow_keys = 0;
+    p_paste = TRUE;
+    curbuf->b_p_ai = FALSE;
+
     for (;;)
     {
 	/* When the end is not defined read everything. */
@@ -9534,8 +9540,11 @@ bracketed_paste(paste_mode_T mode, int drop, garray_T *gap)
 	}
 	idx = 0;
     }
+
     --no_mapping;
     allow_keys = save_allow_keys;
+    p_paste = save_paste;
+    curbuf->b_p_ai = save_ai;
 
     return ret_char;
 }

src/evalfunc.c

@@ -12503,39 +12503,8 @@ f_timer_stopall(typval_T *argvars UNUSED, typval_T *rettv UNUSED)
     static void
 f_tolower(typval_T *argvars, typval_T *rettv)
 {
-    char_u	*p;
-
-    p = vim_strsave(get_tv_string(&argvars[0]));
     rettv->v_type = VAR_STRING;
-    rettv->vval.v_string = p;
-
-    if (p != NULL)
-	while (*p != NUL)
-	{
-#ifdef FEAT_MBYTE
-	    int		l;
-
-	    if (enc_utf8)
-	    {
-		int c, lc;
-
-		c = utf_ptr2char(p);
-		lc = utf_tolower(c);
-		l = utf_ptr2len(p);
-		/* TODO: reallocate string when byte count changes. */
-		if (utf_char2len(lc) == l)
-		    utf_char2bytes(lc, p);
-		p += l;
-	    }
-	    else if (has_mbyte && (l = (*mb_ptr2len)(p)) > 1)
-		p += l;		/* skip multi-byte character */
-	    else
-#endif
-	    {
-		*p = TOLOWER_LOC(*p); /* note that tolower() can be a macro */
-		++p;
-	    }
-	}
+    rettv->vval.v_string = strlow_save(get_tv_string(&argvars[0]));
 }
 
 /*

src/misc2.c

@@ -1602,7 +1602,10 @@ strup_save(char_u *orig)
 		{
 		    s = alloc((unsigned)STRLEN(res) + 1 + newl - l);
 		    if (s == NULL)
-			break;
+		    {
+			vim_free(res);
+			return NULL;
+		    }
 		    mch_memmove(s, res, p - res);
 		    STRCPY(s + (p - res) + newl, p + l);
 		    p = s + (p - res);
@@ -1625,6 +1628,69 @@ strup_save(char_u *orig)
 
     return res;
 }
+
+/*
+ * Make string "s" all lower-case and return it in allocated memory.
+ * Handles multi-byte characters as well as possible.
+ * Returns NULL when out of memory.
+ */
+    char_u *
+strlow_save(char_u *orig)
+{
+    char_u	*p;
+    char_u	*res;
+
+    res = p = vim_strsave(orig);
+
+    if (res != NULL)
+	while (*p != NUL)
+	{
+# ifdef FEAT_MBYTE
+	    int		l;
+
+	    if (enc_utf8)
+	    {
+		int	c, lc;
+		int	newl;
+		char_u	*s;
+
+		c = utf_ptr2char(p);
+		lc = utf_tolower(c);
+
+		/* Reallocate string when byte count changes.  This is rare,
+		 * thus it's OK to do another malloc()/free(). */
+		l = utf_ptr2len(p);
+		newl = utf_char2len(lc);
+		if (newl != l)
+		{
+		    s = alloc((unsigned)STRLEN(res) + 1 + newl - l);
+		    if (s == NULL)
+		    {
+			vim_free(res);
+			return NULL;
+		    }
+		    mch_memmove(s, res, p - res);
+		    STRCPY(s + (p - res) + newl, p + l);
+		    p = s + (p - res);
+		    vim_free(res);
+		    res = s;
+		}
+
+		utf_char2bytes(lc, p);
+		p += newl;
+	    }
+	    else if (has_mbyte && (l = (*mb_ptr2len)(p)) > 1)
+		p += l;		/* skip multi-byte character */
+	    else
+# endif
+	    {
+		*p = TOLOWER_LOC(*p); /* note that tolower() can be a macro */
+		p++;
+	    }
+	}
+
+    return res;
+}
 #endif
 
 /*
@@ -1653,7 +1719,7 @@ vim_strncpy(char_u *to, char_u *from, size_t len)
 
 /*
  * Like strcat(), but make sure the result fits in "tosize" bytes and is
- * always NUL terminated.
+ * always NUL terminated. "from" and "to" may overlap.
  */
     void
 vim_strcat(char_u *to, char_u *from, size_t tosize)
@@ -1667,7 +1733,7 @@ vim_strcat(char_u *to, char_u *from, size_t tosize)
 	to[tosize - 1] = NUL;
     }
     else
-	STRCPY(to + tolen, from);
+	mch_memmove(to + tolen, from, fromlen + 1);
 }
 
 /*
@@ -1740,34 +1806,6 @@ vim_memset(void *ptr, int c, size_t size)
 }
 #endif
 
-/* skipped when generating prototypes, the prototype is in vim.h */
-#ifdef VIM_MEMMOVE
-/*
- * Version of memmove() that handles overlapping source and destination.
- * For systems that don't have a function that is guaranteed to do that (SYSV).
- */
-    void
-mch_memmove(void *src_arg, void *dst_arg, size_t len)
-{
-    /*
-     * A void doesn't have a size, we use char pointers.
-     */
-    char *dst = dst_arg, *src = src_arg;
-
-					/* overlap, copy backwards */
-    if (dst > src && dst < src + len)
-    {
-	src += len;
-	dst += len;
-	while (len-- > 0)
-	    *--dst = *--src;
-    }
-    else				/* copy forwards */
-	while (len-- > 0)
-	    *dst++ = *src++;
-}
-#endif
-
 #if (!defined(HAVE_STRCASECMP) && !defined(HAVE_STRICMP)) || defined(PROTO)
 /*
  * Compare two strings, ignoring case, using current locale.

src/option.c

@@ -6619,6 +6619,15 @@ did_set_string_option(
 	    mch_set_normal_colors();
 #endif
 	}
+	if (varp == &T_BE && termcap_active)
+	{
+	    if (*T_BE == NUL)
+		/* When clearing t_BE we assume the user no longer wants
+		 * bracketed paste, thus disable it by writing t_BD. */
+		out_str(T_BD);
+	    else
+		out_str(T_BE);
+	}
     }
 
 #ifdef FEAT_LINEBREAK

src/os_unix.h

@@ -423,21 +423,17 @@ typedef struct dsc$descriptor   DESC;
 # endif
 #endif
 
-/* memmove is not present on all systems, use memmove, bcopy, memcpy or our
- * own version */
-/* Some systems have (void *) arguments, some (char *). If we use (char *) it
+/* memmove() is not present on all systems, use memmove, bcopy or memcpy.
+ * Some systems have (void *) arguments, some (char *). If we use (char *) it
  * works for all */
-#ifdef USEMEMMOVE
+#if defined(USEMEMMOVE) || (!defined(USEBCOPY) && !defined(USEMEMCPY))
 # define mch_memmove(to, from, len) memmove((char *)(to), (char *)(from), len)
 #else
 # ifdef USEBCOPY
 #  define mch_memmove(to, from, len) bcopy((char *)(from), (char *)(to), len)
 # else
-#  ifdef USEMEMCPY
+    /* ifdef USEMEMCPY */
 #   define mch_memmove(to, from, len) memcpy((char *)(to), (char *)(from), len)
-#  else
-#   define VIM_MEMMOVE	    /* found in misc2.c */
-#  endif
 # endif
 #endif
 

src/po/Makefile

@@ -232,12 +232,13 @@ sk.cp1250.po: sk.po
 	iconv -f iso-8859-2 -t cp1250 sk.po | \
 		sed -e 's/charset=ISO-8859-2/charset=cp1250/' -e 's/# Original translations/# Generated from sk.po, DO NOT EDIT/' > sk.cp1250.po
 
-# Convert zh_CN.po to create zh_CN.cp936.po.
-# set 'charset' to gbk to avoid that msfmt generates a warning
-zh_CN.cp936.po: zh_CN.po
+# Convert zh_CN.UTF-8.po to create zh_CN.cp936.po.
+# Set 'charset' to gbk to avoid that msfmt generates a warning.
+# This used to convert from zh_CN.po, but that results in a conversion error.
+zh_CN.cp936.po: zh_CN.UTF-8.po
 	rm -f zh_CN.cp936.po
-	iconv -f gb2312 -t cp936 zh_CN.po | \
-		sed -e 's/charset=gb2312/charset=gbk/' -e 's/# Original translations/# Generated from zh_CN.po, DO NOT EDIT/' > zh_CN.cp936.po
+	iconv -f UTF-8 -t cp936 zh_CN.UTF-8.po | \
+		sed -e 's/charset=utf-8/charset=gbk/' -e 's/# Original translations/# Generated from zh_CN.po, DO NOT EDIT/' > zh_CN.cp936.po
 
 # Convert ko.UTF-8.po to create ko.po.
 ko.po: ko.UTF-8.po

src/proto/misc2.pro

@@ -40,6 +40,7 @@ char_u *vim_strsave_up(char_u *string);
 char_u *vim_strnsave_up(char_u *string, int len);
 void vim_strup(char_u *p);
 char_u *strup_save(char_u *orig);
+char_u *strlow_save(char_u *orig);
 void del_trailing_spaces(char_u *ptr);
 void vim_strncpy(char_u *to, char_u *from, size_t len);
 void vim_strcat(char_u *to, char_u *from, size_t tosize);

src/testdir/Make_all.mak

@@ -84,7 +84,6 @@ SCRIPTS_ALL = \
 	test_listchars.out \
 	test_listlbr.out \
 	test_search_mbyte.out \
-	test_utf8.out \
 	test_wordcount.out
 
 

src/testdir/test_alot_utf8.vim

@@ -9,3 +9,4 @@ source test_expr_utf8.vim
 source test_matchadd_conceal_utf8.vim
 source test_regexp_utf8.vim
 source test_source_utf8.vim
+source test_utf8.vim

src/testdir/test_functions.vim

@@ -16,3 +16,148 @@ func Test_str2nr()
   call assert_equal(123456789, str2nr('123456789'))
   call assert_equal(-123456789, str2nr('-123456789'))
 endfunc
+
+func Test_tolower()
+  call assert_equal("", tolower(""))
+
+  " Test with all printable ASCII characters.
+  call assert_equal(' !"#$%&''()*+,-./0123456789:;<=>?@abcdefghijklmnopqrstuvwxyz[\]^_`abcdefghijklmnopqrstuvwxyz{|}~',
+          \ tolower(' !"#$%&''()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~'))
+
+  if !has('multi_byte')
+    return
+  endif
+
+  " Test with a few uppercase diacritics.
+  call assert_equal("aàáâãäåāăąǎǟǡả", tolower("AÀÁÂÃÄÅĀĂĄǍǞǠẢ"))
+  call assert_equal("bḃḇ", tolower("BḂḆ"))
+  call assert_equal("cçćĉċč", tolower("CÇĆĈĊČ"))
+  call assert_equal("dďđḋḏḑ", tolower("DĎĐḊḎḐ"))
+  call assert_equal("eèéêëēĕėęěẻẽ", tolower("EÈÉÊËĒĔĖĘĚẺẼ"))
+  call assert_equal("fḟ ", tolower("FḞ "))
+  call assert_equal("gĝğġģǥǧǵḡ", tolower("GĜĞĠĢǤǦǴḠ"))
+  call assert_equal("hĥħḣḧḩ", tolower("HĤĦḢḦḨ"))
+  call assert_equal("iìíîïĩīĭįiǐỉ", tolower("IÌÍÎÏĨĪĬĮİǏỈ"))
+  call assert_equal("jĵ", tolower("JĴ"))
+  call assert_equal("kķǩḱḵ", tolower("KĶǨḰḴ"))
+  call assert_equal("lĺļľŀłḻ", tolower("LĹĻĽĿŁḺ"))
+  call assert_equal("mḿṁ", tolower("MḾṀ"))
+  call assert_equal("nñńņňṅṉ", tolower("NÑŃŅŇṄṈ"))
+  call assert_equal("oòóôõöøōŏőơǒǫǭỏ", tolower("OÒÓÔÕÖØŌŎŐƠǑǪǬỎ"))
+  call assert_equal("pṕṗ", tolower("PṔṖ"))
+  call assert_equal("q", tolower("Q"))
+  call assert_equal("rŕŗřṙṟ", tolower("RŔŖŘṘṞ"))
+  call assert_equal("sśŝşšṡ", tolower("SŚŜŞŠṠ"))
+  call assert_equal("tţťŧṫṯ", tolower("TŢŤŦṪṮ"))
+  call assert_equal("uùúûüũūŭůűųưǔủ", tolower("UÙÚÛÜŨŪŬŮŰŲƯǓỦ"))
+  call assert_equal("vṽ", tolower("VṼ"))
+  call assert_equal("wŵẁẃẅẇ", tolower("WŴẀẂẄẆ"))
+  call assert_equal("xẋẍ", tolower("XẊẌ"))
+  call assert_equal("yýŷÿẏỳỷỹ", tolower("YÝŶŸẎỲỶỸ"))
+  call assert_equal("zźżžƶẑẕ", tolower("ZŹŻŽƵẐẔ"))
+
+  " Test with a few lowercase diacritics, which should remain unchanged.
+  call assert_equal("aàáâãäåāăąǎǟǡả", tolower("aàáâãäåāăąǎǟǡả"))
+  call assert_equal("bḃḇ", tolower("bḃḇ"))
+  call assert_equal("cçćĉċč", tolower("cçćĉċč"))
+  call assert_equal("dďđḋḏḑ", tolower("dďđḋḏḑ"))
+  call assert_equal("eèéêëēĕėęěẻẽ", tolower("eèéêëēĕėęěẻẽ"))
+  call assert_equal("fḟ", tolower("fḟ"))
+  call assert_equal("gĝğġģǥǧǵḡ", tolower("gĝğġģǥǧǵḡ"))
+  call assert_equal("hĥħḣḧḩẖ", tolower("hĥħḣḧḩẖ"))
+  call assert_equal("iìíîïĩīĭįǐỉ", tolower("iìíîïĩīĭįǐỉ"))
+  call assert_equal("jĵǰ", tolower("jĵǰ"))
+  call assert_equal("kķǩḱḵ", tolower("kķǩḱḵ"))
+  call assert_equal("lĺļľŀłḻ", tolower("lĺļľŀłḻ"))
+  call assert_equal("mḿṁ ", tolower("mḿṁ "))
+  call assert_equal("nñńņňʼnṅṉ", tolower("nñńņňʼnṅṉ"))
+  call assert_equal("oòóôõöøōŏőơǒǫǭỏ", tolower("oòóôõöøōŏőơǒǫǭỏ"))
+  call assert_equal("pṕṗ", tolower("pṕṗ"))
+  call assert_equal("q", tolower("q"))
+  call assert_equal("rŕŗřṙṟ", tolower("rŕŗřṙṟ"))
+  call assert_equal("sśŝşšṡ", tolower("sśŝşšṡ"))
+  call assert_equal("tţťŧṫṯẗ", tolower("tţťŧṫṯẗ"))
+  call assert_equal("uùúûüũūŭůűųưǔủ", tolower("uùúûüũūŭůűųưǔủ"))
+  call assert_equal("vṽ", tolower("vṽ"))
+  call assert_equal("wŵẁẃẅẇẘ", tolower("wŵẁẃẅẇẘ"))
+  call assert_equal("ẋẍ", tolower("ẋẍ"))
+  call assert_equal("yýÿŷẏẙỳỷỹ", tolower("yýÿŷẏẙỳỷỹ"))
+  call assert_equal("zźżžƶẑẕ", tolower("zźżžƶẑẕ"))
+
+  " According to https://twitter.com/jifa/status/625776454479970304
+  " Ⱥ (U+023A) and Ⱦ (U+023E) are the *only* code points to increase
+  " in length (2 to 3 bytes) when lowercased. So let's test them.
+  call assert_equal("ⱥ ⱦ", tolower("Ⱥ Ⱦ"))
+endfunc
+
+func Test_toupper()
+  call assert_equal("", toupper(""))
+
+  " Test with all printable ASCII characters.
+  call assert_equal(' !"#$%&''()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`ABCDEFGHIJKLMNOPQRSTUVWXYZ{|}~',
+          \ toupper(' !"#$%&''()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~'))
+
+  if !has('multi_byte')
+    return
+  endif
+
+  " Test with a few lowercase diacritics.
+  call assert_equal("AÀÁÂÃÄÅĀĂĄǍǞǠẢ", toupper("aàáâãäåāăąǎǟǡả"))
+  call assert_equal("BḂḆ", toupper("bḃḇ"))
+  call assert_equal("CÇĆĈĊČ", toupper("cçćĉċč"))
+  call assert_equal("DĎĐḊḎḐ", toupper("dďđḋḏḑ"))
+  call assert_equal("EÈÉÊËĒĔĖĘĚẺẼ", toupper("eèéêëēĕėęěẻẽ"))
+  call assert_equal("FḞ", toupper("fḟ"))
+  call assert_equal("GĜĞĠĢǤǦǴḠ", toupper("gĝğġģǥǧǵḡ"))
+  call assert_equal("HĤĦḢḦḨẖ", toupper("hĥħḣḧḩẖ"))
+  call assert_equal("IÌÍÎÏĨĪĬĮǏỈ", toupper("iìíîïĩīĭįǐỉ"))
+  call assert_equal("JĴǰ", toupper("jĵǰ"))
+  call assert_equal("KĶǨḰḴ", toupper("kķǩḱḵ"))
+  call assert_equal("LĹĻĽĿŁḺ", toupper("lĺļľŀłḻ"))
+  call assert_equal("MḾṀ ", toupper("mḿṁ "))
+  call assert_equal("NÑŃŅŇʼnṄṈ", toupper("nñńņňʼnṅṉ"))
+  call assert_equal("OÒÓÔÕÖØŌŎŐƠǑǪǬỎ", toupper("oòóôõöøōŏőơǒǫǭỏ"))
+  call assert_equal("PṔṖ", toupper("pṕṗ"))
+  call assert_equal("Q", toupper("q"))
+  call assert_equal("RŔŖŘṘṞ", toupper("rŕŗřṙṟ"))
+  call assert_equal("SŚŜŞŠṠ", toupper("sśŝşšṡ"))
+  call assert_equal("TŢŤŦṪṮẗ", toupper("tţťŧṫṯẗ"))
+  call assert_equal("UÙÚÛÜŨŪŬŮŰŲƯǓỦ", toupper("uùúûüũūŭůűųưǔủ"))
+  call assert_equal("VṼ", toupper("vṽ"))
+  call assert_equal("WŴẀẂẄẆẘ", toupper("wŵẁẃẅẇẘ"))
+  call assert_equal("ẊẌ", toupper("ẋẍ"))
+  call assert_equal("YÝŸŶẎẙỲỶỸ", toupper("yýÿŷẏẙỳỷỹ"))
+  call assert_equal("ZŹŻŽƵẐẔ", toupper("zźżžƶẑẕ"))
+
+  " Test that uppercase diacritics, which should remain unchanged.
+  call assert_equal("AÀÁÂÃÄÅĀĂĄǍǞǠẢ", toupper("AÀÁÂÃÄÅĀĂĄǍǞǠẢ"))
+  call assert_equal("BḂḆ", toupper("BḂḆ"))
+  call assert_equal("CÇĆĈĊČ", toupper("CÇĆĈĊČ"))
+  call assert_equal("DĎĐḊḎḐ", toupper("DĎĐḊḎḐ"))
+  call assert_equal("EÈÉÊËĒĔĖĘĚẺẼ", toupper("EÈÉÊËĒĔĖĘĚẺẼ"))
+  call assert_equal("FḞ ", toupper("FḞ "))
+  call assert_equal("GĜĞĠĢǤǦǴḠ", toupper("GĜĞĠĢǤǦǴḠ"))
+  call assert_equal("HĤĦḢḦḨ", toupper("HĤĦḢḦḨ"))
+  call assert_equal("IÌÍÎÏĨĪĬĮİǏỈ", toupper("IÌÍÎÏĨĪĬĮİǏỈ"))
+  call assert_equal("JĴ", toupper("JĴ"))
+  call assert_equal("KĶǨḰḴ", toupper("KĶǨḰḴ"))
+  call assert_equal("LĹĻĽĿŁḺ", toupper("LĹĻĽĿŁḺ"))
+  call assert_equal("MḾṀ", toupper("MḾṀ"))
+  call assert_equal("NÑŃŅŇṄṈ", toupper("NÑŃŅŇṄṈ"))
+  call assert_equal("OÒÓÔÕÖØŌŎŐƠǑǪǬỎ", toupper("OÒÓÔÕÖØŌŎŐƠǑǪǬỎ"))
+  call assert_equal("PṔṖ", toupper("PṔṖ"))
+  call assert_equal("Q", toupper("Q"))
+  call assert_equal("RŔŖŘṘṞ", toupper("RŔŖŘṘṞ"))
+  call assert_equal("SŚŜŞŠṠ", toupper("SŚŜŞŠṠ"))
+  call assert_equal("TŢŤŦṪṮ", toupper("TŢŤŦṪṮ"))
+  call assert_equal("UÙÚÛÜŨŪŬŮŰŲƯǓỦ", toupper("UÙÚÛÜŨŪŬŮŰŲƯǓỦ"))
+  call assert_equal("VṼ", toupper("VṼ"))
+  call assert_equal("WŴẀẂẄẆ", toupper("WŴẀẂẄẆ"))
+  call assert_equal("XẊẌ", toupper("XẊẌ"))
+  call assert_equal("YÝŶŸẎỲỶỸ", toupper("YÝŶŸẎỲỶỸ"))
+  call assert_equal("ZŹŻŽƵẐẔ", toupper("ZŹŻŽƵẐẔ"))
+
+  call assert_equal("ⱥ ⱦ", tolower("Ⱥ Ⱦ"))
+endfunc
+
+

src/testdir/test_paste.vim

@@ -35,6 +35,16 @@ func Test_paste_insert_mode()
   call assert_equal('bar donfoo', getline(3))
   call assert_equal('bar doneeb', getline(4))
   call assert_equal('c', getline(5))
+
+  set ai et tw=10
+  call setline(1, ['a', '    b', 'c'])
+  2
+  call feedkeys("A\<Esc>[200~foo\<CR> bar bar bar\<Esc>[201~\<Esc>", 'xt')
+  call assert_equal('    bfoo', getline(2))
+  call assert_equal(' bar bar bar', getline(3))
+  call assert_equal('c', getline(4))
+
+  set ai& et& tw=0
   bwipe!
 endfunc
 

src/testdir/test_popup.vim

@@ -7,10 +7,10 @@ func! ListMonths()
   if g:setting != ''
     exe ":set" g:setting
   endif
-  let mth=copy(g:months)
+  let mth = copy(g:months)
   let entered = strcharpart(getline('.'),0,col('.'))
   if !empty(entered)
-    let mth=filter(mth, 'v:val=~"^".entered')
+    let mth = filter(mth, 'v:val=~"^".entered')
   endif
   call complete(1, mth) 
   return ''
@@ -468,7 +468,7 @@ endfunc
 " auto-wrap text.
 func Test_completion_ctrl_e_without_autowrap()
   new
-  let tw_save=&tw
+  let tw_save = &tw
   set tw=78
   let li = [
         \ '"                                                        zzz',
@@ -478,8 +478,37 @@ func Test_completion_ctrl_e_without_autowrap()
   call feedkeys("A\<C-X>\<C-N>\<C-E>\<Esc>", "tx")
   call assert_equal(li, getline(1, '$'))
 
-  let &tw=tw_save
+  let &tw = tw_save
   q!
 endfunc
 
+function! DummyCompleteSix()
+  call complete(1, ['Hello', 'World'])
+  return ''
+endfunction
+
+" complete() correctly clears the list of autocomplete candidates
+" See #1411
+func Test_completion_clear_candidate_list()
+  new
+  %d
+  " select first entry from the completion popup
+  call feedkeys("a    xxx\<C-N>\<C-R>=DummyCompleteSix()\<CR>", "tx")
+  call assert_equal('Hello', getline(1))
+  %d
+  " select second entry from the completion popup
+  call feedkeys("a    xxx\<C-N>\<C-R>=DummyCompleteSix()\<CR>\<C-N>", "tx")
+  call assert_equal('World', getline(1))
+  %d
+  " select original text
+  call feedkeys("a    xxx\<C-N>\<C-R>=DummyCompleteSix()\<CR>\<C-N>\<C-N>", "tx")
+  call assert_equal('    xxx', getline(1))
+  %d
+  " back at first entry from completion list
+  call feedkeys("a    xxx\<C-N>\<C-R>=DummyCompleteSix()\<CR>\<C-N>\<C-N>\<C-N>", "tx")
+  call assert_equal('Hello', getline(1))
+
+  bw!
+endfunc
+
 " vim: shiftwidth=2 sts=2 expandtab

src/testdir/test_usercommands.vim

@@ -102,3 +102,107 @@ func Test_CmdUndefined()
   call assert_fails('Dothat', 'E492:')
   call assert_equal('yes', g:didnot)
 endfunc
+
+func Test_CmdErrors()
+  call assert_fails('com! docmd :', 'E183:')
+  call assert_fails('com! \<Tab> :', 'E182:')
+  call assert_fails('com! _ :', 'E182:')
+  call assert_fails('com! X :', 'E841:')
+  call assert_fails('com! - DoCmd :', 'E175:')
+  call assert_fails('com! -xxx DoCmd :', 'E181:')
+  call assert_fails('com! -addr DoCmd :', 'E179:')
+  call assert_fails('com! -complete DoCmd :', 'E179:')
+  call assert_fails('com! -complete=xxx DoCmd :', 'E180:')
+  call assert_fails('com! -complete=custom DoCmd :', 'E467:')
+  call assert_fails('com! -complete=customlist DoCmd :', 'E467:')
+  call assert_fails('com! -complete=behave,CustomComplete DoCmd :', 'E468:')
+  call assert_fails('com! -nargs=x DoCmd :', 'E176:')
+  call assert_fails('com! -count=1 -count=2 DoCmd :', 'E177:')
+  call assert_fails('com! -count=x DoCmd :', 'E178:')
+  call assert_fails('com! -range=x DoCmd :', 'E178:')
+
+  com! -nargs=0 DoCmd :
+  call assert_fails('DoCmd x', 'E488:')
+
+  com! -nargs=1 DoCmd :
+  call assert_fails('DoCmd', 'E471:')
+
+  com! -nargs=+ DoCmd :
+  call assert_fails('DoCmd', 'E471:')
+
+  call assert_fails('com DoCmd :', 'E174:')
+  comclear
+  call assert_fails('delcom DoCmd', 'E184:')
+endfunc
+
+func CustomComplete(A, L, P)
+  return "January\nFebruary\nMars\n"
+endfunc
+
+func CustomCompleteList(A, L, P)
+  return [ "Monday", "Tuesday", "Wednesday" ]
+endfunc
+
+func Test_CmdCompletion()
+  call feedkeys(":com -\<C-A>\<C-B>\"\<CR>", 'tx')
+  call assert_equal('"com -addr bang bar buffer complete count nargs range register', @:)
+
+  call feedkeys(":com -nargs=0 -\<C-A>\<C-B>\"\<CR>", 'tx')
+  call assert_equal('"com -nargs=0 -addr bang bar buffer complete count nargs range register', @:)
+
+  call feedkeys(":com -nargs=\<C-A>\<C-B>\"\<CR>", 'tx')
+  call assert_equal('"com -nargs=* + 0 1 ?', @:)
+
+  call feedkeys(":com -addr=\<C-A>\<C-B>\"\<CR>", 'tx')
+  call assert_equal('"com -addr=arguments buffers lines loaded_buffers quickfix tabs windows', @:)
+
+  call feedkeys(":com -complete=co\<C-A>\<C-B>\"\<CR>", 'tx')
+  call assert_equal('"com -complete=color command compiler', @:)
+
+  command! DoCmd1 :
+  command! DoCmd2 :
+  call feedkeys(":com \<C-A>\<C-B>\"\<CR>", 'tx')
+  call assert_equal('"com DoCmd1 DoCmd2', @:)
+
+  call feedkeys(":DoC\<C-A>\<C-B>\"\<CR>", 'tx')
+  call assert_equal('"DoCmd1 DoCmd2', @:)
+
+  call feedkeys(":delcom DoC\<C-A>\<C-B>\"\<CR>", 'tx')
+  call assert_equal('"delcom DoCmd1 DoCmd2', @:)
+
+  delcom DoCmd1
+  call feedkeys(":delcom DoC\<C-A>\<C-B>\"\<CR>", 'tx')
+  call assert_equal('"delcom DoCmd2', @:)
+
+  call feedkeys(":com DoC\<C-A>\<C-B>\"\<CR>", 'tx')
+  call assert_equal('"com DoCmd2', @:)
+
+  delcom DoCmd2
+  call feedkeys(":delcom DoC\<C-A>\<C-B>\"\<CR>", 'tx')
+  call assert_equal('"delcom DoC', @:)
+
+  call feedkeys(":com DoC\<C-A>\<C-B>\"\<CR>", 'tx')
+  call assert_equal('"com DoC', @:)
+
+  com! -complete=behave DoCmd :
+  call feedkeys(":DoCmd \<C-A>\<C-B>\"\<CR>", 'tx')
+  call assert_equal('"DoCmd mswin xterm', @:)
+
+  " This does not work. Why?
+  "call feedkeys(":DoCmd x\<C-A>\<C-B>\"\<CR>", 'tx')
+  "call assert_equal('"DoCmd xterm', @:)
+
+  com! -complete=custom,CustomComplete DoCmd :
+  call feedkeys(":DoCmd \<C-A>\<C-B>\"\<CR>", 'tx')
+  call assert_equal('"DoCmd January February Mars', @:)
+
+  com! -complete=customlist,CustomCompleteList DoCmd :
+  call feedkeys(":DoCmd \<C-A>\<C-B>\"\<CR>", 'tx')
+  call assert_equal('"DoCmd Monday Tuesday Wednesday', @:)
+
+  com! -complete=custom,CustomCompleteList DoCmd :
+  call assert_fails("call feedkeys(':DoCmd \<C-D>', 'tx')", 'E730:')
+
+  com! -complete=customlist,CustomComp DoCmd :
+  call assert_fails("call feedkeys(':DoCmd \<C-D>', 'tx')", 'E117:')
+endfunc

src/testdir/test_utf8.in

@@ -1,46 +0,0 @@
-Tests for Unicode manipulations                vim: set ft=vim :
- 
-STARTTEST
-:so small.vim
-:set encoding=utf-8
-:"
-:" Visual block Insert adjusts for multi-byte char
-:new
-:call setline(1, ["aaa", "あああ", "bbb"])
-:exe ":norm! gg0l\<C-V>jjIx\<Esc>"
-:let r = getline(1, '$')
-:"
-:bwipeout!
-:$put=r
-:"
-:" Test for built-in function strchars()
-:for str in ["a", "あいa", "A\u20dd", "A\u20dd\u20dd", "\u20dd"]
-:	$put=strchars(str)
-:	$put=strchars(str, 0)
-:	$put=strchars(str, 1)
-:endfor
-:"
-:" Test for customlist completion
-:function! CustomComplete1(lead, line, pos)
-:	return ['あ', 'い']
-:endfunction
-:command -nargs=1 -complete=customlist,CustomComplete1 Test1 echo
-:call feedkeys(":Test1 \<C-L>'\<C-B>$put='\<CR>", 'it')
-:"
-:function! CustomComplete2(lead, line, pos)
-:	return ['あたし', 'あたま', 'あたりめ']
-:endfunction
-:command -nargs=1 -complete=customlist,CustomComplete2 Test2 echo
-:call feedkeys(":Test2 \<C-L>'\<C-B>$put='\<CR>", 'it')
-:"
-:function! CustomComplete3(lead, line, pos)
-:	return ['Nこ', 'Nん', 'Nぶ']
-:endfunction
-:command -nargs=1 -complete=customlist,CustomComplete3 Test3 echo
-:call feedkeys(":Test3 \<C-L>'\<C-B>$put='\<CR>", 'it')
-:"
-:call garbagecollect(1)
-:/^start:/,$wq! test.out
-ENDTEST
- 
-start:

src/testdir/test_utf8.ok

@@ -1,22 +0,0 @@
-start:
-axaa
-xあああ
-bxbb
-1
-1
-1
-3
-3
-3
-2
-2
-1
-3
-3
-1
-1
-1
-1
-Test1 
-Test2 あた
-Test3 N

src/testdir/test_utf8.vim

@@ -0,0 +1,65 @@
+" Tests for Unicode manipulations
+if !has('multi_byte')
+  finish
+endif
+ 
+
+" Visual block Insert adjusts for multi-byte char
+func Test_visual_block_insert()
+  new
+  call setline(1, ["aaa", "あああ", "bbb"])
+  exe ":norm! gg0l\<C-V>jjIx\<Esc>"
+  call assert_equal(['axaa', 'xあああ', 'bxbb'], getline(1, '$'))
+  bwipeout!
+endfunc
+
+" Test for built-in function strchars()
+func Test_strchars()
+  let inp = ["a", "あいa", "A\u20dd", "A\u20dd\u20dd", "\u20dd"]
+  let exp = [[1, 1, 1], [3, 3, 3], [2, 2, 1], [3, 3, 1], [1, 1, 1]]
+  for i in range(len(inp))
+    call assert_equal(exp[i][0], strchars(inp[i]))
+    call assert_equal(exp[i][1], strchars(inp[i], 0))
+    call assert_equal(exp[i][2], strchars(inp[i], 1))
+  endfor
+endfunc
+
+" Test for customlist completion
+function! CustomComplete1(lead, line, pos)
+	return ['あ', 'い']
+endfunction
+
+function! CustomComplete2(lead, line, pos)
+	return ['あたし', 'あたま', 'あたりめ']
+endfunction
+
+function! CustomComplete3(lead, line, pos)
+	return ['Nこ', 'Nん', 'Nぶ']
+endfunction
+
+func Test_customlist_completion()
+  command -nargs=1 -complete=customlist,CustomComplete1 Test1 echo
+  call feedkeys(":Test1 \<C-L>\<C-B>\"\<CR>", 'itx')
+  call assert_equal('"Test1 ', getreg(':'))
+
+  command -nargs=1 -complete=customlist,CustomComplete2 Test2 echo
+  call feedkeys(":Test2 \<C-L>\<C-B>\"\<CR>", 'itx')
+  call assert_equal('"Test2 あた', getreg(':'))
+
+  command -nargs=1 -complete=customlist,CustomComplete3 Test3 echo
+  call feedkeys(":Test3 \<C-L>\<C-B>\"\<CR>", 'itx')
+  call assert_equal('"Test3 N', getreg(':'))
+
+  call garbagecollect(1)
+endfunc
+
+" Yank one 3 byte character and check the mark columns.
+func Test_getvcol()
+  new
+  call setline(1, "x\u2500x")
+  normal 0lvy
+  call assert_equal(2, col("'["))
+  call assert_equal(4, col("']"))
+  call assert_equal(2, virtcol("'["))
+  call assert_equal(2, virtcol("']"))
+endfunc

src/version.c

@@ -764,6 +764,32 @@ static char *(features[]) =
 
 static int included_patches[] =
 {   /* Add new patch number below this line */
+/**/
+    250,
+/**/
+    249,
+/**/
+    248,
+/**/
+    247,
+/**/
+    246,
+/**/
+    245,
+/**/
+    244,
+/**/
+    243,
+/**/
+    242,
+/**/
+    241,
+/**/
+    240,
+/**/
+    239,
+/**/
+    238,
 /**/
     237,
 /**/

src/vim.h

@@ -1714,15 +1714,8 @@ typedef unsigned short disptick_T;	/* display tick type */
 
 typedef void	    *vim_acl_T;		/* dummy to pass an ACL to a function */
 
-/*
- * Include a prototype for mch_memmove(), it may not be in alloc.pro.
- */
-#ifdef VIM_MEMMOVE
-void mch_memmove(void *, void *, size_t);
-#else
-# ifndef mch_memmove
-#  define mch_memmove(to, from, len) memmove(to, from, len)
-# endif
+#ifndef mch_memmove
+# define mch_memmove(to, from, len) memmove((char*)(to), (char*)(from), (size_t)(len))
 #endif
 
 /*