patch 8.0.0388: filtering lines changes folds

Problem:    filtering lines through "cat", without changing the line count,
            changes manual folds.
Solution:   Change how marks and folds are adjusted. (Matthew Malcomson, from
            neovim #6194.

src/Makefile

@@ -2092,6 +2092,7 @@ test1 \
 
 # Run individual NEW style test, assuming that Vim was already compiled.
 test_arglist \
+	test_arabic \
 	test_assert \
 	test_assign \
 	test_autochdir \

src/channel.c

@@ -1966,7 +1966,7 @@ channel_parse_json(channel_T *channel, ch_part_T part)
 	     * more (but still incomplete): set a deadline of 100 msec. */
 	    ch_logn(channel,
 		    "Incomplete message (%d bytes) - wait 100 msec for more",
-		    buflen);
+		    (int)buflen);
 	    reader.js_used = 0;
 	    chanpart->ch_wait_len = buflen;
 #ifdef WIN32
@@ -3299,6 +3299,7 @@ channel_read_block(channel_T *channel, ch_part_T part, int timeout)
 	channel_read(channel, part, "channel_read_block");
     }
 
+    /* We have a complete message now. */
     if (mode == MODE_RAW)
     {
 	msg = channel_get_all(channel, part);

src/edit.c

@@ -1038,8 +1038,10 @@ doESCkey:
 	    if (!p_im)
 		goto normalchar;	/* insert CTRL-Z as normal char */
 	    do_cmdline_cmd((char_u *)"stop");
-	    c = Ctrl_O;
-	    /*FALLTHROUGH*/
+#ifdef CURSOR_SHAPE
+	    ui_cursor_shape();		/* may need to update cursor shape */
+#endif
+	    continue;
 
 	case Ctrl_O:	/* execute one command */
 #ifdef FEAT_COMPL_FUNC

src/ex_cmds.c

@@ -5288,6 +5288,8 @@ do_sub(exarg_T *eap)
 
 			    getvcol(curwin, &curwin->w_cursor, &sc, NULL, NULL);
 			    curwin->w_cursor.col = regmatch.endpos[0].col - 1;
+			    if (curwin->w_cursor.col < 0)
+				curwin->w_cursor.col = 0;
 			    getvcol(curwin, &curwin->w_cursor, NULL, NULL, &ec);
 			    if (subflags.do_number || curwin->w_p_nu)
 			    {

src/ex_getln.c

@@ -2369,9 +2369,16 @@ getexmodeline(
 	if (ga_grow(&line_ga, 40) == FAIL)
 	    break;
 
-	/* Get one character at a time. */
+	/*
+	 * Get one character at a time.
+	 */
 	prev_char = c1;
-	c1 = vgetc();
+
+	/* Check for a ":normal" command and no more characters left. */
+	if (ex_normal_busy > 0 && typebuf.tb_len == 0)
+	    c1 = '\n';
+	else
+	    c1 = vgetc();
 
 	/*
 	 * Handle line editing.

src/fold.c

@@ -1576,16 +1576,23 @@ foldMarkAdjustRecurse(
 		{
 		    /* 5. fold is below line1 and contains line2; need to
 		     * correct nested folds too */
-		    foldMarkAdjustRecurse(&fp->fd_nested, line1 - fp->fd_top,
-				  line2 - fp->fd_top, amount,
-				  amount_after + (fp->fd_top - top));
 		    if (amount == MAXLNUM)
 		    {
+			foldMarkAdjustRecurse(&fp->fd_nested,
+				  line1 - fp->fd_top,
+				  line2 - fp->fd_top,
+				  amount,
+				  amount_after + (fp->fd_top - top));
 			fp->fd_len -= line2 - fp->fd_top + 1;
 			fp->fd_top = line1;
 		    }
 		    else
 		    {
+			foldMarkAdjustRecurse(&fp->fd_nested,
+				  line1 - fp->fd_top,
+				  line2 - fp->fd_top,
+				  amount,
+				  amount_after - amount);
 			fp->fd_len += amount_after - amount;
 			fp->fd_top += amount;
 		    }

src/gen_opt_test.vim

@@ -1,10 +1,10 @@
 " Script to generate testdir/opt_test.vim from option.c
 
-if 0
-  finish
-endif
-
 set cpo=&vim
+
+" Only do this when build with the +eval feature.
+if 1
+
 set nomore
 
 let script = [
@@ -130,7 +130,6 @@ let test_values = {
       \ 'winaltkeys': [['menu', 'no'], ['', 'xxx']],
       \
       \ 'luadll': [[], []],
-      \ 'macatsui': [[], []],
       \ 'perldll': [[], []],
       \ 'pythondll': [[], []],
       \ 'pythonthreedll': [[], []],
@@ -192,4 +191,6 @@ call add(script, 'let &lines = save_lines')
 
 call writefile(script, 'testdir/opt_test.vim')
 
+endif
+
 qa!

src/memline.c

@@ -1863,7 +1863,7 @@ recover_names(
 	    else
 	    {
 #if defined(UNIX) || defined(WIN3264)
-		int	len = STRLEN(dir_name);
+		int	len = (int)STRLEN(dir_name);
 
 		p = dir_name + len;
 		if (after_pathsep(dir_name, p) && len > 1 && p[-1] == p[-2])
@@ -3924,7 +3924,7 @@ makeswapname(
 #endif
 
 #if defined(UNIX) || defined(WIN3264)  /* Need _very_ long file names */
-    int		len = STRLEN(dir_name);
+    int		len = (int)STRLEN(dir_name);
 
     s = dir_name + len;
     if (after_pathsep(dir_name, s) && len > 1 && s[-1] == s[-2])

src/normal.c

@@ -2982,8 +2982,6 @@ do_mouse(
 		|| (mod_mask & MOD_MASK_MULTI_CLICK) == MOD_MASK_2CLICK)
 	    && bt_quickfix(curbuf))
     {
-	if (State & INSERT)
-	    stuffcharReadbuff(Ctrl_O);
 	if (curwin->w_llist_ref == NULL)	/* quickfix window */
 	    do_cmdline_cmd((char_u *)".cc");
 	else					/* location list window */
@@ -6193,10 +6191,12 @@ nv_down(cmdarg_T *cap)
 #if defined(FEAT_WINDOWS) && defined(FEAT_QUICKFIX)
     /* In a quickfix window a <CR> jumps to the error under the cursor. */
     if (bt_quickfix(curbuf) && cap->cmdchar == CAR)
+    {
 	if (curwin->w_llist_ref == NULL)
 	    do_cmdline_cmd((char_u *)".cc");	/* quickfix window */
 	else
 	    do_cmdline_cmd((char_u *)".ll");	/* location list window */
+    }
     else
 #endif
     {

src/option.c

@@ -1278,8 +1278,8 @@ static struct vimoption options[] =
 #endif
 			    SCRIPTID_INIT},
     {"foldmarker",  "fmr",  P_STRING|P_ALLOCED|P_VIM|P_VI_DEF|
-#ifdef FEAT_FOLDING
 						    P_RWIN|P_ONECOMMA|P_NODUP,
+#ifdef FEAT_FOLDING
 			    (char_u *)VAR_WIN, PV_FMR,
 			    {(char_u *)"{{{,}}}", (char_u *)NULL}
 #else

src/screen.c

@@ -4189,6 +4189,10 @@ win_line(
 	}
 	else
 	{
+#ifdef FEAT_LINEBREAK
+	    int c0;
+#endif
+
 	    if (p_extra_free != NULL)
 	    {
 		vim_free(p_extra_free);
@@ -4198,6 +4202,9 @@ win_line(
 	     * Get a character from the line itself.
 	     */
 	    c = *ptr;
+#ifdef FEAT_LINEBREAK
+	    c0 = *ptr;
+#endif
 #ifdef FEAT_MBYTE
 	    if (has_mbyte)
 	    {
@@ -4214,7 +4221,12 @@ win_line(
 			/* Overlong encoded ASCII or ASCII with composing char
 			 * is displayed normally, except a NUL. */
 			if (mb_c < 0x80)
+			{
 			    c = mb_c;
+# ifdef FEAT_LINEBREAK
+			    c0 = mb_c;
+# endif
+			}
 			mb_utf8 = TRUE;
 
 			/* At start of the line we can have a composing char.
@@ -4538,7 +4550,8 @@ win_line(
 		/*
 		 * Found last space before word: check for line break.
 		 */
-		if (wp->w_p_lbr && vim_isbreak(c) && !vim_isbreak(*ptr))
+		if (wp->w_p_lbr && c0 == c
+				      && vim_isbreak(c) && !vim_isbreak(*ptr))
 		{
 # ifdef FEAT_MBYTE
 		    int mb_off = has_mbyte ? (*mb_head_off)(line, ptr - 1) : 0;

src/spellfile.c

@@ -1585,7 +1585,7 @@ spell_read_tree(
     int		prefixtree,	/* TRUE for the prefix tree */
     int		prefixcnt)	/* when "prefixtree" is TRUE: prefix count */
 {
-    int		len;
+    long	len;
     int		idx;
     char_u	*bp;
     idx_T	*ip;
@@ -1595,7 +1595,7 @@ spell_read_tree(
     len = get4c(fd);
     if (len < 0)
 	return SP_TRUNCERROR;
-    if (len >= 0x3ffffff)
+    if (len >= LONG_MAX / (long)sizeof(int))
 	/* Invalid length, multiply with sizeof(int) would overflow. */
 	return SP_FORMERROR;
     if (len > 0)

src/testdir/Make_all.mak

@@ -133,7 +133,8 @@ SCRIPTS_GUI =
 
 # Tests using runtest.vim.vim.
 # Keep test_alot*.res as the last one, sort the others.
-NEW_TESTS = test_arglist.res \
+NEW_TESTS = test_arabic.res \
+	    test_arglist.res \
 	    test_assert.res \
 	    test_autochdir.res \
 	    test_backspace_opt.res \

src/testdir/runtest.vim

@@ -165,6 +165,7 @@ let s:flaky = [
       \ 'Test_collapse_buffers()',
       \ 'Test_communicate()',
       \ 'Test_nb_basic()',
+      \ 'Test_oneshot()',
       \ 'Test_pipe_through_sort_all()',
       \ 'Test_pipe_through_sort_some()',
       \ 'Test_reltime()',

src/testdir/test_arabic.vim

@@ -0,0 +1,92 @@
+" Simplistic testing of Arabic mode.
+
+if !has('arabic')
+  finish
+endif
+
+set encoding=utf-8
+scriptencoding utf-8
+
+" Return list of utf8 sequences of each character at line lnum.
+" Combining characters are treated as a single item.
+func GetCharsUtf8(lnum)
+  call cursor(a:lnum, 1)
+  let chars = []
+  let numchars = strchars(getline('.'), 1)
+  for i in range(1, numchars)
+    exe 'norm ' i . '|'
+    call add(chars, execute('norm g8'))
+  endfor
+  return chars
+endfunc
+
+func Test_arabic_toggle()
+  set arabic
+  call assert_equal(1, &rightleft)
+  call assert_equal(1, &arabicshape)
+  call assert_equal('arabic', &keymap)
+  call assert_equal(1, &delcombine)
+
+  set iminsert=1 imsearch=1
+  set arabic&
+  call assert_equal(0, &rightleft)
+  call assert_equal(1, &arabicshape)
+  call assert_equal('arabic', &keymap)
+  call assert_equal(1, &delcombine)
+  call assert_equal(0, &iminsert)
+  call assert_equal(-1, &imsearch)
+
+  set arabicshape& keymap= delcombine&
+endfunc
+
+func Test_arabic_input()
+  new
+  set arabic
+  " Typing sghl in Arabic insert mode should show the
+  " Arabic word 'Salaam' i.e. 'peace'.
+  call feedkeys('isghl', 'tx')
+  redraw
+  call assert_equal([
+  \ "\nd8 b3 ",
+  \ "\nd9 84 + d8 a7 ",
+  \ "\nd9 85 "], GetCharsUtf8(1))
+
+  " Without shaping, it should give individual Arabic letters.
+  set noarabicshape
+  redraw
+  call assert_equal([
+  \ "\nd8 b3 ",
+  \ "\nd9 84 ",
+  \ "\nd8 a7 ",
+  \ "\nd9 85 "], GetCharsUtf8(1))
+
+  set arabicshape&
+  set arabic&
+  bwipe!
+endfunc
+
+func Test_arabic_toggle_keymap()
+  new
+  set arabic
+  call feedkeys("i12\<C-^>12\<C-^>12", 'tx')
+  redraw
+  call assert_equal('١٢12١٢', getline('.'))
+  set arabic&
+  bwipe!
+endfunc
+
+func Test_delcombine()
+  new
+  set arabic
+  call feedkeys("isghl\<BS>\<BS>", 'tx')
+  redraw
+  call assert_equal(["\nd8 b3 ", "\nd9 84 "], GetCharsUtf8(1))
+
+  " Now the same with nodelcombine
+  set nodelcombine
+  %d
+  call feedkeys("isghl\<BS>\<BS>", 'tx')
+  call assert_equal(["\nd8 b3 "], GetCharsUtf8(1)) 
+  set arabic&
+  bwipe!
+endfunc

src/testdir/test_diffmode.vim

@@ -272,3 +272,78 @@ func Test_setting_cursor()
   call delete('Xtest1')
   call delete('Xtest2')
 endfunc
+
+func Test_diff_move_to()
+  new
+  call setline(1, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
+  diffthis
+  vnew
+  call setline(1, [1, '2x', 3, 4, 4, 5, '6x', 7, '8x', 9, '10x'])
+  diffthis
+  norm ]c
+  call assert_equal(2, line('.'))
+  norm 3]c
+  call assert_equal(9, line('.'))
+  norm 10]c
+  call assert_equal(11, line('.'))
+  norm [c
+  call assert_equal(9, line('.'))
+  norm 2[c
+  call assert_equal(5, line('.'))
+  norm 10[c
+  call assert_equal(2, line('.'))
+  %bwipe!
+endfunc
+
+func Test_diffpatch()
+  " The patch program on MS-Windows may fail or hang.
+  if !executable('patch') || !has('unix')
+    return
+  endif
+  new
+  insert
+***************
+*** 1,3 ****
+  1
+! 2
+  3
+--- 1,4 ----
+  1
+! 2x
+  3
++ 4
+.
+  saveas Xpatch
+  bwipe!
+  new
+  call assert_fails('diffpatch Xpatch', 'E816:')
+  call setline(1, ['1', '2', '3'])
+  diffpatch Xpatch
+  call assert_equal(['1', '2x', '3', '4'], getline(1, '$'))
+  call delete('Xpatch')
+  bwipe!
+endfunc
+
+func Test_diff_too_many_buffers()
+  for i in range(1, 8)
+    exe "new Xtest" . i
+    diffthis
+  endfor
+  new Xtest9
+  call assert_fails('diffthis', 'E96:')
+  %bwipe!
+endfunc
+
+func Test_diff_nomodifiable()
+  new
+  call setline(1, [1, 2, 3, 4])
+  setl nomodifiable
+  diffthis
+  vnew
+  call setline(1, ['1x', 2, 3, 3, 4])
+  diffthis
+  call assert_fails('norm dp', 'E793:')
+  setl nomodifiable
+  call assert_fails('norm do', 'E21:')
+  %bwipe!
+endfunc

src/testdir/test_fold.vim

@@ -1,6 +1,6 @@
 " Test for folding
 
-function! Test_address_fold()
+func! Test_address_fold()
   new
   call setline(1, ['int FuncName() {/*{{{*/', 1, 2, 3, 4, 5, '}/*}}}*/',
 	      \ 'after fold 1', 'after fold 2', 'after fold 3'])
@@ -62,9 +62,9 @@ function! Test_address_fold()
   call assert_equal(['4', '5', '}/*}}}*/'], getreg(0,1,1))
 
   quit!
-endfunction
+endfunc
 
-function! Test_indent_fold()
+func! Test_indent_fold()
     new
     call setline(1, ['', 'a', '    b', '    c'])
     setl fen fdm=indent
@@ -72,9 +72,9 @@ function! Test_indent_fold()
     norm! >>
     let a=map(range(1,4), 'foldclosed(v:val)')
     call assert_equal([-1,-1,-1,-1], a)
-endfu
+endfunc
 
-function! Test_indent_fold()
+func! Test_indent_fold()
     new
     call setline(1, ['', 'a', '    b', '    c'])
     setl fen fdm=indent
@@ -83,9 +83,9 @@ function! Test_indent_fold()
     let a=map(range(1,4), 'foldclosed(v:val)')
     call assert_equal([-1,-1,-1,-1], a)
     bw!
-endfu
+endfunc
 
-function! Test_indent_fold2()
+func! Test_indent_fold2()
     new
     call setline(1, ['', '{{{', '}}}', '{{{', '}}}'])
     setl fen fdm=marker
@@ -94,4 +94,21 @@ function! Test_indent_fold2()
     let a=map(range(1,5), 'foldclosed(v:val)')
     call assert_equal([-1,-1,-1,4,4], a)
     bw!
-endfu
+endfunc
+
+func Test_manual_fold_with_filter()
+  if !executable('cat')
+    return
+  endif
+  new
+  call setline(1, range(1, 20))
+  4,$fold
+  %foldopen
+  10,$fold
+  %foldopen
+  " This filter command should not have an effect
+  1,8! cat
+  call feedkeys('5ggzdzMGdd', 'xt')
+  call assert_equal(['1', '2', '3', '4', '5', '6', '7', '8', '9'], getline(1, '$'))
+  bwipe!
+endfunc

src/testdir/test_gui.vim

@@ -72,6 +72,47 @@ func Test_getfontname_without_arg()
   endif
 endfunc
 
+func Test_quoteplus()
+  let skipped = ''
+
+  if !g:x11_based_gui
+    let skipped = g:not_supported . 'quoteplus'
+  else
+    let quoteplus_saved = @+
+
+    let test_call     = 'Can you hear me?'
+    let test_response = 'Yes, I can.'
+    let vim_exe = exepath(v:progpath)
+    let testee = 'VIMRUNTIME=' . $VIMRUNTIME . '; export VIMRUNTIME;'
+          \ . vim_exe . ' -f -g -u NONE -U NONE --noplugin -c ''%s'''
+    let cmd = 'call feedkeys("'
+          \ . '\"+p'
+          \ . ':s/' . test_call . '/' . test_response . '/\<CR>'
+          \ . '\"+yis'
+          \ . ':q!\<CR>", "tx")'
+    let run_vimtest = printf(testee, cmd)
+
+    " Set the quoteplus register to test_call, and another gvim will launched.
+    " Then, it first tries to paste the content of its own quotedplus register
+    " onto it.  Second, it tries to substitute test_responce for the pasted
+    " sentence.  If the sentence is identical to test_call, the substitution
+    " should succeed.  Third, it tries to yank the result of the substitution
+    " to its own quoteplus register, and last it quits.  When system()
+    " returns, the content of the quoteplus register should be identical to
+    " test_response if those quoteplus registers are synchronized properly
+    " with/through the X11 clipboard.
+    let @+ = test_call
+    call system(run_vimtest)
+    call assert_equal(test_response, @+)
+
+    let @+ = quoteplus_saved
+  endif
+
+  if !empty(skipped)
+    throw skipped
+  endif
+endfunc
+
 func Test_set_guifont()
   let skipped = ''
 

src/testdir/test_listlbr_utf8.vim

@@ -193,3 +193,30 @@ func Test_multibyte_sign_and_colorcolumn()
   call s:compare_lines(expect, lines)
   call s:close_windows()
 endfunc
+
+func Test_illegal_byte_and_breakat()
+  call s:test_windows("setl sbr= brk+=<")
+  vert resize 18
+  call setline(1, repeat("\x80", 6))
+  redraw!
+  let lines = s:screen_lines([1, 2], winwidth(0))
+  let expect = [
+\ "<80><80><80><80><8",
+\ "0><80>            ",
+\ ]
+  call s:compare_lines(expect, lines)
+  call s:close_windows('setl brk&vim')
+endfunc
+
+func Test_multibyte_wrap_and_breakat()
+  call s:test_windows("setl sbr= brk+=>")
+  call setline(1, repeat('a', 17) . repeat('あ', 2))
+  redraw!
+  let lines = s:screen_lines([1, 2], winwidth(0))
+  let expect = [
+\ "aaaaaaaaaaaaaaaaaあ>",
+\ "あ                  ",
+\ ]
+  call s:compare_lines(expect, lines)
+  call s:close_windows('setl brk&vim')
+endfunc

src/testdir/test_substitute.vim

@@ -106,3 +106,11 @@ function! Test_substitute_variants()
     endfor
   endfor
 endfunction
+
+func Test_substitute_repeat()
+  " This caused an invalid memory access.
+  split Xfile
+  s/^/x
+  call feedkeys("Qsc\<CR>y", 'tx')
+  bwipe!
+endfunc

src/undo.c

@@ -1385,7 +1385,7 @@ unserialize_uep(bufinfo_T *bi, int *error, char_u *file_name)
 {
     int		i;
     u_entry_T	*uep;
-    char_u	**array;
+    char_u	**array = NULL;
     char_u	*line;
     int		line_len;
 
@@ -1402,7 +1402,8 @@ unserialize_uep(bufinfo_T *bi, int *error, char_u *file_name)
     uep->ue_size = undo_read_4c(bi);
     if (uep->ue_size > 0)
     {
-	array = (char_u **)U_ALLOC_LINE(sizeof(char_u *) * uep->ue_size);
+	if (uep->ue_size < LONG_MAX / (int)sizeof(char_u *))
+	    array = (char_u **)U_ALLOC_LINE(sizeof(char_u *) * uep->ue_size);
 	if (array == NULL)
 	{
 	    *error = TRUE;
@@ -1410,8 +1411,6 @@ unserialize_uep(bufinfo_T *bi, int *error, char_u *file_name)
 	}
 	vim_memset(array, 0, sizeof(char_u *) * uep->ue_size);
     }
-    else
-	array = NULL;
     uep->ue_array = array;
 
     for (i = 0; i < uep->ue_size; ++i)
@@ -1787,7 +1786,7 @@ u_read_undo(char_u *name, char_u *hash, char_u *orig_name)
     linenr_T	line_lnum;
     colnr_T	line_colnr;
     linenr_T	line_count;
-    int		num_head = 0;
+    long	num_head = 0;
     long	old_header_seq, new_header_seq, cur_header_seq;
     long	seq_last, seq_cur;
     long	last_save_nr = 0;
@@ -1974,7 +1973,8 @@ u_read_undo(char_u *name, char_u *hash, char_u *orig_name)
      * When there are no headers uhp_table is NULL. */
     if (num_head > 0)
     {
-	uhp_table = (u_header_T **)U_ALLOC_LINE(
+	if (num_head < LONG_MAX / (long)sizeof(u_header_T *))
+	    uhp_table = (u_header_T **)U_ALLOC_LINE(
 					     num_head * sizeof(u_header_T *));
 	if (uhp_table == NULL)
 	    goto error;

src/version.c

@@ -764,6 +764,38 @@ static char *(features[]) =
 
 static int included_patches[] =
 {   /* Add new patch number below this line */
+/**/
+    388,
+/**/
+    387,
+/**/
+    386,
+/**/
+    385,
+/**/
+    384,
+/**/
+    383,
+/**/
+    382,
+/**/
+    381,
+/**/
+    380,
+/**/
+    379,
+/**/
+    378,
+/**/
+    377,
+/**/
+    376,
+/**/
+    375,
+/**/
+    374,
+/**/
+    373,
 /**/
     372,
 /**/