updated for version 7.3.994

Problem:    Python: using magic constants.
Solution:   Use descriptive values for ml_flags. (ZyX)

src/Makefile

@@ -1861,6 +1861,19 @@ unittest unittests: $(UNITTEST_TARGETS)
 		./$$t || exit 1; echo $$t passed; \
 	done
 
+# Run individual test, assuming that Vim was already compiled.
+test1 test2 test3 test4 test5 test6 test7 test8 test9 test10 \
+	test11 test12 test13 test14 test15 test16 test17 test18 test19 \
+	test21 test22 test23 test24 test25 test26 test27 test28 test29 \
+	test31 test32 test33 test34 test35 test36 test37 test38 test39 \
+	test41 test42 test43 test44 test45 test46 test47 test48 test49 \
+	test51 test52 test53 test54 test55 test56 test57 test58 test59 \
+	test61 test62 test63 test64 test65 test66 test67 test68 test69 \
+	test71 test72 test73 test74 test75 test76 test77 test78 test79 \
+	test81 test82 test83 test84 test85 test86 test87 test88 test89 \
+	test91 test92 test93 test94 test95 test96 test97 test98 test99:
+	cd testdir; rm $@.out; $(MAKE) -f Makefile $@.out VIMPROG=../$(VIMTARGET)
+
 testclean:
 	cd testdir; $(MAKE) -f Makefile clean
 	if test -d $(PODIR); then \

src/gui_gtk_x11.c

@@ -1447,6 +1447,11 @@ gui_mch_init_check(void)
 	using_gnome = 1;
 #endif
 
+    /* This defaults to argv[0], but we want it to match the name of the
+     * shipped gvim.desktop so that Vim's windows can be associated with this
+     * file. */
+    g_set_prgname("gvim");
+
     /* Don't use gtk_init() or gnome_init(), it exits on failure. */
     if (!gtk_init_check(&gui_argc, &gui_argv))
     {

src/if_python.c

@@ -56,8 +56,6 @@
 # define PY_SSIZE_T_CLEAN
 #endif
 
-static void init_structs(void);
-
 #define PyBytes_FromString PyString_FromString
 #define PyBytes_Check PyString_Check
 
@@ -659,18 +657,9 @@ static PyObject *FunctionGetattr(PyObject *, char *);
  * Internal function prototypes.
  */
 
-static PyObject *globals;
-
-static void PythonIO_Flush(void);
 static int PythonIO_Init(void);
 static int PythonMod_Init(void);
 
-/* Utility functions for the vim/python interface
- * ----------------------------------------------
- */
-
-static int SetBufferLineList(buf_T *, PyInt, PyInt, PyObject *, PyInt *);
-
 
 /******************************************************
  * 1. Python interpreter main program.
@@ -828,7 +817,7 @@ fail:
  * External interface
  */
     static void
-DoPythonCommand(exarg_T *eap, const char *cmd, typval_T *rettv)
+DoPyCommand(const char *cmd, rangeinitializer init_range, runner run, void *arg)
 {
 #ifndef PY_CAN_RECURSE
     static int		recursive = 0;
@@ -861,16 +850,8 @@ DoPythonCommand(exarg_T *eap, const char *cmd, typval_T *rettv)
     if (Python_Init())
 	goto theend;
 
-    if (rettv == NULL)
-    {
-	RangeStart = eap->line1;
-	RangeEnd = eap->line2;
-    }
-    else
-    {
-	RangeStart = (PyInt) curwin->w_cursor.lnum;
-	RangeEnd = RangeStart;
-    }
+    init_range(arg);
+
     Python_Release_Vim();	    /* leave vim */
 
 #if defined(HAVE_LOCALE_H) || defined(X_LOCALE)
@@ -892,27 +873,7 @@ DoPythonCommand(exarg_T *eap, const char *cmd, typval_T *rettv)
     Python_RestoreThread();	    /* enter python */
 #endif
 
-    if (rettv == NULL)
-	PyRun_SimpleString((char *)(cmd));
-    else
-    {
-	PyObject	*r;
-
-	r = PyRun_String((char *)(cmd), Py_eval_input, globals, globals);
-	if (r == NULL)
-	{
-	    if (PyErr_Occurred() && !msg_silent)
-		PyErr_PrintEx(0);
-	    EMSG(_("E858: Eval did not return a valid python object"));
-	}
-	else
-	{
-	    if (ConvertFromPyObject(r, rettv) == -1)
-		EMSG(_("E859: Failed to convert returned python object to vim value"));
-	    Py_DECREF(r);
-	}
-	PyErr_Clear();
-    }
+    run((char *) cmd, arg, &pygilstate);
 
 #ifdef PY_CAN_RECURSE
     PyGILState_Release(pygilstate);
@@ -952,10 +913,10 @@ ex_python(exarg_T *eap)
     script = script_get(eap, eap->arg);
     if (!eap->skip)
     {
-	if (script == NULL)
-	    DoPythonCommand(eap, (char *)eap->arg, NULL);
-	else
-	    DoPythonCommand(eap, (char *)script, NULL);
+	DoPyCommand(script == NULL ? (char *) eap->arg : (char *) script,
+		(rangeinitializer) init_range_cmd,
+		(runner) run_cmd,
+		(void *) eap);
     }
     vim_free(script);
 }
@@ -1001,94 +962,19 @@ ex_pyfile(exarg_T *eap)
     *p++ = '\0';
 
     /* Execute the file */
-    DoPythonCommand(eap, buffer, NULL);
+    DoPyCommand(buffer,
+	    (rangeinitializer) init_range_cmd,
+	    (runner) run_cmd,
+	    (void *) eap);
 }
 
     void
 ex_pydo(exarg_T *eap)
 {
-    linenr_T		i;
-    const char		*code_hdr = "def " DOPY_FUNC "(line, linenr):\n ";
-    const char		*s = (const char *) eap->arg;
-    size_t		len;
-    char		*code;
-    int			status;
-    PyObject		*pyfunc, *pymain;
-    PyGILState_STATE	pygilstate;
-
-    if (Python_Init())
-        return;
-
-    if (u_save(eap->line1 - 1, eap->line2 + 1) != OK)
-    {
-	EMSG(_("cannot save undo information"));
-	return;
-    }
-    len = strlen(code_hdr) + strlen(s);
-    code = malloc(len + 1);
-    STRCPY(code, code_hdr);
-    STRNCAT(code, s, len + 1);
-    pygilstate = PyGILState_Ensure();
-    status = PyRun_SimpleString(code);
-    vim_free(code);
-    if (status)
-    {
-	EMSG(_("failed to run the code"));
-	return;
-    }
-    status = 0; /* good */
-    pymain = PyImport_AddModule("__main__");
-    pyfunc = PyObject_GetAttrString(pymain, DOPY_FUNC);
-    PyGILState_Release(pygilstate);
-
-    for (i = eap->line1; i <= eap->line2; i++)
-    {
-	const char *line;
-	PyObject *pyline, *pylinenr, *pyret;
-
-	line = (char *)ml_get(i);
-	pygilstate = PyGILState_Ensure();
-	pyline = PyString_FromStringAndSize(line, strlen(line));
-	pylinenr = PyLong_FromLong(i);
-	pyret = PyObject_CallFunctionObjArgs(pyfunc, pyline, pylinenr, NULL);
-	Py_DECREF(pyline);
-	Py_DECREF(pylinenr);
-	if (!pyret)
-	{
-	    PyErr_PrintEx(0);
-	    PythonIO_Flush();
-	    status = 1;
-	    goto out;
-	}
-
-	if (pyret && pyret != Py_None)
-	{
-	    if (!PyString_Check(pyret))
-	    {
-		EMSG(_("E863: return value must be an instance of str"));
-		Py_XDECREF(pyret);
-		status = 1;
-		goto out;
-	    }
-	    ml_replace(i, (char_u *) PyString_AsString(pyret), 1);
-	    changed();
-#ifdef SYNTAX_HL
-	    syn_changed(i); /* recompute syntax hl. for this line */
-#endif
-	}
-	Py_XDECREF(pyret);
-	PythonIO_Flush();
-	PyGILState_Release(pygilstate);
-    }
-    pygilstate = PyGILState_Ensure();
-out:
-    Py_DECREF(pyfunc);
-    PyObject_SetAttrString(pymain, DOPY_FUNC, NULL);
-    PyGILState_Release(pygilstate);
-    if (status)
-	return;
-    check_cursor();
-    update_curbuf(NOT_VALID);
+    DoPyCommand((char *)eap->arg,
+	    (rangeinitializer) init_range_cmd,
+	    (runner)run_do,
+	    (void *)eap);
 }
 
 /******************************************************
@@ -1122,9 +1008,6 @@ PythonIO_Init(void)
  * 3. Implementation of the Vim module for Python
  */
 
-static PyObject *ConvertToPyObject(typval_T *);
-static int ConvertFromPyObject(PyObject *, typval_T *);
-
 /* Window type - Implementation functions
  * --------------------------------------
  */
@@ -1525,7 +1408,10 @@ FunctionGetattr(PyObject *self, char *name)
     void
 do_pyeval (char_u *str, typval_T *rettv)
 {
-    DoPythonCommand(NULL, (char *) str, rettv);
+    DoPyCommand((char *) str,
+	    (rangeinitializer) init_range_eval,
+	    (runner) run_eval,
+	    (void *) rettv);
     switch(rettv->v_type)
     {
 	case VAR_DICT: ++rettv->vval.v_dict->dv_refcount; break;

src/if_python3.c

@@ -68,8 +68,6 @@
 # define PY_SSIZE_T_CLEAN
 #endif
 
-static void init_structs(void);
-
 /* The "surrogateescape" error handler is new in Python 3.1 */
 #if PY_VERSION_HEX >= 0x030100f0
 # define CODEC_ERROR_HANDLER "surrogateescape"
@@ -610,8 +608,6 @@ get_py3_exceptions()
 }
 #endif /* DYNAMIC_PYTHON3 */
 
-static PyObject *BufferDir(PyObject *, PyObject *);
-
 static int py3initialised = 0;
 
 #define PYINITIALISED py3initialised
@@ -670,6 +666,7 @@ call_PyType_GenericAlloc(PyTypeObject *type, Py_ssize_t nitems)
     return PyType_GenericAlloc(type,nitems);
 }
 
+static PyObject *BufferDir(PyObject *);
 static PyObject *OutputGetattro(PyObject *, PyObject *);
 static int OutputSetattro(PyObject *, PyObject *, PyObject *);
 static PyObject *BufferGetattro(PyObject *, PyObject *);
@@ -703,8 +700,6 @@ static struct PyModuleDef vimmodule;
  * Internal function prototypes.
  */
 
-static PyObject *globals;
-
 static int PythonIO_Init(void);
 static PyObject *Py3Init_vim(void);
 
@@ -827,7 +822,7 @@ fail:
  * External interface
  */
     static void
-DoPy3Command(exarg_T *eap, const char *cmd, typval_T *rettv)
+DoPyCommand(const char *cmd, rangeinitializer init_range, runner run, void *arg)
 {
 #if defined(MACOS) && !defined(MACOS_X_UNIX)
     GrafPtr		oldPort;
@@ -848,16 +843,8 @@ DoPy3Command(exarg_T *eap, const char *cmd, typval_T *rettv)
     if (Python3_Init())
 	goto theend;
 
-    if (rettv == NULL)
-    {
-	RangeStart = eap->line1;
-	RangeEnd = eap->line2;
-    }
-    else
-    {
-	RangeStart = (PyInt) curwin->w_cursor.lnum;
-	RangeEnd = RangeStart;
-    }
+    init_range(arg);
+
     Python_Release_Vim();	    /* leave vim */
 
 #if defined(HAVE_LOCALE_H) || defined(X_LOCALE)
@@ -881,28 +868,8 @@ DoPy3Command(exarg_T *eap, const char *cmd, typval_T *rettv)
 					(char *)ENC_OPT, CODEC_ERROR_HANDLER);
     cmdbytes = PyUnicode_AsEncodedString(cmdstr, "utf-8", CODEC_ERROR_HANDLER);
     Py_XDECREF(cmdstr);
-    if (rettv == NULL)
-	PyRun_SimpleString(PyBytes_AsString(cmdbytes));
-    else
-    {
-	PyObject	*r;
 
-	r = PyRun_String(PyBytes_AsString(cmdbytes), Py_eval_input,
-			 globals, globals);
-	if (r == NULL)
-	{
-	    if (PyErr_Occurred() && !msg_silent)
-		PyErr_PrintEx(0);
-	    EMSG(_("E860: Eval did not return a valid python 3 object"));
-	}
-	else
-	{
-	    if (ConvertFromPyObject(r, rettv) == -1)
-		EMSG(_("E861: Failed to convert returned python 3 object to vim value"));
-	    Py_DECREF(r);
-	}
-	PyErr_Clear();
-    }
+    run(PyBytes_AsString(cmdbytes), arg, &pygilstate);
     Py_XDECREF(cmdbytes);
 
     PyGILState_Release(pygilstate);
@@ -936,10 +903,10 @@ ex_py3(exarg_T *eap)
     script = script_get(eap, eap->arg);
     if (!eap->skip)
     {
-	if (script == NULL)
-	    DoPy3Command(eap, (char *)eap->arg, NULL);
-	else
-	    DoPy3Command(eap, (char *)script, NULL);
+	DoPyCommand(script == NULL ? (char *) eap->arg : (char *) script,
+		(rangeinitializer) init_range_cmd,
+		(runner) run_cmd,
+		(void *) eap);
     }
     vim_free(script);
 }
@@ -1000,101 +967,19 @@ ex_py3file(exarg_T *eap)
 
 
     /* Execute the file */
-    DoPy3Command(eap, buffer, NULL);
+    DoPyCommand(buffer,
+	    (rangeinitializer) init_range_cmd,
+	    (runner) run_cmd,
+	    (void *) eap);
 }
 
     void
 ex_py3do(exarg_T *eap)
 {
-    linenr_T		i;
-    const char		*code_hdr = "def " DOPY_FUNC "(line, linenr):\n ";
-    const char		*s = (const char *) eap->arg;
-    size_t		len;
-    char		*code;
-    int			status;
-    PyObject		*pyfunc, *pymain;
-    PyGILState_STATE	pygilstate;
-
-    if (Python3_Init())
-	goto theend;
-
-    if (u_save(eap->line1 - 1, eap->line2 + 1) != OK)
-    {
-	EMSG(_("cannot save undo information"));
-	return;
-    }
-    len = strlen(code_hdr) + strlen(s);
-    code = malloc(len + 1);
-    STRCPY(code, code_hdr);
-    STRNCAT(code, s, len + 1);
-    pygilstate = PyGILState_Ensure();
-    status = PyRun_SimpleString(code);
-    vim_free(code);
-    if (status)
-    {
-	EMSG(_("failed to run the code"));
-	return;
-    }
-    status = 0; /* good */
-    pymain = PyImport_AddModule("__main__");
-    pyfunc = PyObject_GetAttrString(pymain, DOPY_FUNC);
-    PyGILState_Release(pygilstate);
-
-    for (i = eap->line1; i <= eap->line2; i++)
-    {
-	const char *line;
-	PyObject *pyline, *pylinenr, *pyret, *pybytes;
-
-	line = (char *)ml_get(i);
-	pygilstate = PyGILState_Ensure();
-	pyline = PyUnicode_Decode(line, strlen(line),
-		(char *)ENC_OPT, CODEC_ERROR_HANDLER);
-	pylinenr = PyLong_FromLong(i);
-	pyret = PyObject_CallFunctionObjArgs(pyfunc, pyline, pylinenr, NULL);
-	Py_DECREF(pyline);
-	Py_DECREF(pylinenr);
-	if (!pyret)
-	{
-	    PyErr_PrintEx(0);
-	    PythonIO_Flush();
-	    status = 1;
-	    goto out;
-	}
-
-	if (pyret && pyret != Py_None)
-	{
-	    if (!PyUnicode_Check(pyret))
-	    {
-		EMSG(_("E863: return value must be an instance of str"));
-		Py_XDECREF(pyret);
-		status = 1;
-		goto out;
-	    }
-	    pybytes = PyUnicode_AsEncodedString(pyret,
-		    (char *)ENC_OPT, CODEC_ERROR_HANDLER);
-	    ml_replace(i, (char_u *) PyBytes_AsString(pybytes), 1);
-	    Py_DECREF(pybytes);
-	    changed();
-#ifdef SYNTAX_HL
-	    syn_changed(i); /* recompute syntax hl. for this line */
-#endif
-	}
-	Py_XDECREF(pyret);
-	PythonIO_Flush();
-	PyGILState_Release(pygilstate);
-    }
-    pygilstate = PyGILState_Ensure();
-out:
-    Py_DECREF(pyfunc);
-    PyObject_SetAttrString(pymain, DOPY_FUNC, NULL);
-    PyGILState_Release(pygilstate);
-    if (status)
-	return;
-    check_cursor();
-    update_curbuf(NOT_VALID);
-
-theend:
-    return;
+    DoPyCommand((char *)eap->arg,
+	    (rangeinitializer)init_range_cmd,
+	    (runner)run_do,
+	    (void *)eap);
 }
 
 /******************************************************
@@ -1120,7 +1005,7 @@ OutputSetattro(PyObject *self, PyObject *nameobj, PyObject *val)
 {
     GET_ATTR_STRING(name, nameobj);
 
-    return OutputSetattr(self, name, val);
+    return OutputSetattr((OutputObject *)(self), name, val);
 }
 
 /***************/
@@ -1148,12 +1033,9 @@ PythonIO_Init(void)
 
 #define BufferType_Check(obj) ((obj)->ob_base.ob_type == &BufferType)
 
-static Py_ssize_t BufferLength(PyObject *);
-static PyObject *BufferItem(PyObject *, Py_ssize_t);
 static PyObject* BufferSubscript(PyObject *self, PyObject *idx);
 static Py_ssize_t BufferAsSubscript(PyObject *self, PyObject *idx, PyObject *val);
 
-
 /* Line range type - Implementation functions
  * --------------------------------------
  */
@@ -1209,7 +1091,7 @@ BufferGetattro(PyObject *self, PyObject*nameobj)
 }
 
     static PyObject *
-BufferDir(PyObject *self UNUSED, PyObject *args UNUSED)
+BufferDir(PyObject *self UNUSED)
 {
     return Py_BuildValue("[sssss]", "name", "number",
 						   "append", "mark", "range");
@@ -1223,7 +1105,7 @@ BufferSubscript(PyObject *self, PyObject* idx)
     if (PyLong_Check(idx))
     {
 	long _idx = PyLong_AsLong(idx);
-	return BufferItem(self,_idx);
+	return BufferItem((BufferObject *)(self), _idx);
     } else if (PySlice_Check(idx))
     {
 	Py_ssize_t start, stop, step, slicelen;
@@ -1238,7 +1120,7 @@ BufferSubscript(PyObject *self, PyObject* idx)
 	{
 	    return NULL;
 	}
-	return BufferSlice(self, start, stop);
+	return BufferSlice((BufferObject *)(self), start, stop);
     }
     else
     {
@@ -1342,7 +1224,7 @@ RangeSubscript(PyObject *self, PyObject* idx)
     if (PyLong_Check(idx))
     {
 	long _idx = PyLong_AsLong(idx);
-	return RangeItem(self,_idx);
+	return RangeItem((RangeObject *)(self), _idx);
     } else if (PySlice_Check(idx))
     {
 	Py_ssize_t start, stop, step, slicelen;
@@ -1354,7 +1236,7 @@ RangeSubscript(PyObject *self, PyObject* idx)
 	{
 	    return NULL;
 	}
-	return RangeSlice(self, start, stop);
+	return RangeSlice((RangeObject *)(self), start, stop);
     }
     else
     {
@@ -1435,7 +1317,7 @@ WindowSetattro(PyObject *self, PyObject *nameobj, PyObject *val)
 {
     GET_ATTR_STRING(name, nameobj);
 
-    return WindowSetattr(self, name, val);
+    return WindowSetattr((WindowObject *)(self), name, val);
 }
 
 /* Tab page list object - Definitions
@@ -1489,8 +1371,6 @@ CurrentSetattro(PyObject *self, PyObject *nameobj, PyObject *value)
 /* Dictionary object - Definitions
  */
 
-static PyInt DictionaryLength(PyObject *);
-
     static PyObject *
 DictionaryGetattro(PyObject *self, PyObject *nameobj)
 {
@@ -1510,15 +1390,12 @@ DictionaryGetattro(PyObject *self, PyObject *nameobj)
 DictionarySetattro(PyObject *self, PyObject *nameobj, PyObject *val)
 {
     GET_ATTR_STRING(name, nameobj);
-    return DictionarySetattr(self, name, val);
+    return DictionarySetattr((DictionaryObject *)(self), name, val);
 }
 
 /* List object - Definitions
  */
 
-static PyInt ListLength(PyObject *);
-static PyObject *ListItem(PyObject *, Py_ssize_t);
-
 static PySequenceMethods ListAsSeq = {
     (lenfunc)		ListLength,	 /* sq_length,	  len(x)   */
     (binaryfunc)	0,		 /* RangeConcat, sq_concat,  x+y   */
@@ -1542,21 +1419,21 @@ static PyMappingMethods ListAsMapping = {
 };
 
     static PyObject *
-ListSubscript(PyObject *self, PyObject* idxObject)
+ListSubscript(PyObject *self, PyObject* idx)
 {
-    if (PyLong_Check(idxObject))
+    if (PyLong_Check(idx))
     {
-	long idx = PyLong_AsLong(idxObject);
-	return ListItem(self, idx);
+	long _idx = PyLong_AsLong(idx);
+	return ListItem((ListObject *)(self), _idx);
     }
-    else if (PySlice_Check(idxObject))
+    else if (PySlice_Check(idx))
     {
 	Py_ssize_t start, stop, step, slicelen;
 
-	if (PySlice_GetIndicesEx(idxObject, ListLength(self), &start, &stop,
-				 &step, &slicelen) < 0)
+	if (PySlice_GetIndicesEx(idx, ListLength((ListObject *)(self)),
+				 &start, &stop, &step, &slicelen) < 0)
 	    return NULL;
-	return ListSlice(self, start, stop);
+	return ListSlice((ListObject *)(self), start, stop);
     }
     else
     {
@@ -1566,21 +1443,21 @@ ListSubscript(PyObject *self, PyObject* idxObject)
 }
 
     static Py_ssize_t
-ListAsSubscript(PyObject *self, PyObject *idxObject, PyObject *obj)
+ListAsSubscript(PyObject *self, PyObject *idx, PyObject *obj)
 {
-    if (PyLong_Check(idxObject))
+    if (PyLong_Check(idx))
     {
-	long idx = PyLong_AsLong(idxObject);
-	return ListAssItem(self, idx, obj);
+	long _idx = PyLong_AsLong(idx);
+	return ListAssItem((ListObject *)(self), _idx, obj);
     }
-    else if (PySlice_Check(idxObject))
+    else if (PySlice_Check(idx))
     {
 	Py_ssize_t start, stop, step, slicelen;
 
-	if (PySlice_GetIndicesEx(idxObject, ListLength(self), &start, &stop,
-				 &step, &slicelen) < 0)
+	if (PySlice_GetIndicesEx(idx, ListLength((ListObject *)(self)),
+				 &start, &stop, &step, &slicelen) < 0)
 	    return -1;
-	return ListAssSlice(self, start, stop, obj);
+	return ListAssSlice((ListObject *)(self), start, stop, obj);
     }
     else
     {
@@ -1604,7 +1481,7 @@ ListGetattro(PyObject *self, PyObject *nameobj)
 ListSetattro(PyObject *self, PyObject *nameobj, PyObject *val)
 {
     GET_ATTR_STRING(name, nameobj);
-    return ListSetattr(self, name, val);
+    return ListSetattr((ListObject *)(self), name, val);
 }
 
 /* Function object - Definitions
@@ -1790,7 +1667,10 @@ LineToString(const char *str)
     void
 do_py3eval (char_u *str, typval_T *rettv)
 {
-    DoPy3Command(NULL, (char *) str, rettv);
+    DoPyCommand((char *) str,
+	    (rangeinitializer) init_range_eval,
+	    (runner) run_eval,
+	    (void *) rettv);
     switch(rettv->v_type)
     {
 	case VAR_DICT: ++rettv->vval.v_dict->dv_refcount; break;

src/normal.c

@@ -701,6 +701,7 @@ normal_cmd(oap, toplevel)
 	else
 	    c = 'c';
 	msg_nowait = TRUE;	/* don't delay going to insert mode */
+	old_mapped_len = 0;	/* do go to Insert mode */
     }
 #endif
 

src/regexp.c

@@ -51,6 +51,8 @@
 # define BT_REGEXP_DUMP
 /* save the debugging data to a file instead of displaying it */
 # define BT_REGEXP_LOG
+# define BT_REGEXP_DEBUG_LOG
+# define BT_REGEXP_DEBUG_LOG_NAME	"bt_regexp_debug.log"
 #endif
 
 /*
@@ -5756,7 +5758,7 @@ regrepeat(p, maxcount)
       case SIDENT + ADD_NL:
 	while (count < maxcount)
 	{
-	    if (vim_isIDc(*scan) && (testval || !VIM_ISDIGIT(*scan)))
+	    if (vim_isIDc(PTR2CHAR(scan)) && (testval || !VIM_ISDIGIT(*scan)))
 	    {
 		mb_ptr_adv(scan);
 	    }
@@ -5817,7 +5819,7 @@ regrepeat(p, maxcount)
       case SFNAME + ADD_NL:
 	while (count < maxcount)
 	{
-	    if (vim_isfilec(*scan) && (testval || !VIM_ISDIGIT(*scan)))
+	    if (vim_isfilec(PTR2CHAR(scan)) && (testval || !VIM_ISDIGIT(*scan)))
 	    {
 		mb_ptr_adv(scan);
 	    }
@@ -7828,11 +7830,11 @@ vim_regcomp(expr_arg, re_flags)
 
     if (prog == NULL)	    /* error compiling regexp with initial engine */
     {
-#ifdef DEBUG
+#ifdef BT_REGEXP_DEBUG_LOG
 	if (regexp_engine != BACKTRACKING_ENGINE)   /* debugging log for NFA */
 	{
 	    FILE *f;
-	    f = fopen("debug.log", "a");
+	    f = fopen(BT_REGEXP_DEBUG_LOG_NAME, "a");
 	    if (f)
 	    {
 		if (!syntax_error)
@@ -7842,7 +7844,8 @@ vim_regcomp(expr_arg, re_flags)
 		fclose(f);
 	    }
 	    else
-		EMSG("(NFA) Could not open \"debug.log\" to write !!!");
+		EMSG2("(NFA) Could not open \"%s\" to write !!!",
+                        BT_REGEXP_DEBUG_LOG_NAME);
 	    /*
 	    if (syntax_error)
 		EMSG("NFA Regexp: Syntax Error !");

src/regexp_nfa.c

@@ -9,14 +9,14 @@
 /* Comment this out to disable log files. They can get pretty big */
 # define ENABLE_LOG
 # define LOG_NAME "log_nfarun.log"
+# define NFA_REGEXP_DEBUG_LOG
+# define NFA_REGEXP_DEBUG_LOG_NAME	"nfa_regexp_debug.log"
 #endif
 
 /* Upper limit allowed for {m,n} repetitions handled by NFA */
 #define	    NFA_BRACES_MAXLIMIT		    10
 /* For allocating space for the postfix representation */
 #define	    NFA_POSTFIX_MULTIPLIER	    (NFA_BRACES_MAXLIMIT + 2)*2
-/* Size of stack, used when converting the postfix regexp into NFA */
-#define	    NFA_STACK_SIZE		    1024
 
 enum
 {
@@ -224,16 +224,16 @@ nfa_regcomp_start(expr, re_flags)
     char_u	*expr;
     int		re_flags;	    /* see vim_regcomp() */
 {
-    int		postfix_size;
+    size_t	postfix_size;
 
     nstate = 0;
     istate = 0;
     /* A reasonable estimation for size */
-    nstate_max = (STRLEN(expr) + 1) * NFA_POSTFIX_MULTIPLIER;
+    nstate_max = (int)(STRLEN(expr) + 1) * NFA_POSTFIX_MULTIPLIER;
 
     /* Some items blow up in size, such as [A-z].  Add more space for that.
      * TODO: some patterns may still fail. */
-//    nstate_max += 1000;
+    nstate_max += 1000;
 
     /* Size for postfix representation of expr. */
     postfix_size = sizeof(*post_start) * nstate_max;
@@ -1826,13 +1826,13 @@ nfa_postfix_dump(expr, retval)
 	else if (retval == OK)
 	    fprintf(f, ">>> NFA engine succeeded !\n");
 	fprintf(f, "Regexp: \"%s\"\nPostfix notation (char): \"", expr);
-	for (p=post_start; *p; p++)
+	for (p = post_start; *p && p < post_end; p++)
 	{
 	    nfa_set_code(*p);
 	    fprintf(f, "%s, ", code);
 	}
 	fprintf(f, "\"\nPostfix notation (int): ");
-	for (p=post_start; *p; p++)
+	for (p = post_start; *p && p < post_end; p++)
 		fprintf(f, "%d ", *p);
 	fprintf(f, "\n\n");
 	fclose(f);
@@ -2143,6 +2143,7 @@ post2nfa(postfix, end, nfa_calc_size)
     nfa_state_T	*s;
     nfa_state_T	*s1;
     nfa_state_T	*matchstate;
+    nfa_state_T	*ret = NULL;
 
     if (postfix == NULL)
 	return NULL;
@@ -2158,9 +2159,9 @@ post2nfa(postfix, end, nfa_calc_size)
     if (nfa_calc_size == FALSE)
     {
 	/* Allocate space for the stack. Max states on the stack : nstate */
-	stack = (Frag_T *) lalloc((nstate + 1)*sizeof(Frag_T), TRUE);
+	stack = (Frag_T *) lalloc((nstate + 1) * sizeof(Frag_T), TRUE);
 	stackp = stack;
-	stack_end = stack + NFA_STACK_SIZE;
+	stack_end = stack + (nstate + 1);
     }
 
     for (p = postfix; p < end; ++p)
@@ -2177,7 +2178,7 @@ post2nfa(postfix, end, nfa_calc_size)
 	     * No new state added here. */
 	    if (nfa_calc_size == TRUE)
 	    {
-		nstate += 0;
+		/* nstate += 0; */
 		break;
 	    }
 	    e2 = POP();
@@ -2190,7 +2191,7 @@ post2nfa(postfix, end, nfa_calc_size)
 	    /* Negation of a character */
 	    if (nfa_calc_size == TRUE)
 	    {
-		nstate += 0;
+		/* nstate += 0; */
 		break;
 	    }
 	    e1 = POP();
@@ -2204,14 +2205,14 @@ post2nfa(postfix, end, nfa_calc_size)
 	    /* Alternation */
 	    if (nfa_calc_size == TRUE)
 	    {
-		nstate ++;
+		nstate++;
 		break;
 	    }
 	    e2 = POP();
 	    e1 = POP();
 	    s = new_state(NFA_SPLIT, e1.start, e2.start);
 	    if (s == NULL)
-		return NULL;
+		goto theend;
 	    PUSH(frag(s, append(e1.out, e2.out)));
 	    break;
 
@@ -2219,13 +2220,13 @@ post2nfa(postfix, end, nfa_calc_size)
 	    /* Zero or more */
 	    if (nfa_calc_size == TRUE)
 	    {
-		nstate ++;
+		nstate++;
 		break;
 	    }
 	    e = POP();
 	    s = new_state(NFA_SPLIT, e.start, NULL);
 	    if (s == NULL)
-		return NULL;
+		goto theend;
 	    patch(e.out, s);
 	    PUSH(frag(s, list1(&s->out1)));
 	    break;
@@ -2234,13 +2235,13 @@ post2nfa(postfix, end, nfa_calc_size)
 	    /* one or zero atoms=> greedy match */
 	    if (nfa_calc_size == TRUE)
 	    {
-		nstate ++;
+		nstate++;
 		break;
 	    }
 	    e = POP();
 	    s = new_state(NFA_SPLIT, e.start, NULL);
 	    if (s == NULL)
-		return NULL;
+		goto theend;
 	    PUSH(frag(s, append(e.out, list1(&s->out1))));
 	    break;
 
@@ -2248,13 +2249,13 @@ post2nfa(postfix, end, nfa_calc_size)
 	    /* zero or one atoms => non-greedy match */
 	    if (nfa_calc_size == TRUE)
 	    {
-		nstate ++;
+		nstate++;
 		break;
 	    }
 	    e = POP();
 	    s = new_state(NFA_SPLIT, NULL, e.start);
 	    if (s == NULL)
-		return NULL;
+		goto theend;
 	    PUSH(frag(s, append(e.out, list1(&s->out))));
 	    break;
 
@@ -2262,13 +2263,13 @@ post2nfa(postfix, end, nfa_calc_size)
 	    /* One or more */
 	    if (nfa_calc_size == TRUE)
 	    {
-		nstate ++;
+		nstate++;
 		break;
 	    }
 	    e = POP();
 	    s = new_state(NFA_SPLIT, e.start, NULL);
 	    if (s == NULL)
-		return NULL;
+		goto theend;
 	    patch(e.out, s);
 	    PUSH(frag(e.start, list1(&s->out1)));
 	    break;
@@ -2278,12 +2279,12 @@ post2nfa(postfix, end, nfa_calc_size)
 	     * with max/min count of 0 */
 	    if (nfa_calc_size == TRUE)
 	    {
-		nstate ++;
+		nstate++;
 		break;
 	    }
 	    s = new_state(NFA_SKIP_CHAR, NULL, NULL);
 	    if (s == NULL)
-		return NULL;
+		goto theend;
 	    PUSH(frag(s, list1(&s->out)));
 	    break;
 
@@ -2293,7 +2294,7 @@ post2nfa(postfix, end, nfa_calc_size)
 	     * END_INVISIBLE, similarly to MOPEN.
 	     */
 	    /* TODO: Maybe this drops the speed? */
-	    return NULL;
+	    goto theend;
 
 	    if (nfa_calc_size == TRUE)
 	    {
@@ -2303,12 +2304,12 @@ post2nfa(postfix, end, nfa_calc_size)
 	    e = POP();
 	    s1 = new_state(NFA_END_INVISIBLE, NULL, NULL);
 	    if (s1 == NULL)
-		return NULL;
+		goto theend;
 	    patch(e.out, s1);
 
 	    s = new_state(NFA_START_INVISIBLE, e.start, s1);
 	    if (s == NULL)
-		return NULL;
+		goto theend;
 	    PUSH(frag(s, list1(&s1->out)));
 	    break;
 
@@ -2357,10 +2358,10 @@ post2nfa(postfix, end, nfa_calc_size)
 	    {
 		s = new_state(mopen, NULL, NULL);
 		if (s == NULL)
-		    return NULL;
+		    goto theend;
 		s1 = new_state(mclose, NULL, NULL);
 		if (s1 == NULL)
-		    return NULL;
+		    goto theend;
 		patch(list1(&s->out), s1);
 		PUSH(frag(s, list1(&s1->out)));
 		break;
@@ -2371,11 +2372,11 @@ post2nfa(postfix, end, nfa_calc_size)
 	    e = POP();
 	    s = new_state(mopen, e.start, NULL);   /* `(' */
 	    if (s == NULL)
-		return NULL;
+		goto theend;
 
 	    s1 = new_state(mclose, NULL, NULL);   /* `)' */
 	    if (s1 == NULL)
-		return NULL;
+		goto theend;
 	    patch(e.out, s1);
 
 	    if (mopen == NFA_MULTIBYTE || mopen == NFA_COMPOSING)
@@ -2392,12 +2393,12 @@ post2nfa(postfix, end, nfa_calc_size)
 	    /* Operands */
 	    if (nfa_calc_size == TRUE)
 	    {
-		nstate ++;
+		nstate++;
 		break;
 	    }
 	    s = new_state(*p, NULL, NULL);
 	    if (s == NULL)
-		return NULL;
+		goto theend;
 	    PUSH(frag(s, list1(&s->out)));
 	    break;
 
@@ -2407,8 +2408,8 @@ post2nfa(postfix, end, nfa_calc_size)
 
     if (nfa_calc_size == TRUE)
     {
-	nstate ++;
-	return NULL;	/* Return value when counting size is ignored anyway */
+	nstate++;
+	goto theend;	/* Return value when counting size is ignored anyway */
     }
 
     e = POP();
@@ -2418,14 +2419,16 @@ post2nfa(postfix, end, nfa_calc_size)
     if (istate >= nstate)
 	EMSG_RET_NULL(_("E876: (NFA regexp) Not enough space to store the whole NFA "));
 
-    vim_free(stack);
-
     matchstate = &state_ptr[istate++]; /* the match state */
     matchstate->c = NFA_MATCH;
     matchstate->out = matchstate->out1 = NULL;
 
     patch(e.out, matchstate);
-    return e.start;
+    ret = e.start;
+
+theend:
+    vim_free(stack);
+    return ret;
 
 #undef POP1
 #undef PUSH1
@@ -2583,7 +2586,7 @@ addstate(l, state, m, off, lid, match)
 		save.startpos[subidx] = m->startpos[subidx];
 		save.endpos[subidx] = m->endpos[subidx];
 		m->startpos[subidx].lnum = reglnum;
-		m->startpos[subidx].col = reginput - regline + off;
+		m->startpos[subidx].col = (colnr_T)(reginput - regline + off);
 	    }
 	    else
 	    {
@@ -2631,7 +2634,7 @@ addstate(l, state, m, off, lid, match)
 		save.startpos[subidx] = m->startpos[subidx];
 		save.endpos[subidx] = m->endpos[subidx];
 		m->endpos[subidx].lnum = reglnum;
-		m->endpos[subidx].col = reginput - regline + off;
+		m->endpos[subidx].col = (colnr_T)(reginput - regline + off);
 	    }
 	    else
 	    {
@@ -2667,11 +2670,11 @@ check_char_class(class, c)
     switch (class)
     {
 	case NFA_CLASS_ALNUM:
-	    if (isalnum(c))
+	    if (c >= 1 && c <= 255 && isalnum(c))
 		return OK;
 	    break;
 	case NFA_CLASS_ALPHA:
-	    if (isalpha(c))
+	    if (c >= 1 && c <= 255 && isalpha(c))
 		return OK;
 	    break;
 	case NFA_CLASS_BLANK:
@@ -2679,7 +2682,7 @@ check_char_class(class, c)
 		return OK;
 	    break;
 	case NFA_CLASS_CNTRL:
-	    if (iscntrl(c))
+	    if (c >= 1 && c <= 255 && iscntrl(c))
 		return OK;
 	    break;
 	case NFA_CLASS_DIGIT:
@@ -2687,7 +2690,7 @@ check_char_class(class, c)
 		return OK;
 	    break;
 	case NFA_CLASS_GRAPH:
-	    if (isgraph(c))
+	    if (c >= 1 && c <= 255 && isgraph(c))
 		return OK;
 	    break;
 	case NFA_CLASS_LOWER:
@@ -2699,7 +2702,7 @@ check_char_class(class, c)
 		return OK;
 	    break;
 	case NFA_CLASS_PUNCT:
-	    if (ispunct(c))
+	    if (c >= 1 && c <= 255 && ispunct(c))
 		return OK;
 	    break;
 	case NFA_CLASS_SPACE:
@@ -2833,7 +2836,6 @@ nfa_regmatch(start, submatch, m)
     int		old_reglnum = -1;
     int		reginput_updated = FALSE;
     thread_T	*t;
-    char_u	*cc;
     char_u	*old_reginput = NULL;
     char_u	*old_regline = NULL;
     nfa_state_T	*sta;
@@ -2849,12 +2851,12 @@ nfa_regmatch(start, submatch, m)
     int		*listids = NULL;
     int		j = 0;
     int		len = 0;
-#ifdef DEBUG
-    FILE	*debug = fopen("list.log", "a");
+#ifdef NFA_REGEXP_DEBUG_LOG
+    FILE	*debug = fopen(NFA_REGEXP_DEBUG_LOG_NAME, "a");
 
     if (debug == NULL)
     {
-	EMSG(_("(NFA) COULD NOT OPEN list.log !"));
+	EMSG2(_("(NFA) COULD NOT OPEN %s !"), NFA_REGEXP_DEBUG_LOG_NAME);
 	return FALSE;
     }
 #endif
@@ -2931,7 +2933,6 @@ again:
 	}
 	if (c == NUL)
 	    n = 0;
-	cc = (char_u *)&c;
 
 	/* swap lists */
 	thislist = &list[flag];
@@ -2950,9 +2951,14 @@ again:
 	fprintf(log_fd, "\n");
 #endif
 
-#ifdef DEBUG
+#ifdef NFA_REGEXP_DEBUG_LOG
 	fprintf(debug, "\n-------------------\n");
 #endif
+	/*
+	 * If the state lists are empty we can stop.
+	 */
+	if (thislist->n == 0 && neglist->n == 0)
+	    break;
 
 	/* compute nextlist */
 	for (i = 0; i < thislist->n || neglist->n > 0; ++i)
@@ -2960,13 +2966,13 @@ again:
 	    if (neglist->n > 0)
 	    {
 		t = &neglist->t[0];
-		neglist->n --;
+		neglist->n--;
 		i--;
 	    }
 	    else
 		t = &thislist->t[i];
 
-#ifdef DEBUG
+#ifdef NFA_REGEXP_DEBUG_LOG
 	    nfa_set_code(t->state->c);
 	    fprintf(debug, "%s, ", code);
 #endif
@@ -3261,12 +3267,12 @@ again:
 		break;
 
 	    case NFA_KWORD:	/*  \k	*/
-		result = vim_iswordp(cc);
+		result = vim_iswordp(reginput);
 		ADD_POS_NEG_STATE(t->state);
 		break;
 
 	    case NFA_SKWORD:	/*  \K	*/
-		result = !VIM_ISDIGIT(c) && vim_iswordp(cc);
+		result = !VIM_ISDIGIT(c) && vim_iswordp(reginput);
 		ADD_POS_NEG_STATE(t->state);
 		break;
 
@@ -3281,12 +3287,12 @@ again:
 		break;
 
 	    case NFA_PRINT:	/*  \p	*/
-		result = ptr2cells(cc) == 1;
+		result = ptr2cells(reginput) == 1;
 		ADD_POS_NEG_STATE(t->state);
 		break;
 
 	    case NFA_SPRINT:	/*  \P	*/
-		result = !VIM_ISDIGIT(c) && ptr2cells(cc) == 1;
+		result = !VIM_ISDIGIT(c) && ptr2cells(reginput) == 1;
 		ADD_POS_NEG_STATE(t->state);
 		break;
 
@@ -3380,8 +3386,30 @@ again:
 		ADD_POS_NEG_STATE(t->state);
 		break;
 
+	    case NFA_MOPEN + 0:
+	    case NFA_MOPEN + 1:
+	    case NFA_MOPEN + 2:
+	    case NFA_MOPEN + 3:
+	    case NFA_MOPEN + 4:
+	    case NFA_MOPEN + 5:
+	    case NFA_MOPEN + 6:
+	    case NFA_MOPEN + 7:
+	    case NFA_MOPEN + 8:
+	    case NFA_MOPEN + 9:
+		/* handled below */
+		break;
+
+	    case NFA_SKIP_CHAR:
+	    case NFA_ZSTART:
+		/* TODO: should not happen? */
+		break;
+
 	    default:	/* regular character */
+		/* TODO: put this in #ifdef later */
+		if (t->state->c < -256)
+		    EMSGN("INTERNAL: Negative state char: %ld", t->state->c);
 		result = (no_Magic(t->state->c) == c);
+
 		if (!result)
 		    result = ireg_ic == TRUE
 				&& MB_TOLOWER(t->state->c) == MB_TOLOWER(c);
@@ -3436,7 +3464,7 @@ theend:
     if (listids != NULL)
 	vim_free(listids);
 #undef ADD_POS_NEG_STATE
-#ifdef DEBUG
+#ifdef NFA_REGEXP_DEBUG_LOG
     fclose(debug);
 #endif
 
@@ -3620,7 +3648,7 @@ nfa_regcomp(expr, re_flags)
     int		re_flags;
 {
     nfa_regprog_T	*prog;
-    int			prog_size;
+    size_t		prog_size;
     int			*postfix;
 
     if (expr == NULL)

src/testdir/test64.in

@@ -286,13 +286,18 @@ STARTTEST
 
 :"""" Combining different tests and features
 :call add(tl, ['[[:alpha:]]\{-2,6}', '787abcdiuhsasiuhb4', 'ab'])
-:call add(tl, ['[^[=a=]]\+', 'ddaãâbcd', 'dd'])
 :call add(tl, ['', 'abcd', ''])
 :call add(tl, ['\v(())', 'any possible text', ''])
 :call add(tl, ['\v%(ab(xyz)c)', '   abxyzc ', 'abxyzc', 'xyz'])
 :call add(tl, ['\v(test|)empty', 'tesempty', 'empty', ''])
 :call add(tl, ['\v(a|aa)(a|aa)', 'aaa', 'aa', 'a', 'a'])
 
+:"""" \%u and friends
+:call add(tl, ['\%d32', 'yes no', ' '])
+:call add(tl, ['\%o40', 'yes no', ' '])
+:call add(tl, ['\%x20', 'yes no', ' '])
+:call add(tl, ['\%u0020', 'yes no', ' '])
+:call add(tl, ['\%U00000020', 'yes no', ' '])
 
 :"""" Run the tests
 

src/testdir/test64.ok

@@ -221,10 +221,14 @@ OK - .*John\&.*Bob
 OK - .*John\&.*Bob
 OK - \v(test1)@=.*yep
 OK - [[:alpha:]]\{-2,6}
-OK - [^[=a=]]\+
 OK - 
 OK - \v(())
 OK - \v%(ab(xyz)c)
 OK - \v(test|)empty
 OK - \v(a|aa)(a|aa)
+OK - \%d32
+OK - \%o40
+OK - \%x20
+OK - \%u0020
+OK - \%U00000020
 192.168.0.1

src/testdir/test87.ok

@@ -59,10 +59,10 @@ ll:[1]
 ['c', 1]
 ['d', ['e']]
 0.0
-"\0":	Vim(let):E861:
-{"\0": 1}:	Vim(let):E861:
+"\0":	Vim(let):E859:
+{"\0": 1}:	Vim(let):E859:
 undefined_name:	Vim(let):Trace
-vim:	Vim(let):E861:
+vim:	Vim(let):E859:
 [1]
 [1, 10, 11, 10, 11, 10, 11, 10, 11, 10, 11, 10, 1]
 Abc

src/testdir/test95.in

@@ -1,4 +1,4 @@
-Test for regexp patterns with multi-byte support.
+Test for regexp patterns with multi-byte support, using utf-8.
 See test64 for the non-multi-byte tests.
 
 A pattern that gives the expected result produces OK, so that we know it was
@@ -7,6 +7,7 @@ actually tried.
 STARTTEST
 :so small.vim
 :so mbyte.vim
+:set nocp encoding=utf-8 viminfo+=nviminfo
 :" tl is a List of Lists with:
 :"    regexp pattern
 :"    text to test the pattern on
@@ -25,6 +26,18 @@ STARTTEST
 :call add(tl, [' [^ ]\+', 'start มabcdม ', ' มabcdม'])
 :call add(tl, ['[ม[:alpha:][=a=]]\+', '879 aiaãมâมaiuvna ', 'aiaãมâมaiuvna'])
 
+:" this is not a normal "i" but 0xec
+:call add(tl, ['\p\+', 'ìa', 'ìa'])
+
+:"""" Test recognition of some character classes
+:call add(tl, ['\i\+', '&*§xx ', 'xx'])
+:call add(tl, ['\%#=1\i\+', '&*§xx ', 'xx'])
+:call add(tl, ['\f\+', '&*Ÿfname ', 'fname'])
+:call add(tl, ['\%#=1\i\+', '&*Ÿfname ', 'fname'])
+
+:"""" Combining different tests and features
+:call add(tl, ['[^[=a=]]\+', 'ddaãâbcd', 'dd'])
+
 :"""" Run the tests
 
 :"

src/testdir/test95.ok

@@ -4,3 +4,9 @@ OK - [[=a=]]\+
 OK - [^ม ]\+
 OK -  [^ ]\+
 OK - [ม[:alpha:][=a=]]\+
+OK - \p\+
+OK - \i\+
+OK - \%#=1\i\+
+OK - \f\+
+OK - \%#=1\i\+
+OK - [^[=a=]]\+

src/version.c

@@ -728,6 +728,42 @@ static char *(features[]) =
 
 static int included_patches[] =
 {   /* Add new patch number below this line */
+/**/
+    994,
+/**/
+    993,
+/**/
+    992,
+/**/
+    991,
+/**/
+    990,
+/**/
+    989,
+/**/
+    988,
+/**/
+    987,
+/**/
+    986,
+/**/
+    985,
+/**/
+    984,
+/**/
+    983,
+/**/
+    982,
+/**/
+    981,
+/**/
+    980,
+/**/
+    979,
+/**/
+    978,
+/**/
+    977,
 /**/
     976,
 /**/