Skip to content

Commit 488fa49

Browse files
author
Victor Stinner
committed
Rewrite PyUnicode_Append(); unicode_modifiable() is more strict
* Rename unicode_resizable() to unicode_modifiable() * Rename _PyUnicode_Dirty() to unicode_check_modifiable() to make it clear that the function is private * Inline PyUnicode_Concat() and unicode_append_inplace() in PyUnicode_Append() to simplify the code * unicode_modifiable() return 0 if the hash has been computed or if the string is not an exact unicode string * Remove _PyUnicode_DIRTY(): no need to reset the hash anymore, because if the hash has already been computed, you cannot modify a string inplace anymore * PyUnicode_Concat() checks for integer overflow
1 parent 24c74be commit 488fa49

1 file changed

Lines changed: 84 additions & 79 deletions

File tree

‎Objects/unicodeobject.c‎

Lines changed: 84 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -165,9 +165,6 @@ extern "C" {
165165
*_to++ = (to_type) *_iter++; \
166166
} while (0)
167167

168-
/* The Unicode string has been modified: reset the hash */
169-
#define _PyUnicode_DIRTY(op) do { _PyUnicode_HASH(op) = -1; } while (0)
170-
171168
/* This dictionary holds all interned unicode strings. Note that references
172169
to strings in this dictionary are *not* counted in the string's ob_refcnt.
173170
When the interned string reaches a refcnt of 0 the string deallocation
@@ -226,6 +223,8 @@ static void copy_characters(
226223
PyObject *to, Py_ssize_t to_start,
227224
PyObject *from, Py_ssize_t from_start,
228225
Py_ssize_t how_many);
226+
static int unicode_modifiable(PyObject *unicode);
227+
229228

230229
static PyObject *
231230
unicode_fromascii(const unsigned char *s, Py_ssize_t size);
@@ -645,10 +644,11 @@ resize_compact(PyObject *unicode, Py_ssize_t length)
645644
Py_ssize_t new_size;
646645
int share_wstr;
647646
PyObject *new_unicode;
648-
649647
assert(PyUnicode_IS_READY(unicode));
648+
assert(PyUnicode_IS_COMPACT(unicode));
649+
650650
char_size = PyUnicode_KIND(unicode);
651-
if (PyUnicode_IS_COMPACT_ASCII(unicode))
651+
if (PyUnicode_IS_ASCII(unicode))
652652
struct_size = sizeof(PyASCIIObject);
653653
else
654654
struct_size = sizeof(PyCompactUnicodeObject);
@@ -676,7 +676,7 @@ resize_compact(PyObject *unicode, Py_ssize_t length)
676676
_PyUnicode_LENGTH(unicode) = length;
677677
if (share_wstr) {
678678
_PyUnicode_WSTR(unicode) = PyUnicode_DATA(unicode);
679-
if (!PyUnicode_IS_COMPACT_ASCII(unicode))
679+
if (!PyUnicode_IS_ASCII(unicode))
680680
_PyUnicode_WSTR_LENGTH(unicode) = length;
681681
}
682682
PyUnicode_WRITE(PyUnicode_KIND(unicode), PyUnicode_DATA(unicode),
@@ -691,8 +691,6 @@ resize_inplace(PyObject *unicode, Py_ssize_t length)
691691
assert(!PyUnicode_IS_COMPACT(unicode));
692692
assert(Py_REFCNT(unicode) == 1);
693693

694-
_PyUnicode_DIRTY(unicode);
695-
696694
if (PyUnicode_IS_READY(unicode)) {
697695
Py_ssize_t char_size;
698696
Py_ssize_t new_size;
@@ -1115,15 +1113,13 @@ unicode_convert_wchar_to_ucs4(const wchar_t *begin, const wchar_t *end,
11151113
#endif
11161114

11171115
static int
1118-
_PyUnicode_Dirty(PyObject *unicode)
1116+
unicode_check_modifiable(PyObject *unicode)
11191117
{
1120-
assert(_PyUnicode_CHECK(unicode));
1121-
if (Py_REFCNT(unicode) != 1) {
1118+
if (!unicode_modifiable(unicode)) {
11221119
PyErr_SetString(PyExc_SystemError,
1123-
"Cannot modify a string having more than 1 reference");
1120+
"Cannot modify a string currently used");
11241121
return -1;
11251122
}
1126-
_PyUnicode_DIRTY(unicode);
11271123
return 0;
11281124
}
11291125

@@ -1289,7 +1285,7 @@ PyUnicode_CopyCharacters(PyObject *to, Py_ssize_t to_start,
12891285
if (how_many == 0)
12901286
return 0;
12911287

1292-
if (_PyUnicode_Dirty(to))
1288+
if (unicode_check_modifiable(to))
12931289
return -1;
12941290

12951291
err = _copy_characters(to, to_start, from, from_start, how_many, 1);
@@ -1537,12 +1533,17 @@ unicode_is_singleton(PyObject *unicode)
15371533
#endif
15381534

15391535
static int
1540-
unicode_resizable(PyObject *unicode)
1536+
unicode_modifiable(PyObject *unicode)
15411537
{
1538+
assert(_PyUnicode_CHECK(unicode));
15421539
if (Py_REFCNT(unicode) != 1)
15431540
return 0;
1541+
if (_PyUnicode_HASH(unicode) != -1)
1542+
return 0;
15441543
if (PyUnicode_CHECK_INTERNED(unicode))
15451544
return 0;
1545+
if (!PyUnicode_CheckExact(unicode))
1546+
return 0;
15461547
#ifdef Py_DEBUG
15471548
/* singleton refcount is greater than 1 */
15481549
assert(!unicode_is_singleton(unicode));
@@ -1577,7 +1578,7 @@ unicode_resize(PyObject **p_unicode, Py_ssize_t length)
15771578
return 0;
15781579
}
15791580

1580-
if (!unicode_resizable(unicode)) {
1581+
if (!unicode_modifiable(unicode)) {
15811582
PyObject *copy = resize_copy(unicode, length);
15821583
if (copy == NULL)
15831584
return -1;
@@ -3591,11 +3592,12 @@ PyUnicode_WriteChar(PyObject *unicode, Py_ssize_t index, Py_UCS4 ch)
35913592
PyErr_BadArgument();
35923593
return -1;
35933594
}
3595+
assert(PyUnicode_IS_READY(unicode));
35943596
if (index < 0 || index >= PyUnicode_GET_LENGTH(unicode)) {
35953597
PyErr_SetString(PyExc_IndexError, "string index out of range");
35963598
return -1;
35973599
}
3598-
if (_PyUnicode_Dirty(unicode))
3600+
if (unicode_check_modifiable(unicode))
35993601
return -1;
36003602
PyUnicode_WRITE(PyUnicode_KIND(unicode), PyUnicode_DATA(unicode),
36013603
index, ch);
@@ -10566,6 +10568,7 @@ PyUnicode_Concat(PyObject *left, PyObject *right)
1056610568
{
1056710569
PyObject *u = NULL, *v = NULL, *w;
1056810570
Py_UCS4 maxchar, maxchar2;
10571+
Py_ssize_t u_len, v_len, new_len;
1056910572

1057010573
/* Coerce the two arguments */
1057110574
u = PyUnicode_FromObject(left);
@@ -10585,18 +10588,25 @@ PyUnicode_Concat(PyObject *left, PyObject *right)
1058510588
return v;
1058610589
}
1058710590

10591+
u_len = PyUnicode_GET_LENGTH(u);
10592+
v_len = PyUnicode_GET_LENGTH(v);
10593+
if (u_len > PY_SSIZE_T_MAX - v_len) {
10594+
PyErr_SetString(PyExc_OverflowError,
10595+
"strings are too large to concat");
10596+
goto onError;
10597+
}
10598+
new_len = u_len + v_len;
10599+
1058810600
maxchar = PyUnicode_MAX_CHAR_VALUE(u);
1058910601
maxchar2 = PyUnicode_MAX_CHAR_VALUE(v);
1059010602
maxchar = Py_MAX(maxchar, maxchar2);
1059110603

1059210604
/* Concat the two Unicode strings */
10593-
w = PyUnicode_New(
10594-
PyUnicode_GET_LENGTH(u) + PyUnicode_GET_LENGTH(v),
10595-
maxchar);
10605+
w = PyUnicode_New(new_len, maxchar);
1059610606
if (w == NULL)
1059710607
goto onError;
10598-
copy_characters(w, 0, u, 0, PyUnicode_GET_LENGTH(u));
10599-
copy_characters(w, PyUnicode_GET_LENGTH(u), v, 0, PyUnicode_GET_LENGTH(v));
10608+
copy_characters(w, 0, u, 0, u_len);
10609+
copy_characters(w, u_len, v, 0, v_len);
1060010610
Py_DECREF(u);
1060110611
Py_DECREF(v);
1060210612
assert(_PyUnicode_CheckConsistency(w, 1));
@@ -10608,49 +10618,12 @@ PyUnicode_Concat(PyObject *left, PyObject *right)
1060810618
return NULL;
1060910619
}
1061010620

10611-
static void
10612-
unicode_append_inplace(PyObject **p_left, PyObject *right)
10613-
{
10614-
Py_ssize_t left_len, right_len, new_len;
10615-
10616-
assert(PyUnicode_IS_READY(*p_left));
10617-
assert(PyUnicode_IS_READY(right));
10618-
10619-
left_len = PyUnicode_GET_LENGTH(*p_left);
10620-
right_len = PyUnicode_GET_LENGTH(right);
10621-
if (left_len > PY_SSIZE_T_MAX - right_len) {
10622-
PyErr_SetString(PyExc_OverflowError,
10623-
"strings are too large to concat");
10624-
goto error;
10625-
}
10626-
new_len = left_len + right_len;
10627-
10628-
/* Now we own the last reference to 'left', so we can resize it
10629-
* in-place.
10630-
*/
10631-
if (unicode_resize(p_left, new_len) != 0) {
10632-
/* XXX if _PyUnicode_Resize() fails, 'left' has been
10633-
* deallocated so it cannot be put back into
10634-
* 'variable'. The MemoryError is raised when there
10635-
* is no value in 'variable', which might (very
10636-
* remotely) be a cause of incompatibilities.
10637-
*/
10638-
goto error;
10639-
}
10640-
/* copy 'right' into the newly allocated area of 'left' */
10641-
copy_characters(*p_left, left_len, right, 0, right_len);
10642-
_PyUnicode_DIRTY(*p_left);
10643-
return;
10644-
10645-
error:
10646-
Py_DECREF(*p_left);
10647-
*p_left = NULL;
10648-
}
10649-
1065010621
void
1065110622
PyUnicode_Append(PyObject **p_left, PyObject *right)
1065210623
{
1065310624
PyObject *left, *res;
10625+
Py_UCS4 maxchar, maxchar2;
10626+
Py_ssize_t left_len, right_len, new_len;
1065410627

1065510628
if (p_left == NULL) {
1065610629
if (!PyErr_Occurred())
@@ -10669,34 +10642,66 @@ PyUnicode_Append(PyObject **p_left, PyObject *right)
1066910642
if (PyUnicode_READY(right))
1067010643
goto error;
1067110644

10672-
if (PyUnicode_CheckExact(left) && left != unicode_empty
10673-
&& PyUnicode_CheckExact(right) && right != unicode_empty
10674-
&& unicode_resizable(left)
10675-
&& (_PyUnicode_KIND(right) <= _PyUnicode_KIND(left)
10676-
|| _PyUnicode_WSTR(left) != NULL))
10677-
{
10645+
/* Shortcuts */
10646+
if (left == unicode_empty) {
10647+
Py_DECREF(left);
10648+
Py_INCREF(right);
10649+
*p_left = right;
10650+
return;
10651+
}
10652+
if (right == unicode_empty)
10653+
return;
10654+
10655+
left_len = PyUnicode_GET_LENGTH(left);
10656+
right_len = PyUnicode_GET_LENGTH(right);
10657+
if (left_len > PY_SSIZE_T_MAX - right_len) {
10658+
PyErr_SetString(PyExc_OverflowError,
10659+
"strings are too large to concat");
10660+
goto error;
10661+
}
10662+
new_len = left_len + right_len;
10663+
10664+
if (unicode_modifiable(left)
10665+
&& PyUnicode_CheckExact(right)
10666+
&& PyUnicode_KIND(right) <= PyUnicode_KIND(left)
1067810667
/* Don't resize for ascii += latin1. Convert ascii to latin1 requires
1067910668
to change the structure size, but characters are stored just after
1068010669
the structure, and so it requires to move all characters which is
1068110670
not so different than duplicating the string. */
10682-
if (!(PyUnicode_IS_ASCII(left) && !PyUnicode_IS_ASCII(right)))
10683-
{
10684-
unicode_append_inplace(p_left, right);
10685-
assert(p_left == NULL || _PyUnicode_CheckConsistency(*p_left, 1));
10686-
return;
10671+
&& !(PyUnicode_IS_ASCII(left) && !PyUnicode_IS_ASCII(right)))
10672+
{
10673+
/* append inplace */
10674+
if (unicode_resize(p_left, new_len) != 0) {
10675+
/* XXX if _PyUnicode_Resize() fails, 'left' has been
10676+
* deallocated so it cannot be put back into
10677+
* 'variable'. The MemoryError is raised when there
10678+
* is no value in 'variable', which might (very
10679+
* remotely) be a cause of incompatibilities.
10680+
*/
10681+
goto error;
1068710682
}
10683+
/* copy 'right' into the newly allocated area of 'left' */
10684+
copy_characters(*p_left, left_len, right, 0, right_len);
1068810685
}
10686+
else {
10687+
maxchar = PyUnicode_MAX_CHAR_VALUE(left);
10688+
maxchar2 = PyUnicode_MAX_CHAR_VALUE(right);
10689+
maxchar = Py_MAX(maxchar, maxchar2);
1068910690

10690-
res = PyUnicode_Concat(left, right);
10691-
if (res == NULL)
10692-
goto error;
10693-
Py_DECREF(left);
10694-
*p_left = res;
10691+
/* Concat the two Unicode strings */
10692+
res = PyUnicode_New(new_len, maxchar);
10693+
if (res == NULL)
10694+
goto error;
10695+
copy_characters(res, 0, left, 0, left_len);
10696+
copy_characters(res, left_len, right, 0, right_len);
10697+
Py_DECREF(left);
10698+
*p_left = res;
10699+
}
10700+
assert(_PyUnicode_CheckConsistency(*p_left, 1));
1069510701
return;
1069610702

1069710703
error:
10698-
Py_DECREF(*p_left);
10699-
*p_left = NULL;
10704+
Py_CLEAR(*p_left);
1070010705
}
1070110706

1070210707
void

0 commit comments

Comments
 (0)