Public interface

The public interface for the library. More...

Functions
UTF8_API size_t	utf8len (const char *text)
	Get the length in code points of a UTF-8 encoded string. More...
UTF8_API size_t	utf16toutf8 (const utf16_t *input, size_t inputSize, char *target, size_t targetSize, int32_t *errors)
	Convert a UTF-16 encoded string to a UTF-8 encoded string. More...
UTF8_API size_t	utf32toutf8 (const unicode_t *input, size_t inputSize, char *target, size_t targetSize, int32_t *errors)
	Convert a UTF-32 encoded string to a UTF-8 encoded string. More...
UTF8_API size_t	widetoutf8 (const wchar_t *input, size_t inputSize, char *target, size_t targetSize, int32_t *errors)
	Convert a wide string to a UTF-8 encoded string. More...
UTF8_API size_t	utf8toutf16 (const char *input, size_t inputSize, utf16_t *target, size_t targetSize, int32_t *errors)
	Convert a UTF-8 encoded string to a UTF-16 encoded string. More...
UTF8_API size_t	utf8toutf32 (const char *input, size_t inputSize, unicode_t *target, size_t targetSize, int32_t *errors)
	Convert a UTF-8 encoded string to a UTF-32 encoded string. More...
UTF8_API size_t	utf8towide (const char *input, size_t inputSize, wchar_t *target, size_t targetSize, int32_t *errors)
	Convert a UTF-8 encoded string to a wide string. More...
UTF8_API const char *	utf8seek (const char *text, size_t textSize, const char *textStart, off_t offset, int direction)
	Seek into a UTF-8 encoded string. More...
UTF8_API size_t	utf8envlocale ()
	Returns the environment's locale as an enum value. More...
UTF8_API size_t	utf8toupper (const char *input, size_t inputSize, char *target, size_t targetSize, size_t locale, int32_t *errors)
	Convert UTF-8 encoded text to uppercase. More...
UTF8_API size_t	utf8tolower (const char *input, size_t inputSize, char *target, size_t targetSize, size_t locale, int32_t *errors)
	Convert UTF-8 encoded text to lowercase. More...
UTF8_API size_t	utf8totitle (const char *input, size_t inputSize, char *target, size_t targetSize, size_t locale, int32_t *errors)
	Convert UTF-8 encoded text to titlecase. More...
UTF8_API size_t	utf8casefold (const char *input, size_t inputSize, char *target, size_t targetSize, size_t locale, int32_t *errors)
	Remove case distinction from UTF-8 encoded text. More...
UTF8_API uint8_t	utf8isnormalized (const char *input, size_t inputSize, size_t flags, size_t *offset)
	Check if a string is stable in the specified Unicode Normalization Form. More...
UTF8_API size_t	utf8normalize (const char *input, size_t inputSize, char *target, size_t targetSize, size_t flags, int32_t *errors)
	Normalize a string to the specified Unicode Normalization Form. More...
UTF8_API size_t	utf8iscategory (const char *input, size_t inputSize, size_t flags)
	Check if the input string conforms to the category specified by the flags. More...

Detailed Description

The public interface for the library.

Function Documentation

utf8len()

UTF8_API size_t utf8len

(

const char *

text

)

Get the length in code points of a UTF-8 encoded string.

Example:

uint8_t CheckPassword(const char* password)
{
    size_t length = utf8len(password);
    return (length == utf8len("hunter2"));
}

Parameters

[in]

text

UTF-8 encoded string.

Returns

Length in code points.

utf16toutf8()

UTF8_API size_t utf16toutf8	(	const utf16_t *	input,
		size_t	inputSize,
		char *	target,
		size_t	targetSize,
		int32_t *	errors
	)

Convert a UTF-16 encoded string to a UTF-8 encoded string.

Note

This function should only be called directly if you are positive that you are working with UTF-16 encoded text. If you're working with wide strings, take a look at widetoutf8 instead.

Example:

uint8_t Player_SetNameUtf16(const utf16_t* name, size_t nameSize)
{
    char buffer[256];
    size_t buffer_size = 255;
    size_t converted_size;
    int32_t errors;
 
    if ((converted_size = utf16toutf8(name, nameSize, buffer, buffer_size, &errors)) == 0 ||
        errors != UTF8_ERR_NONE)
    {
        return 0;
    }
    buffer[converted_size] = 0;
 
    return Player_SetName(converted_name);
}

Parameters

[in]	input	UTF-16 encoded string.
[in]	inputSize	Size of the input in bytes.
[out]	target	Output buffer for the result, can be NULL.
[in]	targetSize	Size of the output buffer in bytes.
[out]	errors	Output for errors.

Returns

Amount of bytes needed for storing output.

Return values

UTF8_ERR_NONE	No errors.
UTF8_ERR_INVALID_DATA	Failed to decode data.
UTF8_ERR_OVERLAPPING_PARAMETERS	Input and output buffers overlap in memory.
UTF8_ERR_NOT_ENOUGH_SPACE	Target buffer size is insufficient for result.

See also

utf32toutf8

widetoutf8

utf32toutf8()

UTF8_API size_t utf32toutf8	(	const unicode_t *	input,
		size_t	inputSize,
		char *	target,
		size_t	targetSize,
		int32_t *	errors
	)

Convert a UTF-32 encoded string to a UTF-8 encoded string.

Note

This function should only be called directly if you are positive that you are working with UTF-32 encoded text. If you're working with wide strings, take a look at widetoutf8 instead.

Example:

uint8_t Database_ExecuteQuery_Unicode(const unicode_t* query, size_t querySize)
{
    char* converted = NULL;
    size_t converted_size;
    uint8_t result = 0;
    int32_t errors;
 
    if ((converted_size = utf32toutf8(query, querySize, NULL, 0, &errors)) == 0 ||
        errors != UTF8_ERR_NONE)
    {
        goto cleanup;
    }
 
    converted = (char*)malloc(converted_size + 1);
    utf32toutf8(query, querySize, converted, converted_size, NULL);
    converted[converted_size] = 0;
 
    result = Database_ExecuteQuery(converted);
 
cleanup:
    if (converted != NULL)
    {
        free(converted);
        converted = 0;
    }
 
    return result;
}

Parameters

[in]	input	UTF-32 encoded string.
[in]	inputSize	Size of the input in bytes.
[out]	target	Output buffer for the result, can be NULL.
[in]	targetSize	Size of the output buffer in bytes.
[out]	errors	Output for errors.

Returns

Amount of bytes needed for storing output.

Return values

UTF8_ERR_NONE	No errors.
UTF8_ERR_INVALID_DATA	Failed to decode data.
UTF8_ERR_OVERLAPPING_PARAMETERS	Input and output buffers overlap in memory.
UTF8_ERR_NOT_ENOUGH_SPACE	Target buffer size is insufficient for result.

See also

utf16toutf8

widetoutf8

widetoutf8()

UTF8_API size_t widetoutf8	(	const wchar_t *	input,
		size_t	inputSize,
		char *	target,
		size_t	targetSize,
		int32_t *	errors
	)

Convert a wide string to a UTF-8 encoded string.

Depending on the platform, wide strings are either UTF-16 or UTF-32 encoded. This function takes a wide string as input and automatically calls the correct conversion function.

This allows for a cross-platform treatment of wide text and is preferable to using the UTF-16 or UTF-32 versions directly.

Example:

texture_t Texture_Load_Wide(const wchar_t* input)
{
    char* converted = NULL;
    size_t converted_size;
    size_t input_size = wcslen(input) * sizeof(wchar_t);
    texture_t result = NULL;
    int32_t errors;
 
    if ((converted_size = widetoutf8(input, input_size, NULL, 0, &errors)) == 0 ||
        errors != UTF8_ERR_NONE)
    {
        goto cleanup;
    }
 
    converted = (char*)malloc(converted_size + 1);
    widetoutf8(input, input_size, converted, converted_size, NULL);
    converted[converted_size / sizeof(wchar_t)] = 0;
 
    result = Texture_Load(converted);
 
cleanup:
    if (converted != NULL)
    {
        free(converted);
        converted = NULL;
    }
 
    return result;
}

Parameters

[in]	input	Wide-encoded string.
[in]	inputSize	Size of the input in bytes.
[out]	target	Output buffer for the result, can be NULL.
[in]	targetSize	Size of the output buffer in bytes.
[out]	errors	Output for errors.

Returns

Amount of bytes needed for storing output.

Return values

UTF8_ERR_NONE	No errors.
UTF8_ERR_INVALID_DATA	Failed to decode data.
UTF8_ERR_OVERLAPPING_PARAMETERS	Input and output buffers overlap in memory.
UTF8_ERR_NOT_ENOUGH_SPACE	Target buffer size is insufficient for result.

See also

utf8towide

utf16toutf8

utf32toutf8

utf8toutf16()

UTF8_API size_t utf8toutf16	(	const char *	input,
		size_t	inputSize,
		utf16_t *	target,
		size_t	targetSize,
		int32_t *	errors
	)

Convert a UTF-8 encoded string to a UTF-16 encoded string.

Note

This function should only be called directly if you are positive that you must convert to UTF-16, independent of platform. If you're working with wide strings, take a look at utf8towide instead.

Erroneous byte sequences such as missing or illegal bytes or overlong encoding of code points (e.g. using five bytes to encode a sequence that can be represented by two bytes) are converted to the replacement character U+FFFD.

Code points outside the Basic Multilingual Plane (BMP) will be converted to surrogate pairs, which use four bytes instead of two.

Example:

void Font_DrawText(int x, int y, const char* text)
{
    utf16_t buffer[256];
    size_t buffer_size = 255 * sizeof(utf16_t);
    int32_t errors;
    
    size_t converted_size = utf8toutf16(text, strlen(text), buffer, buffer_size, &errors);
    if (converted_size > 0 &&
        errors == UTF8_ERR_NONE)
    {
        Legacy_DrawText(g_FontCurrent, x, y, (unsigned short*)buffer, converted_size / sizeof(utf16_t));
    }
}

Parameters

[in]	input	UTF-8 encoded string.
[in]	inputSize	Size of the input in bytes.
[out]	target	Output buffer for the result, can be NULL.
[in]	targetSize	Size of the output buffer in bytes.
[out]	errors	Output for errors.

Returns

Amount of bytes needed for storing output.

Return values

UTF8_ERR_NONE	No errors.
UTF8_ERR_INVALID_DATA	Failed to decode data.
UTF8_ERR_OVERLAPPING_PARAMETERS	Input and output buffers overlap in memory.
UTF8_ERR_NOT_ENOUGH_SPACE	Target buffer size is insufficient for result.

See also

utf8towide

utf8toutf32

utf8toutf32()

UTF8_API size_t utf8toutf32	(	const char *	input,
		size_t	inputSize,
		unicode_t *	target,
		size_t	targetSize,
		int32_t *	errors
	)

Convert a UTF-8 encoded string to a UTF-32 encoded string.

Note

This function should only be called directly if you are positive that you must convert to UTF-32, independent of platform. If you're working with wide strings, take a look at utf8towide instead.

Erroneous byte sequences such as missing or illegal bytes or overlong encoding of code points (e.g. using five bytes to encode a sequence that can be represented by two bytes) are converted to the replacement character U+FFFD.

Example:

void TextField_AddCharacter(const char* encoded)
{
    unicode_t code_point = 0;
    int32_t errors;
 
    utf8toutf32(encoded, strlen(encoded), &code_point, sizeof(unicode_t), &errors);
    if (errors == UTF8_ERR_NONE)
    {
        TextField_AddCodePoint(code_point);
    }
}

Parameters

[in]	input	UTF-8 encoded string.
[in]	inputSize	Size of the input in bytes.
[out]	target	Output buffer for the result, can be NULL.
[in]	targetSize	Size of the output buffer in bytes.
[out]	errors	Output for errors.

Returns

Amount of bytes needed for storing output.

Return values

UTF8_ERR_NONE	No errors.
UTF8_ERR_INVALID_DATA	Failed to decode data.
UTF8_ERR_OVERLAPPING_PARAMETERS	Input and output buffers overlap in memory.
UTF8_ERR_NOT_ENOUGH_SPACE	Target buffer size is insufficient for result.

See also

utf8towide

utf8toutf16

utf8towide()

UTF8_API size_t utf8towide	(	const char *	input,
		size_t	inputSize,
		wchar_t *	target,
		size_t	targetSize,
		int32_t *	errors
	)

Convert a UTF-8 encoded string to a wide string.

Depending on the platform, wide strings are either UTF-16 or UTF-32 encoded. This function takes a UTF-8 encoded string as input and automatically calls the correct conversion function.

This allows for a cross-platform treatment of wide text and is preferable to using the UTF-16 or UTF-32 versions directly.

Erroneous byte sequences such as missing or illegal bytes or overlong encoding of code points (e.g. using five bytes to encode a sequence that can be represented by two bytes) are converted to the replacement character U+FFFD.

Note

Code points outside the Basic Multilingual Plane (BMP) are converted to surrogate pairs when using UTF-16. This means that strings containing characters outside the BMP converted on a platform with UTF-32 wide strings are not compatible with platforms with UTF-16 wide strings.

Hence, it is preferable to store all data as UTF-8 and only convert to

wide strings when required by a third-party interface.

Example:

void Window_SetTitle(void* windowHandle, const char* text)
{
    size_t input_size = strlen(text);
    wchar_t* converted = NULL;
    size_t converted_size;
    int32_t errors;
 
    converted_size = utf8towide(text, input_size, NULL, 0, &errors);
    if (converted_size == 0 ||
        errors != UTF8_ERR_NONE)
    {
        goto cleanup;
    }
 
    converted = (wchar_t*)malloc(converted_size + sizeof(wchar_t));
    utf8towide(text, input_size, converted, converted_size, NULL);
    converted[converted_size / sizeof(wchar_t)] = 0;
 
    SetWindowTextW((HWND)windowHandle, converted);
 
cleanup:
    if (converted != NULL)
    {
        free(converted);
        converted = NULL;
    }
}

Parameters

[in]	input	UTF-8 encoded string.
[in]	inputSize	Size of the input in bytes.
[out]	target	Output buffer for the result, can be NULL.
[in]	targetSize	Size of the output buffer in bytes.
[out]	errors	Output for errors.

Returns

Amount of bytes needed for storing output.

Return values

UTF8_ERR_NONE	No errors.
UTF8_ERR_INVALID_DATA	Failed to decode data.
UTF8_ERR_OVERLAPPING_PARAMETERS	Input and output buffers overlap in memory.
UTF8_ERR_NOT_ENOUGH_SPACE	Target buffer size is insufficient for result.

See also

widetoutf8

utf8toutf16

utf8toutf32

utf8seek()

UTF8_API const char* utf8seek	(	const char *	text,
		size_t	textSize,
		const char *	textStart,
		off_t	offset,
		int	direction
	)

Seek into a UTF-8 encoded string.

Working with UTF-8 encoded strings can be tricky due to the nature of the variable-length encoding. Because one character no longer equals one byte, it can be difficult to skip around in a UTF-8 encoded string without decoding the code points.

This function provides an interface similar to fseek in order to enable skipping to another part of the string.

Note

textStart must come before text in memory when seeking from the current or end position.

Example:

const char* text = "Press \xE0\x80\x13 to continue.";
const char fixed[1024];
const char* commandStart;
const char* commandEnd;
 
memset(fixed, 0, sizeof(fixed));
 
commandStart = strstr(text, "\xE0\x80\x13");
if (commandStart == 0)
{
    return 0;
}
 
strncpy(fixed, text, commandStart - text);
strcat(fixed, "ENTER");
 
commandEnd = utf8seek(commandStart, strlen(commandStart), text, 1, SEEK_CUR);
if (commandEnd != commandStart)
{
    strcat(fixed, commandEnd);
}

Parameters

[in]	text	Input string.
[in]	textSize	Size of the complete input string in bytes, starting from textStart.
[in]	textStart	Start of input string.
[in]	offset	Requested offset in code points.
[in]	direction	Direction to seek in. SEEK_SET Offset is from the start of the string. SEEK_CUR Offset is from the current position of the string. SEEK_END Offset is from the end of the string.

Returns

Pointer to offset string or no change on error.

See also

utf8iscategory

utf8envlocale()

UTF8_API size_t utf8envlocale

(

)

Returns the environment's locale as an enum value.

This function retrieves the (thread-local) environment locale as an enum value in the list of locales. This value can be used with functions in this library that change behavior on certain inputs, depending on the specified locale.

Unfortunately, no cross-platform way of setting and retrieving the system locale is available without adding dependencies to the library. Please refer to your operating system's manual to determine how to setup the system locale on your target system.

Warning

This function should not be used as a replacement for platform- specific methods for retrieving the locale. Its intended usage is to "guess" the desired locale by looking at the system locale.

Example:

uint8_t Employee_PrintNames(const char** names, size_t nameCount)
{
    size_t locale = utf8envlocale();
    char buffer[256];
    size_t buffer_size = 255;
    int32_t errors;
    size_t i;
 
    for (i = 0; i < nameCount; ++i)
    {
        size_t buffer_filled;
 
        memset(buffer, 0, buffer_size);
 
        if ((buffer_filled = utf8toupper(names[i], strlen(names[i]), buffer, buffer_size, locale, &errors)) == 0 ||
            errors != UTF8_ERR_NONE)
        {
            return 0;
        }
 
        Log_Print(buffer, buffer_filled);
    }
 
    return 1;
}

Returns

A specific locale or UTF8_LOCALE_DEFAULT.

See also

utf8toupper

utf8tolower

utf8totitle

utf8casefold

utf8toupper()

UTF8_API size_t utf8toupper	(	const char *	input,
		size_t	inputSize,
		char *	target,
		size_t	targetSize,
		size_t	locale,
		int32_t *	errors
	)

Convert UTF-8 encoded text to uppercase.

This function allows conversion of UTF-8 encoded strings to uppercase without first changing the encoding to UTF-32. Conversion is fully compliant with the Unicode 7.0 standard.

Although most code points can be converted in-place, there are notable exceptions. For example, U+00DF (LATIN SMALL LETTER SHARP S) maps to "U+0053 U+0053" (LATIN CAPITAL LETTER S and LATIN CAPITAL LETTER S) when converted to uppercase. Therefor, it is advised to first determine the size in bytes of the output by calling the function with a NULL output buffer.

Only a handful of scripts make a distinction between upper and lowercase. In addition to modern scripts, such as Latin, Greek, Armenian and Cyrillic, a few historic or archaic scripts have case. The vast majority of scripts do not have case distinctions.

Note

Case mapping is not reversible. That is, toUpper(toLower(x)) != toLower(toUpper(x)).

Warning

Certain code points (or combinations of code points) apply rules based on the locale. For more information about these exceptional code points, please refer to the Unicode standard: ftp://ftp.unicode.org/Public/UNIDATA/SpecialCasing.txt

Example:

void Button_Draw(int32_t x, int32_t y, const char* text)
{
    size_t input_size = strlen(text);
    char* converted = NULL;
    size_t converted_size;
    int32_t text_box_width, text_box_height;
    int32_t errors;
 
    if ((utf8toupper(text, input_size, NULL, 0, UTF8_LOCALE_DEFAULT, &errors)) == 0 ||
        errors != UTF8_ERR_NONE)
    {
        goto cleanup;
    }
 
    converted = (char*)malloc(converted_size + 1);
 
    if (converted == NULL ||
        utf8toupper(text, input_size, converted, converted_size, UTF8_LOCALE_DEFAULT, &errors) == 0 ||
        errors != UTF8_ERR_NONE)
    {
        goto cleanup;
    }
 
    converted[converted_size] = 0;
 
    Font_GetTextDimensions(converted, &text_box_width, &text_box_height);
 
    Draw_BoxFilled(x - 4, y - 4, text_box_width + 8, text_box_height + 8, 0x088A08);
    Draw_BoxOutline(x - 4, y - 4, text_box_width + 8, text_box_height + 8, 0xA9F5A9);
    Font_DrawText(x + 2, y + 1, converted, 0x000000);
    Font_DrawText(x, y, converted, 0xFFFFFF);
 
cleanup:
    if (converted != NULL)
    {
        free(converted);
        converted = NULL;
    }
}

Parameters

[in]	input	UTF-8 encoded string.
[in]	inputSize	Size of the input in bytes.
[out]	target	Output buffer for the result, can be NULL.
[in]	targetSize	Size of the output buffer in bytes.
[in]	locale	Enables locale-specific behavior in the implementation. List of valid locales.
[out]	errors	Output for errors.

Returns

Amount of bytes needed to contain output.

Return values

UTF8_ERR_NONE	No errors.
UTF8_ERR_INVALID_DATA	Failed to decode data.
UTF8_ERR_INVALID_LOCALE	Invalid locale specified.
UTF8_ERR_OVERLAPPING_PARAMETERS	Input and output buffers overlap in memory.
UTF8_ERR_NOT_ENOUGH_SPACE	Target buffer size is insufficient for result.

See also

utf8tolower

utf8totitle

utf8casefold

utf8tolower()

UTF8_API size_t utf8tolower	(	const char *	input,
		size_t	inputSize,
		char *	target,
		size_t	targetSize,
		size_t	locale,
		int32_t *	errors
	)

Convert UTF-8 encoded text to lowercase.

This function allows conversion of UTF-8 encoded strings to lowercase without first changing the encoding to UTF-32. Conversion is fully compliant with the Unicode 7.0 standard.

Although most code points can be converted to lowercase in-place, there are notable exceptions. For example, U+0130 (LATIN CAPITAL LETTER I WITH DOT ABOVE) maps to "U+0069 U+0307" (LATIN SMALL LETTER I and COMBINING DOT ABOVE) when converted to lowercase. Therefor, it is advised to first determine the size in bytes of the output by calling the function with a NULL output buffer.

Only a handful of scripts make a distinction between upper- and lowercase. In addition to modern scripts, such as Latin, Greek, Armenian and Cyrillic, a few historic or archaic scripts have case. The vast majority of scripts do not have case distinctions.

Note

Case mapping is not reversible. That is, toUpper(toLower(x)) != toLower(toUpper(x)).

Warning

Certain code points (or combinations of code points) apply rules based on the locale. For more information about these exceptional code points, please refer to the Unicode standard: ftp://ftp.unicode.org/Public/UNIDATA/SpecialCasing.txt

Example:

author_t* Author_ByName(const char* name)
{
    author_t* result = NULL;
    size_t name_size = strlen(name);
    char* converted = NULL;
    size_t converted_size;
    int32_t errors;
    size_t i;
    
    if ((converted_size = utf8tolower(name, name_size, NULL, 0, UTF8_LOCALE_DEFAULT, &errors)) == 0 ||
        errors != UTF8_ERR_NONE)
    {
        goto cleanup;
    }
 
    converted = (char*)malloc(converted_size + 1);
 
    if (converted == NULL ||
        utf8tolower(name, name_size, converted, converted_size, UTF8_LOCALE_DEFAULT, &errors) == 0 ||
        errors != UTF8_ERR_NONE)
    {
        goto cleanup;
    }
 
    converted[converted_size] = 0;
    
    for (i = 0; i < g_AuthorCount; ++i)
    {
        if (!strcmp(g_Author[i].name, converted))
        {
            result = &g_Author[i];
            break;
        }
    }
 
cleanup:
    if (converted != NULL)
    {
        free(converted);
        converted = NULL;
    }
    return result;
}

Parameters

[in]	input	UTF-8 encoded string.
[in]	inputSize	Size of the input in bytes.
[out]	target	Output buffer for the result, can be NULL.
[in]	targetSize	Size of the output buffer in bytes.
[in]	locale	Enables locale-specific behavior in the implementation. List of valid locales.
[out]	errors	Output for errors.

Returns

Amount of bytes needed for storing output.

Return values

UTF8_ERR_NONE	No errors.
UTF8_ERR_INVALID_DATA	Failed to decode data.
UTF8_ERR_INVALID_LOCALE	Invalid locale specified.
UTF8_ERR_OVERLAPPING_PARAMETERS	Input and output buffers overlap in memory.
UTF8_ERR_NOT_ENOUGH_SPACE	Target buffer size is insufficient for result.

See also

utf8toupper

utf8totitle

utf8casefold

utf8totitle()

UTF8_API size_t utf8totitle	(	const char *	input,
		size_t	inputSize,
		char *	target,
		size_t	targetSize,
		size_t	locale,
		int32_t *	errors
	)

Convert UTF-8 encoded text to titlecase.

This function allows conversion of UTF-8 encoded strings to titlecase without first changing the encoding to UTF-32. Conversion is fully compliant with the Unicode 7.0 standard.

Titlecase requires a bit more explanation than uppercase and lowercase, because it is not a common text transformation. Titlecase uses uppercase for the first letter of each word and lowercase for the rest. Words are defined as "collections of code points with general category Lu, Ll, Lt, Lm or Lo according to the Unicode database".

Effectively, any type of punctuation can break up a word, even if this is not grammatically valid. This happens because the titlecasing algorithm does not and cannot take grammar rules into account.

Text	Titlecase
The running man	The Running Man
NATO Alliance	Nato Alliance
You're amazing at building libraries	You'Re Amazing At Building Libraries

Although most code points can be converted to titlecase in-place, there are notable exceptions. For example, U+00DF (LATIN SMALL LETTER SHARP S) maps to "U+0053 U+0073" (LATIN CAPITAL LETTER S and LATIN SMALL LETTER S) when converted to titlecase. Therefor, it is advised to first determine the size in bytes of the output by calling the function with a NULL output buffer.

Only a handful of scripts make a distinction between upper- and lowercase. In addition to modern scripts, such as Latin, Greek, Armenian and Cyrillic, a few historic or archaic scripts have case. The vast majority of scripts do not have case distinctions.

Note

Case mapping is not reversible. That is, toUpper(toLower(x)) != toLower(toUpper(x)).

Warning

Certain code points (or combinations of code points) apply rules based on the locale. For more information about these exceptional code points, please refer to the Unicode standard: ftp://ftp.unicode.org/Public/UNIDATA/SpecialCasing.txt

Example:

void Book_SetTitle(book_t* book, const char* title)
{
    size_t converted_size;
    int32_t errors;
    size_t i;
 
    if ((converted_size = utf8totitle(title, strlen(title), book->title, sizeof(book->title) - 1, UTF8_LOCALE_DEFAULT, &errors)) == 0 ||
        errors != UTF8_ERR_NONE)
    {
        memset(book->title, 0, sizeof(book->title));
 
        return;
    }
    book->title[converted_size] = 0;
}

Parameters

[in]	input	UTF-8 encoded string.
[in]	inputSize	Size of the input in bytes.
[out]	target	Output buffer for the result, can be NULL.
[in]	targetSize	Size of the output buffer in bytes.
[in]	locale	Enables locale-specific behavior in the implementation. List of valid locales.
[out]	errors	Output for errors.

Returns

Amount of bytes needed for storing output.

Return values

UTF8_ERR_NONE	No errors.
UTF8_ERR_INVALID_DATA	Failed to decode data.
UTF8_ERR_INVALID_LOCALE	Invalid locale specified.
UTF8_ERR_OVERLAPPING_PARAMETERS	Input and output buffers overlap in memory.
UTF8_ERR_NOT_ENOUGH_SPACE	Target buffer size is insufficient for result.

See also

utf8tolower

utf8toupper

utf8casefold

utf8casefold()

UTF8_API size_t utf8casefold	(	const char *	input,
		size_t	inputSize,
		char *	target,
		size_t	targetSize,
		size_t	locale,
		int32_t *	errors
	)

Remove case distinction from UTF-8 encoded text.

Case folding is the process of eliminating differences between code points concerning case mapping. It is most commonly used for comparing strings in a case-insensitive manner. Conversion is fully compliant with the Unicode 7.0 standard.

Although similar to lowercasing text, there are significant differences. For one, case folding does not take locale into account when converting. In some cases, case folding can be up to 20% faster than lowercasing the same text, but the result cannot be treated as correct lowercased text.

Only two locale-specific exception are made when case folding text. In Turkish, U+0049 LATIN CAPITAL LETTER I maps to U+0131 LATIN SMALL LETTER DOTLESS I and U+0130 LATIN CAPITAL LETTER I WITH DOT ABOVE maps to U+0069 LATIN SMALL LETTER I.

Although most code points can be case folded in-place, there are notable exceptions. For example, U+0130 (LATIN CAPITAL LETTER I WITH DOT ABOVE) maps to "U+0069 U+0307" (LATIN SMALL LETTER I and COMBINING DOT ABOVE) when converted to lowercase. Therefor, it is advised to first determine the size in bytes of the output by calling the function with a NULL output buffer.

Only a handful of scripts make a distinction between upper- and lowercase. In addition to modern scripts, such as Latin, Greek, Armenian and Cyrillic, a few historic or archaic scripts have case. The vast majority of scripts do not have case distinctions.

Example:

int32_t Command_ParseCommand(const char* argument)
{
    char* buffer = NULL;
    size_t buffer_size = 0;
    int32_t errors;
    int32_t result = 0;
 
    if ((buffer_size = utf8casefold(argument, strlen(argument), NULL, 0, UTF8_LOCALE_DEFAULT, &errors)) == 0 ||
        errors != UTF8_ERR_NONE)
    {
        result = -1;
 
        goto cleanup;
    }
 
    buffer = (char*)malloc(buffer_size);
 
    if (buffer == NULL ||
        utf8casefold(argument, strlen(argument), buffer, buffer_size, UTF8_LOCALE_DEFAULT, &errors) == 0 ||
        errors != UTF8_ERR_NONE)
    {
        result = -1;
 
        goto cleanup;
    }
 
    if (!strncmp(buffer, "-username", strlen("-username")))
    {
        result = eCommand_Username;
    }
    else if (
        !strncmp(buffer, "-password", strlen("-password")))
    {
        result = eCommand_Password;
    }
    else if (
        !strncmp(buffer, "-message", strlen("-message")))
    {
        result = eCommand_Message;
    }
 
cleanup:
    if (buffer != NULL)
    {
        free(buffer);
        buffer = NULL;
    }
 
    return result;
}

Parameters

[in]	input	UTF-8 encoded string.
[in]	inputSize	Size of the input in bytes.
[out]	target	Output buffer for the result, can be NULL.
[in]	targetSize	Size of the output buffer in bytes.
[in]	locale	Enables locale-specific behavior in the implementation. List of valid locales.
[out]	errors	Output for errors.

Returns

Amount of bytes needed for storing output.

Return values

UTF8_ERR_NONE	No errors.
UTF8_ERR_INVALID_DATA	Failed to decode data.
UTF8_ERR_INVALID_LOCALE	Invalid locale specified.
UTF8_ERR_OVERLAPPING_PARAMETERS	Input and output buffers overlap in memory.
UTF8_ERR_NOT_ENOUGH_SPACE	Target buffer size is insufficient for result.

See also

utf8tolower

utf8toupper

utf8totitle

utf8isnormalized()

UTF8_API uint8_t utf8isnormalized	(	const char *	input,
		size_t	inputSize,
		size_t	flags,
		size_t *	offset
	)

Check if a string is stable in the specified Unicode Normalization Form.

This function can be used as a preprocessing step, before attempting to normalize a string. Normalization is a very expensive process, it is often cheaper to first determine if the string is unstable in the requested normalization form.

The result of the check will be YES if the string is stable and MAYBE or NO if it is unstable. If the result is MAYBE, the string does not necessarily have to be normalized.

If the result is unstable, the offset parameter is set to the offset for the first unstable code point. If the string is stable, the offset is equivalent to the length of the string in bytes.

You must specify the desired Unicode Normalization Form by using a combination of flags:

Unicode	Flags
Normalization Form C (NFC)	UTF8_NORMALIZE_COMPOSE
Normalization Form KC (NFKC)	UTF8_NORMALIZE_COMPOSE + UTF8_NORMALIZE_COMPATIBILITY
Normalization Form D (NFD)	UTF8_NORMALIZE_DECOMPOSE
Normalization Form KD (NFKD)	UTF8_NORMALIZE_DECOMPOSE + UTF8_NORMALIZE_COMPATIBILITY

For more information, please review Unicode Standard Annex #15 - Unicode Normalization Forms.

Example:

uint8_t Text_InspectComposed(const char* text)
{
    const char* src = text;
    size_t src_size = strlen(text);
    size_t offset;
    size_t total_offset;
 
    if (utf8isnormalized(src, src_size, UTF8_NORMALIZE_COMPOSE, &offset) == UTF8_NORMALIZATION_RESULT_YES)
    {
        printf("Clean!\n");
 
        return 1;
    }
 
    total_offset = offset;
 
    do
    {
        const char* next;
 
        printf("Unstable at byte %d\n", total_offset);
 
        next = utf8seek(src, text, 1, SEEK_CUR);
        if (next == src)
        {
            break;
        }
 
        total_offset += offset;
 
        src = next;
        src_size -= next - src;
    }
    while (utf8isnormalized(src, src_size, UTF8_NORMALIZE_COMPOSE, &offset) != UTF8_NORMALIZATION_RESULT_YES);
 
    return 0;
}

Parameters

[in]	input	UTF-8 encoded string.
[in]	inputSize	Size of the input in bytes.
[in]	flags	Desired normalization form. Must be a combination of normalization flags.
[out]	offset	Offset to first unstable code point or length of input in bytes if stable.

Return values

UTF8_NORMALIZATION_RESULT_YES	Input is stable and does not have to be normalized.
UTF8_NORMALIZATION_RESULT_MAYBE	Input is unstable, but normalization may be skipped.
UTF8_NORMALIZATION_RESULT_NO	Input is unstable and must be normalized.

See also

utf8normalize

utf8normalize()

UTF8_API size_t utf8normalize	(	const char *	input,
		size_t	inputSize,
		char *	target,
		size_t	targetSize,
		size_t	flags,
		int32_t *	errors
	)

Normalize a string to the specified Unicode Normalization Form.

The Unicode standard defines two standards for equivalence between characters: canonical and compatibility equivalence. Canonically equivalent characters and sequence represent the same abstract character and must be rendered with the same appearance and behavior. Compatibility equivalent characters have a weaker equivalence and may be rendered differently.

Unicode Normalization Forms are formally defined standards that can be used to test whether any two strings of characters are equivalent to each other. This equivalence may be canonical or compatibility.

The algorithm puts all combining marks into a specified order and uses the rules for decomposition and composition to transform the string into one of four Unicode Normalization Forms. A binary comparison can then be used to determine equivalence.

These are the Unicode Normalization Forms:

Form	Description
Normalization Form D (NFD)	Canonical decomposition
Normalization Form C (NFC)	Canonical decomposition, followed by canonical composition
Normalization Form KD (NFKD)	Compatibility decomposition
Normalization Form KC (NFKC)	Compatibility decomposition, followed by canonical composition

utf8normalize can be used to transform text into one of these forms. You must specify the desired Unicode Normalization Form by using a combination of flags:

Form	Flags
Normalization Form D (NFD)	UTF8_NORMALIZE_DECOMPOSE
Normalization Form C (NFC)	UTF8_NORMALIZE_COMPOSE
Normalization Form KD (NFKD)	UTF8_NORMALIZE_DECOMPOSE + UTF8_NORMALIZE_COMPATIBILITY
Normalization Form KC (NFKC)	UTF8_NORMALIZE_COMPOSE + UTF8_NORMALIZE_COMPATIBILITY

For more information, please review Unicode Standard Annex #15 - Unicode Normalization Forms.

Note

Unnormalized text is rare in the wild. As an example, all text found on the Internet as HTML source code must be encoded as NFC, as specified by the W3C.

Example:

void Font_RenderTextNormalized(const char* input)
{
    const char* src = NULL;
    const char* src_start;
    size_t src_size;
    char* converted = NULL;
    size_t converted_size = 0;
    size_t input_size = strlen(input);
 
    if (utf8isnormalized(input, input_size, UTF8_NORMALIZE_COMPOSE, NULL) != UTF8_NORMALIZATION_RESULT_YES)
    {
        int32_t errors;
 
        converted_size = utf8normalize(input, input_size, NULL, 0, UTF8_NORMALIZE_COMPOSE, &errors);
        if (converted_size > 0 &&
            errors == UTF8_ERR_NONE)
        {
            converted = (char*)malloc(converted_size + 1);
            utf8normalize(input, input_size, converted, converted_size, UTF8_NORMALIZE_COMPOSE, NULL);
            converted[converted_size] = 0;
 
            src = (const char*)converted;
            src_size = converted_size;
        }
    }
 
    if (src == NULL)
    {
        src = (const char*)input;
        src_size = input_size;
    }
 
    src_start = src;
 
    while (src_size > 0)
    {
        const char* next;
        int32_t errors;
 
        next = utf8seek(src, src_size, src_start, 1, SEEK_CUR);
        if (next == src)
        {
            break;
        }
 
        unicode_t code_point;
        utf8toutf32(src, (size_t)(next - src), &code_point, sizeof(unicode_t), &errors);
        if (errors != UTF8_ERR_NONE)
        {
            break;
        }
 
        Font_RenderCodePoint(code_point);
 
        src_size -= next - src;
        src = next;
    }
 
    if (converted != NULL)
    {
        free(converted);
        converted = NULL;
    }
}

Parameters

[in]	input	UTF-8 encoded string.
[in]	inputSize	Size of the input in bytes.
[out]	target	Output buffer for the result, can be NULL.
[in]	targetSize	Size of the output buffer in bytes.
[in]	flags	Desired normalization form. Must be a combination of UTF8_NORMALIZE_COMPOSE, UTF8_NORMALIZE_DECOMPOSE and UTF8_NORMALIZE_COMPATIBILITY.
[out]	errors	Output for errors.

Returns

Amount of bytes needed for storing output.

Return values

UTF8_ERR_NONE	No errors.
UTF8_ERR_INVALID_FLAG	Invalid combination of flags was specified.
UTF8_ERR_INVALID_DATA	Failed to decode data.
UTF8_ERR_OVERLAPPING_PARAMETERS	Input and output buffers overlap in memory.
UTF8_ERR_NOT_ENOUGH_SPACE	Target buffer size is insufficient for result.

See also

utf8isnormalized

utf8iscategory()

UTF8_API size_t utf8iscategory	(	const char *	input,
		size_t	inputSize,
		size_t	flags
	)

Check if the input string conforms to the category specified by the flags.

This function can be used to check if the code points in a string are part of a category. Valid flags are members of the list of categories. The category for a code point is defined as part of the entry in UnicodeData.txt, the data file for the Unicode code point database.

Note

The function is greedy. This means it will try to match as many code points with the matching category flags as possible and return the offset in the input in bytes. If this is undesired behavior, use utf8seek to seek in the input first before matching it with the category flags.

By default, the function will treat grapheme clusters as a single code point. This means that the following string:

Code point	Canonical combining class	General category	Name
U+0045	0	Lu (Uppercase letter)	LATIN CAPITAL LETTER E
U+0300	230	Mn (Non-spacing mark)	COMBINING GRAVE ACCENT

Will match with UTF8_CATEGORY_LETTER_UPPERCASE in its entirety, because the COMBINING GRAVE ACCENT is treated as part of the grapheme cluster. This is useful when e.g. creating a text parser, because you do not have to normalize the text first.

If this is undesired behavior, specify the UTF8_CATEGORY_IGNORE_GRAPHEME_CLUSTER flag.

Warning

In order to maintain backwards compatibility with POSIX functions like isdigit and isspace, compatibility flags have been provided. Note, however, that the result is only guaranteed to be correct for code points in the Basic Latin range, between U+0000 and 0+007F. Combining a compatibility flag with a regular category flag will result in undefined behavior.

Example:

const char* Parser_NextIdentifier(char** output, size_t* outputSize, const char* input, size_t inputSize)
{
    const char* src = input;
    size_t src_size = inputSize;
    size_t whitespace_size;
    size_t identifier_size;
 
    whitespace_size = utf8iscategory(src, src_size, UTF8_CATEGORY_SEPARATOR_SPACE);
    if (whitespace_size == 0)
    {
        whitespace_size = utf8iscategory(src, src_size, UTF8_CATEGORY_ISSPACE);
    }
 
    if (whitespace_size > 0)
    {
        if (whitespace_size >= src_size)
        {
            return src + src_size;
        }
 
        src += whitespace_size;
        src_size -= whitespace_size;
    }
 
    identifier_size = utf8iscategory(src, src_size, UTF8_CATEGORY_LETTER | UTF8_CATEGORY_PUNCTUATION_CONNECTOR | UTF8_CATEGORY_PUNCTUATION_DASH);
    if (identifier_size == 0)
    {
        return src;
    }
 
     *output = (char*)malloc(identifier_size + 1);
    memcpy(*output, src, identifier_size);
    (*output)[identifier_size] = 0;
     *outputSize = identifier_size;
 
    if (identifier_size >= src_size)
    {
        return src + src_size;
    }
 
    return src + identifier_size;
}

Parameters

[in]	input	UTF-8 encoded string.
[in]	inputSize	Size of the input in bytes.
[in]	flags	Requested category. Must be a combination of category flags or a single compatibility flag.

Returns

Number of bytes in the input that conform to the specified category flags.

See also

utf8seek