aboutsummaryrefslogtreecommitdiff
path: root/src/chat.cpp
blob: 0bd5c16709bb54565f677e7d34123cd6af173608 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
/*
Minetest
Copyright (C) 2013 celeron55, Perttu Ahola <celeron55@gmail.com>

This program is free software; you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation; either version 2.1 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Lesser General Public License for more details.

You should have received a copy of the GNU Lesser General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/

#include "chat.h"
#include "debug.h"
#include "strfnd.h"
#include <cctype>
#include <sstream>
#include "util/string.h"
#include "util/numeric.h"

ChatBuffer::ChatBuffer(u32 scrollback):
	m_scrollback(scrollback),
	m_unformatted(),
	m_cols(0),
	m_rows(0),
	m_scroll(0),
	m_formatted(),
	m_empty_formatted_line()
{
	if (m_scrollback == 0)
		m_scrollback = 1;
	m_empty_formatted_line.first = true;
}

ChatBuffer::~ChatBuffer()
{
}

void ChatBuffer::addLine(std::wstring name, std::wstring text)
{
	ChatLine line(name, text);
	m_unformatted.push_back(line);

	if (m_rows > 0)
	{
		// m_formatted is valid and must be kept valid
		bool scrolled_at_bottom = (m_scroll == getBottomScrollPos());
		u32 num_added = formatChatLine(line, m_cols, m_formatted);
		if (scrolled_at_bottom)
			m_scroll += num_added;
	}

	// Limit number of lines by m_scrollback
	if (m_unformatted.size() > m_scrollback)
	{
		deleteOldest(m_unformatted.size() - m_scrollback);
	}
}

void ChatBuffer::clear()
{
	m_unformatted.clear();
	m_formatted.clear();
	m_scroll = 0;
}

u32 ChatBuffer::getLineCount() const
{
	return m_unformatted.size();
}

u32 ChatBuffer::getScrollback() const
{
	return m_scrollback;
}

const ChatLine& ChatBuffer::getLine(u32 index) const
{
	assert(index < getLineCount());
	return m_unformatted[index];
}

void ChatBuffer::step(f32 dtime)
{
	for (u32 i = 0; i < m_unformatted.size(); ++i)
	{
		m_unformatted[i].age += dtime;
	}
}

void ChatBuffer::deleteOldest(u32 count)
{
	u32 del_unformatted = 0;
	u32 del_formatted = 0;

	while (count > 0 && del_unformatted < m_unformatted.size())
	{
		++del_unformatted;

		// keep m_formatted in sync
		if (del_formatted < m_formatted.size())
		{
			assert(m_formatted[del_formatted].first);
			++del_formatted;
			while (del_formatted < m_formatted.size() &&
					!m_formatted[del_formatted].first)
				++del_formatted;
		}

		--count;
	}

	m_unformatted.erase(m_unformatted.begin(), m_unformatted.begin() + del_unformatted);
	m_formatted.erase(m_formatted.begin(), m_formatted.begin() + del_formatted);
}

void ChatBuffer::deleteByAge(f32 maxAge)
{
	u32 count = 0;
	while (count < m_unformatted.size() && m_unformatted[count].age > maxAge)
		++count;
	deleteOldest(count);
}

u32 ChatBuffer::getColumns() const
{
	return m_cols;
}

u32 ChatBuffer::getRows() const
{
	return m_rows;
}

void ChatBuffer::reformat(u32 cols, u32 rows)
{
	if (cols == 0 || rows == 0)
	{
		// Clear formatted buffer
		m_cols = 0;
		m_rows = 0;
		m_scroll = 0;
		m_formatted.clear();
	}
	else if (cols != m_cols || rows != m_rows)
	{
		// TODO: Avoid reformatting ALL lines (even inivisble ones)
		// each time the console size changes.

		// Find out the scroll position in *unformatted* lines
		u32 restore_scroll_unformatted = 0;
		u32 restore_scroll_formatted = 0;
		bool at_bottom = (m_scroll == getBottomScrollPos());
		if (!at_bottom)
		{
			for (s32 i = 0; i < m_scroll; ++i)
			{
				if (m_formatted[i].first)
					++restore_scroll_unformatted;
			}
		}

		// If number of columns change, reformat everything
		if (cols != m_cols)
		{
			m_formatted.clear();
			for (u32 i = 0; i < m_unformatted.size(); ++i)
			{
				if (i == restore_scroll_unformatted)
					restore_scroll_formatted = m_formatted.size();
				formatChatLine(m_unformatted[i], cols, m_formatted);
			}
		}

		// Update the console size
		m_cols = cols;
		m_rows = rows;

		// Restore the scroll position
		if (at_bottom)
		{
			scrollBottom();
		}
		else
		{
			scrollAbsolute(restore_scroll_formatted);
		}
	}
}

const ChatFormattedLine& ChatBuffer::getFormattedLine(u32 row) const
{
	s32 index = m_scroll + (s32) row;
	if (index >= 0 && index < (s32) m_formatted.size())
		return m_formatted[index];
	else
		return m_empty_formatted_line;
}

void ChatBuffer::scroll(s32 rows)
{
	scrollAbsolute(m_scroll + rows);
}

void ChatBuffer::scrollAbsolute(s32 scroll)
{
	s32 top = getTopScrollPos();
	s32 bottom = getBottomScrollPos();

	m_scroll = scroll;
	if (m_scroll < top)
		m_scroll = top;
	if (m_scroll > bottom)
		m_scroll = bottom;
}

void ChatBuffer::scrollBottom()
{
	m_scroll = getBottomScrollPos();
}

void ChatBuffer::scrollTop()
{
	m_scroll = getTopScrollPos();
}

u32 ChatBuffer::formatChatLine(const ChatLine& line, u32 cols,
		std::vector<ChatFormattedLine>& destination) const
{
	u32 num_added = 0;
	std::vector<ChatFormattedFragment> next_frags;
	ChatFormattedLine next_line;
	ChatFormattedFragment temp_frag;
	u32 out_column = 0;
	u32 in_pos = 0;
	u32 hanging_indentation = 0;

	// Format the sender name and produce fragments
	if (!line.name.empty())
	{
		temp_frag.text = L"<";
		temp_frag.column = 0;
		//temp_frag.bold = 0;
		next_frags.push_back(temp_frag);
		temp_frag.text = line.name;
		temp_frag.column = 0;
		//temp_frag.bold = 1;
		next_frags.push_back(temp_frag);
		temp_frag.text = L"> ";
		temp_frag.column = 0;
		//temp_frag.bold = 0;
		next_frags.push_back(temp_frag);
	}

	// Choose an indentation level
	if (line.name.empty())
	{
		// Server messages
		hanging_indentation = 0;
	}
	else if (line.name.size() + 3 <= cols/2)
	{
		// Names shorter than about half the console width
		hanging_indentation = line.name.size() + 3;
	}
	else
	{
		// Very long names
		hanging_indentation = 2;
	}

	next_line.first = true;
	bool text_processing = false;

	// Produce fragments and layout them into lines
	while (!next_frags.empty() || in_pos < line.text.size())
	{
		// Layout fragments into lines
		while (!next_frags.empty())
		{
			ChatFormattedFragment& frag = next_frags[0];
			if (frag.text.size() <= cols - out_column)
			{
				// Fragment fits into current line
				frag.column = out_column;
				next_line.fragments.push_back(frag);
				out_column += frag.text.size();
				next_frags.erase(next_frags.begin());
			}
			else
			{
				// Fragment does not fit into current line
				// So split it up
				temp_frag.text = frag.text.substr(0, cols - out_column);
				temp_frag.column = out_column;
				//temp_frag.bold = frag.bold;
				next_line.fragments.push_back(temp_frag);
				frag.text = frag.text.substr(cols - out_column);
				out_column = cols;
			}
			if (out_column == cols || text_processing)
			{
				// End the current line
				destination.push_back(next_line);
				num_added++;
				next_line.fragments.clear();
				next_line.first = false;

				out_column = text_processing ? hanging_indentation : 0;
			}
		}

		// Produce fragment
		if (in_pos < line.text.size())
		{
			u32 remaining_in_input = line.text.size() - in_pos;
			u32 remaining_in_output = cols - out_column;

			// Determine a fragment length <= the minimum of
			// remaining_in_{in,out}put. Try to end the fragment
			// on a word boundary.
			u32 frag_length = 1, space_pos = 0;
			while (frag_length < remaining_in_input &&
					frag_length < remaining_in_output)
			{
				if (isspace(line.text[in_pos + frag_length]))
					space_pos = frag_length;
				++frag_length;
			}
			if (space_pos != 0 && frag_length < remaining_in_input)
				frag_length = space_pos + 1;

			temp_frag.text = line.text.substr(in_pos, frag_length);
			temp_frag.column = 0;
			//temp_frag.bold = 0;
			next_frags.push_back(temp_frag);
			in_pos += frag_length;
			text_processing = true;
		}
	}

	// End the last line
	if (num_added == 0 || !next_line.fragments.empty())
	{
		destination.push_back(next_line);
		num_added++;
	}

	return num_added;
}

s32 ChatBuffer::getTopScrollPos() const
{
	s32 formatted_count = (s32) m_formatted.size();
	s32 rows = (s32) m_rows;
	if (rows == 0)
		return 0;
	else if (formatted_count <= rows)
		return formatted_count - rows;
	else
		return 0;
}

s32 ChatBuffer::getBottomScrollPos() const
{
	s32 formatted_count = (s32) m_formatted.size();
	s32 rows = (s32) m_rows;
	if (rows == 0)
		return 0;
	else
		return formatted_count - rows;
}



ChatPrompt::ChatPrompt(std::wstring prompt, u32 history_limit):
	m_prompt(prompt),
	m_line(L""),
	m_history(),
	m_history_index(0),
	m_history_limit(history_limit),
	m_cols(0),
	m_view(0),
	m_cursor(0),
	m_nick_completion_start(0),
	m_nick_completion_end(0)
{
}

ChatPrompt::~ChatPrompt()
{
}

void ChatPrompt::input(wchar_t ch)
{
	m_line.insert(m_cursor, 1, ch);
	m_cursor++;
	clampView();
	m_nick_completion_start = 0;
	m_nick_completion_end = 0;
}

std::wstring ChatPrompt::submit()
{
	std::wstring line = m_line;
	m_line.clear();
	if (!line.empty())
		m_history.push_back(line);
	if (m_history.size() > m_history_limit)
		m_history.erase(m_history.begin());
	m_history_index = m_history.size();
	m_view = 0;
	m_cursor = 0;
	m_nick_completion_start = 0;
	m_nick_completion_end = 0;
	return line;
}

void ChatPrompt::clear()
{
	m_line.clear();
	m_view = 0;
	m_cursor = 0;
	m_nick_completion_start = 0;
	m_nick_completion_end = 0;
}

void ChatPrompt::replace(std::wstring line)
{
	m_line =  line;
	m_view = m_cursor = line.size();
	clampView();
	m_nick_completion_start = 0;
	m_nick_completion_end = 0;
}

void ChatPrompt::historyPrev()
{
	if (m_history_index != 0)
	{
		--m_history_index;
		replace(m_history[m_history_index]);
	}
}

void ChatPrompt::historyNext()
{
	if (m_history_index + 1 >= m_history.size())
	{
		m_history_index = m_history.size();
		replace(L"");
	}
	else
	{
		++m_history_index;
		replace(m_history[m_history_index]);
	}
}

void ChatPrompt::nickCompletion(const std::list<std::string>& names, bool backwards)
{
	// Two cases:
	// (a) m_nick_completion_start == m_nick_completion_end == 0
	//     Then no previous nick completion is active.
	//     Get the word around the cursor and replace with any nick
	//     that has that word as a prefix.
	// (b) else, continue a previous nick completion.
	//     m_nick_completion_start..m_nick_completion_end are the
	//     interval where the originally used prefix was. Cycle
	//     through the list of completions of that prefix.
	u32 prefix_start = m_nick_completion_start;
	u32 prefix_end = m_nick_completion_end;
	bool initial = (prefix_end == 0);
	if (initial)
	{
		// no previous nick completion is active
		prefix_start = prefix_end = m_cursor;
		while (prefix_start > 0 && !isspace(m_line[prefix_start-1]))
			--prefix_start;
		while (prefix_end < m_line.size() && !isspace(m_line[prefix_end]))
			++prefix_end;
		if (prefix_start == prefix_end)
			return;
	}
	std::wstring prefix = m_line.substr(prefix_start, prefix_end - prefix_start);

	// find all names that start with the selected prefix
	std::vector<std::wstring> completions;
	for (std::list<std::string>::const_iterator
			i = names.begin();
			i != names.end(); ++i)
	{
		if (str_starts_with(narrow_to_wide(*i), prefix, true))
		{
			std::wstring completion = narrow_to_wide(*i);
			if (prefix_start == 0)
				completion += L":";
			completions.push_back(completion);
		}
	}
	if (completions.empty())
		return;

	// find a replacement string and the word that will be replaced
	u32 word_end = prefix_end;
	u32 replacement_index = 0;
	if (!initial)
	{
		while (word_end < m_line.size() && !isspace(m_line[word_end]))
			++word_end;
		std::wstring word = m_line.substr(prefix_start, word_end - prefix_start);

		// cycle through completions
		for (u32 i = 0; i < completions.size(); ++i)
		{
			if (str_equal(word, completions[i], true))
			{
				if (backwards)
					replacement_index = i + completions.size() - 1;
				else
					replacement_index = i + 1;
				replacement_index %= completions.size();
				break;
			}
		}
	}
	std::wstring replacement = completions[replacement_index] + L" ";
	if (word_end < m_line.size() && isspace(word_end))
		++word_end;

	// replace existing word with replacement word,
	// place the cursor at the end and record the completion prefix
	m_line.replace(prefix_start, word_end - prefix_start, replacement);
	m_cursor = prefix_start + replacement.size();
	clampView();
	m_nick_completion_start = prefix_start;
	m_nick_completion_end = prefix_end;
}

void ChatPrompt::reformat(u32 cols)
{
	if (cols <= m_prompt.size())
	{
		m_cols = 0;
		m_view = m_cursor;
	}
	else
	{
		s32 length = m_line.size();
		bool was_at_end = (m_view + m_cols >= length + 1);
		m_cols = cols - m_prompt.size();
		if (was_at_end)
			m_view = length;
		clampView();
	}
}

std::wstring ChatPrompt::getVisiblePortion() const
{
	return m_prompt + m_line.substr(m_view, m_cols);
}

s32 ChatPrompt::getVisibleCursorPosition() const
{
	return m_cursor - m_view + m_prompt.size();
}

void ChatPrompt::cursorOperation(CursorOp op, CursorOpDir dir, CursorOpScope scope)
{
	s32 old_cursor = m_cursor;
	s32 new_cursor = m_cursor;

	s32 length = m_line.size();
	s32 increment = (dir == CURSOROP_DIR_RIGHT) ? 1 : -1;

	if (scope == CURSOROP_SCOPE_CHARACTER)
	{
		new_cursor += increment;
	}
	else if (scope == CURSOROP_SCOPE_WORD)
	{
		if (increment > 0)
		{
			// skip one word to the right
			while (new_cursor < length && isspace(m_line[new_cursor]))
				new_cursor++;
			while (new_cursor < length && !isspace(m_line[new_cursor]))
				new_cursor++;
			while (new_cursor < length && isspace(m_line[new_cursor]))
				new_cursor++;
		}
		else
		{
			// skip one word to the left
			while (new_cursor >= 1 && isspace(m_line[new_cursor - 1]))
				new_cursor--;
			while (new_cursor >= 1 && !isspace(m_line[new_cursor - 1]))
				new_cursor--;
		}
	}
	else if (scope == CURSOROP_SCOPE_LINE)
	{
		new_cursor += increment * length;
	}

	new_cursor = MYMAX(MYMIN(new_cursor, length), 0);

	if (op == CURSOROP_MOVE)
	{
		m_cursor = new_cursor;
	}
	else if (op == CURSOROP_DELETE)
	{
		if (new_cursor < old_cursor)
		{
			m_line.erase(new_cursor, old_cursor - new_cursor);
			m_cursor = new_cursor;
		}
		else if (new_cursor > old_cursor)
		{
			m_line.erase(old_cursor, new_cursor - old_cursor);
			m_cursor = old_cursor;
		}
	}

	clampView();

	m_nick_completion_start = 0;
	m_nick_completion_end = 0;
}

void ChatPrompt::clampView()
{
	s32 length = m_line.size();
	if (length + 1 <= m_cols)
	{
		m_view = 0;
	}
	else
	{
		m_view = MYMIN(m_view, length + 1 - m_cols);
		m_view = MYMIN(m_view, m_cursor);
		m_view = MYMAX(m_view, m_cursor - m_cols + 1);
		m_view = MYMAX(m_view, 0);
	}
}



ChatBackend::ChatBackend():
	m_console_buffer(500),
	m_recent_buffer(6),
	m_prompt(L"]", 500)
{
}

ChatBackend::~ChatBackend()
{
}

void ChatBackend::addMessage(std::wstring name, std::wstring text)
{
	// Note: A message may consist of multiple lines, for example the MOTD.
	WStrfnd fnd(text);
	while (!fnd.atend())
	{
		std::wstring line = fnd.next(L"\n");
		m_console_buffer.addLine(name, line);
		m_recent_buffer.addLine(name, line);
	}
}

void ChatBackend::addUnparsedMessage(std::wstring message)
{
	// TODO: Remove the need to parse chat messages client-side, by sending
	// separate name and text fields in TOCLIENT_CHAT_MESSAGE.

	if (message.size() >= 2 && message[0] == L'<')
	{
		std::size_t closing = message.find_first_of(L'>', 1);
		if (closing != std::wstring::npos &&
				closing + 2 <= message.size() &&
				message[closing+1] == L' ')
		{
			std::wstring name = message.substr(1, closing - 1);
			std::wstring text = message.substr(closing + 2);
			addMessage(name, text);
			return;
		}
	}

	// Unable to parse, probably a server message.
	addMessage(L"", message);
}

ChatBuffer& ChatBackend::getConsoleBuffer()
{
	return m_console_buffer;
}

ChatBuffer& ChatBackend::getRecentBuffer()
{
	return m_recent_buffer;
}

std::wstring ChatBackend::getRecentChat()
{
	std::wostringstream stream;
	for (u32 i = 0; i < m_recent_buffer.getLineCount(); ++i)
	{
		const ChatLine& line = m_recent_buffer.getLine(i);
		if (i != 0)
			stream << L"\n";
		if (!line.name.empty())
			stream << L"<" << line.name << L"> ";
		stream << line.text;
	}
	return stream.str();
}

ChatPrompt& ChatBackend::getPrompt()
{
	return m_prompt;
}

void ChatBackend::reformat(u32 cols, u32 rows)
{
	m_console_buffer.reformat(cols, rows);

	// no need to reformat m_recent_buffer, its formatted lines
	// are not used

	m_prompt.reformat(cols);
}

void ChatBackend::clearRecentChat()
{
	m_recent_buffer.clear();
}

void ChatBackend::step(float dtime)
{
	m_recent_buffer.step(dtime);
	m_recent_buffer.deleteByAge(60.0);

	// no need to age messages in anything but m_recent_buffer
}

void ChatBackend::scroll(s32 rows)
{
	m_console_buffer.scroll(rows);
}

void ChatBackend::scrollPageDown()
{
	m_console_buffer.scroll(m_console_buffer.getRows());
}

void ChatBackend::scrollPageUp()
{
	m_console_buffer.scroll(-m_console_buffer.getRows());
}
2854 2855 2856 2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 2905 2906 2907 2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952 2953 2954 2955 2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966 2967 2968 2969 2970 2971 2972 2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983 2984 2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019 3020 3021 3022 3023 3024 3025 3026 3027 3028 3029 3030 3031 3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060 3061 3062 3063 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193 3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325 3326 3327 3328 3329 3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341 3342 3343 3344 3345 3346 3347 3348 3349 3350 3351 3352 3353 3354 3355 3356 3357 3358 3359 3360 3361 3362 3363 3364 3365 3366 3367 3368 3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379 3380 3381 3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401 3402 3403 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415 3416 3417 3418 3419 3420 3421 3422 3423 3424 3425 3426 3427 3428 3429 3430 3431 3432 3433 3434 3435 3436 3437 3438 3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452 3453 3454 3455 3456 3457 3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478 3479 3480 3481 3482 3483 3484 3485 3486 3487 3488 3489 3490 3491 3492 3493 3494 3495 3496 3497 3498 3499 3500 3501 3502 3503 3504 3505 3506 3507 3508 3509 3510 3511 3512 3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524 3525 3526 3527 3528 3529 3530 3531 3532 3533 3534 3535 3536 3537 3538 3539 3540 3541 3542 3543 3544 3545 3546 3547 3548 3549 3550 3551 3552 3553 3554 3555 3556 3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574 3575 3576 3577 3578 3579 3580 3581 3582 3583 3584 3585 3586 3587 3588 3589 3590 3591 3592 3593 3594 3595 3596 3597 3598 3599 3600 3601 3602 3603 3604 3605 3606 3607 3608 3609 3610 3611 3612 3613 3614 3615 3616 3617 3618 3619 3620 3621 3622 3623 3624 3625 3626 3627 3628 3629 3630 3631 3632 3633 3634 3635 3636 3637 3638 3639 3640 3641 3642 3643 3644 3645 3646 3647 3648 3649 3650 3651 3652 3653 3654 3655 3656 3657 3658 3659 3660 3661 3662 3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 3688 3689 3690 3691 3692 3693 3694 3695 3696 3697 3698 3699 3700 3701 3702 3703 3704 3705 3706 3707 3708 3709 3710 3711 3712 3713 3714 3715 3716 3717 3718 3719 3720 3721 3722 3723 3724 3725 3726 3727 3728 3729 3730 3731 3732 3733 3734 3735 3736 3737 3738 3739 3740 3741 3742 3743 3744 3745 3746 3747 3748 3749 3750 3751 3752 3753 3754 3755 3756 3757 3758 3759 3760 3761 3762 3763 3764 3765 3766 3767 3768 3769 3770 3771 3772 3773 3774 3775 3776 3777 3778 3779 3780 3781 3782 3783 3784 3785 3786 3787 3788 3789 3790 3791 3792 3793 3794 3795 3796 3797 3798 3799 3800 3801 3802 3803 3804 3805 3806 3807 3808 3809 3810 3811 3812 3813 3814 3815 3816 3817 3818 3819 3820 3821 3822 3823 3824 3825 3826 3827 3828 3829 3830 3831 3832 3833 3834 3835 3836 3837 3838 3839 3840 3841 3842 3843 3844 3845 3846 3847 3848 3849 3850 3851 3852 3853 3854 3855 3856 3857 3858 3859 3860 3861 3862 3863 3864 3865 3866 3867 3868 3869 3870 3871 3872 3873 3874 3875 3876 3877 3878 3879 3880 3881 3882 3883 3884 3885 3886 3887 3888 3889 3890
/*
   Basic Unicode string class for Irrlicht.
   Copyright (c) 2009-2011 John Norman

   This software is provided 'as-is', without any express or implied
   warranty. In no event will the authors be held liable for any
   damages arising from the use of this software.

   Permission is granted to anyone to use this software for any
   purpose, including commercial applications, and to alter it and
   redistribute it freely, subject to the following restrictions:

   1. The origin of this software must not be misrepresented; you
      must not claim that you wrote the original software. If you use
      this software in a product, an acknowledgment in the product
      documentation would be appreciated but is not required.

   2. Altered source versions must be plainly marked as such, and
      must not be misrepresented as being the original software.

   3. This notice may not be removed or altered from any source
      distribution.

   The original version of this class can be located at:
   http://irrlicht.suckerfreegames.com/

   John Norman
   john@suckerfreegames.com
*/

#ifndef __IRR_USTRING_H_INCLUDED__
#define __IRR_USTRING_H_INCLUDED__

#if (__cplusplus > 199711L) || (_MSC_VER >= 1600) || defined(__GXX_EXPERIMENTAL_CXX0X__)
#	define USTRING_CPP0X
#	if defined(__GXX_EXPERIMENTAL_CXX0X__) && ((__GNUC__ > 4) || ((__GNUC__ == 4) && (__GNUC_MINOR__ >= 5)))
#		define USTRING_CPP0X_NEWLITERALS
#	endif
#endif

#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#ifdef _WIN32
#define __BYTE_ORDER 0
#define __LITTLE_ENDIAN 0
#define __BIG_ENDIAN 1
#elif __MACH__
#include <machine/endian.h>
#else
#include <endian.h>
#endif

#ifdef USTRING_CPP0X
#	include <utility>
#endif

#ifndef USTRING_NO_STL
#	include <string>
#	include <iterator>
#	include <ostream>
#endif

#include "irrTypes.h"
#include "irrAllocator.h"
#include "irrArray.h"
#include "irrMath.h"
#include "irrString.h"
#include "path.h"

//! UTF-16 surrogate start values.
static const irr::u16 UTF16_HI_SURROGATE = 0xD800;
static const irr::u16 UTF16_LO_SURROGATE = 0xDC00;

//! Is a UTF-16 code point a surrogate?
#define UTF16_IS_SURROGATE(c)		(((c) & 0xF800) == 0xD800)
#define UTF16_IS_SURROGATE_HI(c)	(((c) & 0xFC00) == 0xD800)
#define UTF16_IS_SURROGATE_LO(c)	(((c) & 0xFC00) == 0xDC00)


namespace irr
{

	// Define our character types.
#ifdef USTRING_CPP0X_NEWLITERALS	// C++0x
	typedef char32_t uchar32_t;
	typedef char16_t uchar16_t;
	typedef char uchar8_t;
#else
	typedef u32 uchar32_t;
	typedef u16 uchar16_t;
	typedef u8 uchar8_t;
#endif

namespace core
{

namespace unicode
{

//! The unicode replacement character.  Used to replace invalid characters.
const irr::u16 UTF_REPLACEMENT_CHARACTER = 0xFFFD;

//! Convert a UTF-16 surrogate pair into a UTF-32 character.
//! \param high The high value of the pair.
//! \param low The low value of the pair.
//! \return The UTF-32 character expressed by the surrogate pair.
inline uchar32_t toUTF32(uchar16_t high, uchar16_t low)
{
	// Convert the surrogate pair into a single UTF-32 character.
	uchar32_t x = ((high & ((1 << 6) -1)) << 10) | (low & ((1 << 10) -1));
	uchar32_t wu = ((high >> 6) & ((1 << 5) - 1)) + 1;
	return (wu << 16) | x;
}

//! Swaps the endianness of a 16-bit value.
//! \return The new value.
inline uchar16_t swapEndian16(const uchar16_t& c)
{
	return ((c >> 8) & 0x00FF) | ((c << 8) & 0xFF00);
}

//! Swaps the endianness of a 32-bit value.
//! \return The new value.
inline uchar32_t swapEndian32(const uchar32_t& c)
{
	return  ((c >> 24) & 0x000000FF) |
			((c >> 8)  & 0x0000FF00) |
			((c << 8)  & 0x00FF0000) |
			((c << 24) & 0xFF000000);
}

//! The Unicode byte order mark.
const u16 BOM = 0xFEFF;

//! The size of the Unicode byte order mark in terms of the Unicode character size.
const u8 BOM_UTF8_LEN = 3;
const u8 BOM_UTF16_LEN = 1;
const u8 BOM_UTF32_LEN = 1;

//! Unicode byte order marks for file operations.
const u8 BOM_ENCODE_UTF8[3] = { 0xEF, 0xBB, 0xBF };
const u8 BOM_ENCODE_UTF16_BE[2] = { 0xFE, 0xFF };
const u8 BOM_ENCODE_UTF16_LE[2] = { 0xFF, 0xFE };
const u8 BOM_ENCODE_UTF32_BE[4] = { 0x00, 0x00, 0xFE, 0xFF };
const u8 BOM_ENCODE_UTF32_LE[4] = { 0xFF, 0xFE, 0x00, 0x00 };

//! The size in bytes of the Unicode byte marks for file operations.
const u8 BOM_ENCODE_UTF8_LEN = 3;
const u8 BOM_ENCODE_UTF16_LEN = 2;
const u8 BOM_ENCODE_UTF32_LEN = 4;

//! Unicode encoding type.
enum EUTF_ENCODE
{
	EUTFE_NONE		= 0,
	EUTFE_UTF8,
	EUTFE_UTF16,
	EUTFE_UTF16_LE,
	EUTFE_UTF16_BE,
	EUTFE_UTF32,
	EUTFE_UTF32_LE,
	EUTFE_UTF32_BE
};

//! Unicode endianness.
enum EUTF_ENDIAN
{
	EUTFEE_NATIVE	= 0,
	EUTFEE_LITTLE,
	EUTFEE_BIG
};

//! Returns the specified unicode byte order mark in a byte array.
//! The byte order mark is the first few bytes in a text file that signifies its encoding.
/** \param mode The Unicode encoding method that we want to get the byte order mark for.
		If EUTFE_UTF16 or EUTFE_UTF32 is passed, it uses the native system endianness. **/
//! \return An array that contains a byte order mark.
inline core::array<u8> getUnicodeBOM(EUTF_ENCODE mode)
{
#define COPY_ARRAY(source, size) \
	memcpy(ret.pointer(), source, size); \
	ret.set_used(size)

	core::array<u8> ret(4);
	switch (mode)
	{
		case EUTFE_UTF8:
			COPY_ARRAY(BOM_ENCODE_UTF8, BOM_ENCODE_UTF8_LEN);
			break;
		case EUTFE_UTF16:
			#ifdef __BIG_ENDIAN__
				COPY_ARRAY(BOM_ENCODE_UTF16_BE, BOM_ENCODE_UTF16_LEN);
			#else
				COPY_ARRAY(BOM_ENCODE_UTF16_LE, BOM_ENCODE_UTF16_LEN);
			#endif
			break;
		case EUTFE_UTF16_BE:
			COPY_ARRAY(BOM_ENCODE_UTF16_BE, BOM_ENCODE_UTF16_LEN);
			break;
		case EUTFE_UTF16_LE:
			COPY_ARRAY(BOM_ENCODE_UTF16_LE, BOM_ENCODE_UTF16_LEN);
			break;
		case EUTFE_UTF32:
			#ifdef __BIG_ENDIAN__
				COPY_ARRAY(BOM_ENCODE_UTF32_BE, BOM_ENCODE_UTF32_LEN);
			#else
				COPY_ARRAY(BOM_ENCODE_UTF32_LE, BOM_ENCODE_UTF32_LEN);
			#endif
			break;
		case EUTFE_UTF32_BE:
			COPY_ARRAY(BOM_ENCODE_UTF32_BE, BOM_ENCODE_UTF32_LEN);
			break;
		case EUTFE_UTF32_LE:
			COPY_ARRAY(BOM_ENCODE_UTF32_LE, BOM_ENCODE_UTF32_LEN);
			break;
		case EUTFE_NONE:
			// TODO sapier: fixed warning only,
			// don't know if something needs to be done here
			break;
	}
	return ret;

#undef COPY_ARRAY
}

//! Detects if the given data stream starts with a unicode BOM.
//! \param data The data stream to check.
//! \return The unicode BOM associated with the data stream, or EUTFE_NONE if none was found.
inline EUTF_ENCODE determineUnicodeBOM(const char* data)
{
	if (memcmp(data, BOM_ENCODE_UTF8, 3) == 0) return EUTFE_UTF8;
	if (memcmp(data, BOM_ENCODE_UTF16_BE, 2) == 0) return EUTFE_UTF16_BE;
	if (memcmp(data, BOM_ENCODE_UTF16_LE, 2) == 0) return EUTFE_UTF16_LE;
	if (memcmp(data, BOM_ENCODE_UTF32_BE, 4) == 0) return EUTFE_UTF32_BE;
	if (memcmp(data, BOM_ENCODE_UTF32_LE, 4) == 0) return EUTFE_UTF32_LE;
	return EUTFE_NONE;
}

} // end namespace unicode


//! UTF-16 string class.
template <typename TAlloc = irrAllocator<uchar16_t> >
class ustring16
{
public:

	///------------------///
	/// iterator classes ///
	///------------------///

	//! Access an element in a unicode string, allowing one to change it.
	class _ustring16_iterator_access
	{
		public:
			_ustring16_iterator_access(const ustring16<TAlloc>* s, u32 p) : ref(s), pos(p) {}

			//! Allow the class to be interpreted as a single UTF-32 character.
			operator uchar32_t() const
			{
				return _get();
			}

			//! Allow one to change the character in the unicode string.
			//! \param c The new character to use.
			//! \return Myself.
			_ustring16_iterator_access& operator=(const uchar32_t c)
			{
				_set(c);
				return *this;
			}

			//! Increments the value by 1.
			//! \return Myself.
			_ustring16_iterator_access& operator++()
			{
				_set(_get() + 1);
				return *this;
			}

			//! Increments the value by 1, returning the old value.
			//! \return A unicode character.
			uchar32_t operator++(int)
			{
				uchar32_t old = _get();
				_set(old + 1);
				return old;
			}

			//! Decrements the value by 1.
			//! \return Myself.
			_ustring16_iterator_access& operator--()
			{
				_set(_get() - 1);
				return *this;
			}

			//! Decrements the value by 1, returning the old value.
			//! \return A unicode character.
			uchar32_t operator--(int)
			{
				uchar32_t old = _get();
				_set(old - 1);
				return old;
			}

			//! Adds to the value by a specified amount.
			//! \param val The amount to add to this character.
			//! \return Myself.
			_ustring16_iterator_access& operator+=(int val)
			{
				_set(_get() + val);
				return *this;
			}

			//! Subtracts from the value by a specified amount.
			//! \param val The amount to subtract from this character.
			//! \return Myself.
			_ustring16_iterator_access& operator-=(int val)
			{
				_set(_get() - val);
				return *this;
			}

			//! Multiples the value by a specified amount.
			//! \param val The amount to multiply this character by.
			//! \return Myself.
			_ustring16_iterator_access& operator*=(int val)
			{
				_set(_get() * val);
				return *this;
			}

			//! Divides the value by a specified amount.
			//! \param val The amount to divide this character by.
			//! \return Myself.
			_ustring16_iterator_access& operator/=(int val)
			{
				_set(_get() / val);
				return *this;
			}

			//! Modulos the value by a specified amount.
			//! \param val The amount to modulo this character by.
			//! \return Myself.
			_ustring16_iterator_access& operator%=(int val)
			{
				_set(_get() % val);
				return *this;
			}

			//! Adds to the value by a specified amount.
			//! \param val The amount to add to this character.
			//! \return A unicode character.
			uchar32_t operator+(int val) const
			{
				return _get() + val;
			}

			//! Subtracts from the value by a specified amount.
			//! \param val The amount to subtract from this character.
			//! \return A unicode character.
			uchar32_t operator-(int val) const
			{
				return _get() - val;
			}

			//! Multiplies the value by a specified amount.
			//! \param val The amount to multiply this character by.
			//! \return A unicode character.
			uchar32_t operator*(int val) const
			{
				return _get() * val;
			}

			//! Divides the value by a specified amount.
			//! \param val The amount to divide this character by.
			//! \return A unicode character.
			uchar32_t operator/(int val) const
			{
				return _get() / val;
			}

			//! Modulos the value by a specified amount.
			//! \param val The amount to modulo this character by.
			//! \return A unicode character.
			uchar32_t operator%(int val) const
			{
				return _get() % val;
			}

		private:
			//! Gets a uchar32_t from our current position.
			uchar32_t _get() const
			{
				const uchar16_t* a = ref->c_str();
				if (!UTF16_IS_SURROGATE(a[pos]))
					return static_cast<uchar32_t>(a[pos]);
				else
				{
					if (pos + 1 >= ref->size_raw())
						return 0;

					return unicode::toUTF32(a[pos], a[pos + 1]);
				}
			}

			//! Sets a uchar32_t at our current position.
			void _set(uchar32_t c)
			{
				ustring16<TAlloc>* ref2 = const_cast<ustring16<TAlloc>*>(ref);
				const uchar16_t* a = ref2->c_str();
				if (c > 0xFFFF)
				{
					// c will be multibyte, so split it up into the high and low surrogate pairs.
					uchar16_t x = static_cast<uchar16_t>(c);
					uchar16_t vh = UTF16_HI_SURROGATE | ((((c >> 16) & ((1 << 5) - 1)) - 1) << 6) | (x >> 10);
					uchar16_t vl = UTF16_LO_SURROGATE | (x & ((1 << 10) - 1));

					// If the previous position was a surrogate pair, just replace them.  Else, insert the low pair.
					if (UTF16_IS_SURROGATE_HI(a[pos]) && pos + 1 != ref2->size_raw())
						ref2->replace_raw(vl, static_cast<u32>(pos) + 1);
					else ref2->insert_raw(vl, static_cast<u32>(pos) + 1);

					ref2->replace_raw(vh, static_cast<u32>(pos));
				}
				else
				{
					// c will be a single byte.
					uchar16_t vh = static_cast<uchar16_t>(c);

					// If the previous position was a surrogate pair, remove the extra byte.
					if (UTF16_IS_SURROGATE_HI(a[pos]))
						ref2->erase_raw(static_cast<u32>(pos) + 1);

					ref2->replace_raw(vh, static_cast<u32>(pos));
				}
			}

			const ustring16<TAlloc>* ref;
			u32 pos;
	};
	typedef typename ustring16<TAlloc>::_ustring16_iterator_access access;


	//! Iterator to iterate through a UTF-16 string.
#ifndef USTRING_NO_STL
	class _ustring16_const_iterator : public std::iterator<
		std::bidirectional_iterator_tag,	// iterator_category
		access,								// value_type
		ptrdiff_t,							// difference_type
		const access,						// pointer
		const access						// reference
	>
#else
	class _ustring16_const_iterator
#endif
	{
		public:
			typedef _ustring16_const_iterator _Iter;
			typedef std::iterator<std::bidirectional_iterator_tag, access, ptrdiff_t, const access, const access> _Base;
			typedef const access const_pointer;
			typedef const access const_reference;

#ifndef USTRING_NO_STL
			typedef typename _Base::value_type value_type;
			typedef typename _Base::difference_type difference_type;
			typedef typename _Base::difference_type distance_type;
			typedef typename _Base::pointer pointer;
			typedef const_reference reference;
#else
			typedef access value_type;
			typedef u32 difference_type;
			typedef u32 distance_type;
			typedef const_pointer pointer;
			typedef const_reference reference;
#endif

			//! Constructors.
			_ustring16_const_iterator(const _Iter& i) : ref(i.ref), pos(i.pos) {}
			_ustring16_const_iterator(const ustring16<TAlloc>& s) : ref(&s), pos(0) {}
			_ustring16_const_iterator(const ustring16<TAlloc>& s, const u32 p) : ref(&s), pos(0)
			{
				if (ref->size_raw() == 0 || p == 0)
					return;

				// Go to the appropriate position.
				u32 i = p;
				u32 sr = ref->size_raw();
				const uchar16_t* a = ref->c_str();
				while (i != 0 && pos < sr)
				{
					if (UTF16_IS_SURROGATE_HI(a[pos]))
						pos += 2;
					else ++pos;
					--i;
				}
			}

			//! Test for equalness.
			bool operator==(const _Iter& iter) const
			{
				if (ref == iter.ref && pos == iter.pos)
					return true;
				return false;
			}

			//! Test for unequalness.
			bool operator!=(const _Iter& iter) const
			{
				if (ref != iter.ref || pos != iter.pos)
					return true;
				return false;
			}

			//! Switch to the next full character in the string.
			_Iter& operator++()
			{	// ++iterator
				if (pos == ref->size_raw()) return *this;
				const uchar16_t* a = ref->c_str();
				if (UTF16_IS_SURROGATE_HI(a[pos]))
					pos += 2;			// TODO: check for valid low surrogate?
				else ++pos;
				if (pos > ref->size_raw()) pos = ref->size_raw();
				return *this;
			}

			//! Switch to the next full character in the string, returning the previous position.
			_Iter operator++(int)
			{	// iterator++
				_Iter _tmp(*this);
				++*this;
				return _tmp;
			}

			//! Switch to the previous full character in the string.
			_Iter& operator--()
			{	// --iterator
				if (pos == 0) return *this;
				const uchar16_t* a = ref->c_str();
				--pos;
				if (UTF16_IS_SURROGATE_LO(a[pos]) && pos != 0)	// low surrogate, go back one more.
					--pos;
				return *this;
			}

			//! Switch to the previous full character in the string, returning the previous position.
			_Iter operator--(int)
			{	// iterator--
				_Iter _tmp(*this);
				--*this;
				return _tmp;
			}

			//! Advance a specified number of full characters in the string.
			//! \return Myself.
			_Iter& operator+=(const difference_type v)
			{
				if (v == 0) return *this;
				if (v < 0) return operator-=(v * -1);

				if (pos >= ref->size_raw())
					return *this;

				// Go to the appropriate position.
				// TODO: Don't force u32 on an x64 OS.  Make it agnostic.
				u32 i = (u32)v;
				u32 sr = ref->size_raw();
				const uchar16_t* a = ref->c_str();
				while (i != 0 && pos < sr)
				{
					if (UTF16_IS_SURROGATE_HI(a[pos]))
						pos += 2;
					else ++pos;
					--i;
				}
				if (pos > sr)
					pos = sr;

				return *this;
			}

			//! Go back a specified number of full characters in the string.
			//! \return Myself.
			_Iter& operator-=(const difference_type v)
			{
				if (v == 0) return *this;
				if (v > 0) return operator+=(v * -1);

				if (pos == 0)
					return *this;

				// Go to the appropriate position.
				// TODO: Don't force u32 on an x64 OS.  Make it agnostic.
				u32 i = (u32)v;
				const uchar16_t* a = ref->c_str();
				while (i != 0 && pos != 0)
				{
					--pos;
					if (UTF16_IS_SURROGATE_LO(a[pos]) != 0 && pos != 0)
						--pos;
					--i;
				}

				return *this;
			}

			//! Return a new iterator that is a variable number of full characters forward from the current position.
			_Iter operator+(const difference_type v) const
			{
				_Iter ret(*this);
				ret += v;
				return ret;
			}

			//! Return a new iterator that is a variable number of full characters backward from the current position.
			_Iter operator-(const difference_type v) const
			{
				_Iter ret(*this);
				ret -= v;
				return ret;
			}

			//! Returns the distance between two iterators.
			difference_type operator-(const _Iter& iter) const
			{
				// Make sure we reference the same object!
				if (ref != iter.ref)
					return difference_type();

				_Iter i = iter;
				difference_type ret;

				// Walk up.
				if (pos > i.pos)
				{
					while (pos > i.pos)
					{
						++i;
						++ret;
					}
					return ret;
				}

				// Walk down.
				while (pos < i.pos)
				{
					--i;
					--ret;
				}
				return ret;
			}

			//! Accesses the full character at the iterator's position.
			const_reference operator*() const
			{
				if (pos >= ref->size_raw())
				{
					const uchar16_t* a = ref->c_str();
					u32 p = ref->size_raw();
					if (UTF16_IS_SURROGATE_LO(a[p]))
						--p;
					reference ret(ref, p);
					return ret;
				}
				const_reference ret(ref, pos);
				return ret;
			}

			//! Accesses the full character at the iterator's position.
			reference operator*()
			{
				if (pos >= ref->size_raw())
				{
					const uchar16_t* a = ref->c_str();
					u32 p = ref->size_raw();
					if (UTF16_IS_SURROGATE_LO(a[p]))
						--p;
					reference ret(ref, p);
					return ret;
				}
				reference ret(ref, pos);
				return ret;
			}

			//! Accesses the full character at the iterator's position.
			const_pointer operator->() const
			{
				return operator*();
			}

			//! Accesses the full character at the iterator's position.
			pointer operator->()
			{
				return operator*();
			}

			//! Is the iterator at the start of the string?
			bool atStart() const
			{
				return pos == 0;
			}

			//! Is the iterator at the end of the string?
			bool atEnd() const
			{
				const uchar16_t* a = ref->c_str();
				if (UTF16_IS_SURROGATE(a[pos]))
					return (pos + 1) >= ref->size_raw();
				else return pos >= ref->size_raw();
			}

			//! Moves the iterator to the start of the string.
			void toStart()
			{
				pos = 0;
			}

			//! Moves the iterator to the end of the string.
			void toEnd()
			{
				pos = ref->size_raw();
			}

			//! Returns the iterator's position.
			//! \return The iterator's position.
			u32 getPos() const
			{
				return pos;
			}

		protected:
			const ustring16<TAlloc>* ref;
			u32 pos;
	};

	//! Iterator to iterate through a UTF-16 string.
	class _ustring16_iterator : public _ustring16_const_iterator
	{
		public:
			typedef _ustring16_iterator _Iter;
			typedef _ustring16_const_iterator _Base;
			typedef typename _Base::const_pointer const_pointer;
			typedef typename _Base::const_reference const_reference;


			typedef typename _Base::value_type value_type;
			typedef typename _Base::difference_type difference_type;
			typedef typename _Base::distance_type distance_type;
			typedef access pointer;
			typedef access reference;

			using _Base::pos;
			using _Base::ref;

			//! Constructors.
			_ustring16_iterator(const _Iter& i) : _ustring16_const_iterator(i) {}
			_ustring16_iterator(const ustring16<TAlloc>& s) : _ustring16_const_iterator(s) {}
			_ustring16_iterator(const ustring16<TAlloc>& s, const u32 p) : _ustring16_const_iterator(s, p) {}

			//! Accesses the full character at the iterator's position.
			reference operator*() const
			{
				if (pos >= ref->size_raw())
				{
					const uchar16_t* a = ref->c_str();
					u32 p = ref->size_raw();
					if (UTF16_IS_SURROGATE_LO(a[p]))
						--p;
					reference ret(ref, p);
					return ret;
				}
				reference ret(ref, pos);
				return ret;
			}

			//! Accesses the full character at the iterator's position.
			reference operator*()
			{
				if (pos >= ref->size_raw())
				{
					const uchar16_t* a = ref->c_str();
					u32 p = ref->size_raw();
					if (UTF16_IS_SURROGATE_LO(a[p]))
						--p;
					reference ret(ref, p);
					return ret;
				}
				reference ret(ref, pos);
				return ret;
			}

			//! Accesses the full character at the iterator's position.
			pointer operator->() const
			{
				return operator*();
			}

			//! Accesses the full character at the iterator's position.
			pointer operator->()
			{
				return operator*();
			}
	};

	typedef typename ustring16<TAlloc>::_ustring16_iterator iterator;
	typedef typename ustring16<TAlloc>::_ustring16_const_iterator const_iterator;

	///----------------------///
	/// end iterator classes ///
	///----------------------///

	//! Default constructor
	ustring16()
	: array(0), allocated(1), used(0)
	{
#if __BYTE_ORDER == __BIG_ENDIAN
		encoding = unicode::EUTFE_UTF16_BE;
#else
		encoding = unicode::EUTFE_UTF16_LE;
#endif
		array = allocator.allocate(1); // new u16[1];
		array[0] = 0x0;
	}


	//! Constructor
	ustring16(const ustring16<TAlloc>& other)
	: array(0), allocated(0), used(0)
	{
#if __BYTE_ORDER == __BIG_ENDIAN
		encoding = unicode::EUTFE_UTF16_BE;
#else
		encoding = unicode::EUTFE_UTF16_LE;
#endif
		*this = other;
	}


	//! Constructor from other string types
	template <class B, class A>
	ustring16(const string<B, A>& other)
	: array(0), allocated(0), used(0)
	{
#if __BYTE_ORDER == __BIG_ENDIAN
		encoding = unicode::EUTFE_UTF16_BE;
#else
		encoding = unicode::EUTFE_UTF16_LE;
#endif
		*this = other;
	}


#ifndef USTRING_NO_STL
	//! Constructor from std::string
	template <class B, class A, typename Alloc>
	ustring16(const std::basic_string<B, A, Alloc>& other)
	: array(0), allocated(0), used(0)
	{
#if __BYTE_ORDER == __BIG_ENDIAN
		encoding = unicode::EUTFE_UTF16_BE;
#else
		encoding = unicode::EUTFE_UTF16_LE;
#endif
		*this = other.c_str();
	}


	//! Constructor from iterator.
	template <typename Itr>
	ustring16(Itr first, Itr last)
	: array(0), allocated(0), used(0)
	{
#if __BYTE_ORDER == __BIG_ENDIAN
		encoding = unicode::EUTFE_UTF16_BE;
#else
		encoding = unicode::EUTFE_UTF16_LE;
#endif
		reserve(std::distance(first, last));
		array[used] = 0;

		for (; first != last; ++first)
			append((uchar32_t)*first);
	}
#endif


#ifndef USTRING_CPP0X_NEWLITERALS
	//! Constructor for copying a character string from a pointer.
	ustring16(const char* const c)
	: array(0), allocated(0), used(0)
	{
#if __BYTE_ORDER == __BIG_ENDIAN
		encoding = unicode::EUTFE_UTF16_BE;
#else
		encoding = unicode::EUTFE_UTF16_LE;
#endif

		loadDataStream(c, strlen(c));
		//append((uchar8_t*)c);
	}


	//! Constructor for copying a character string from a pointer with a given length.
	ustring16(const char* const c, u32 length)
	: array(0), allocated(0), used(0)
	{
#if __BYTE_ORDER == __BIG_ENDIAN
		encoding = unicode::EUTFE_UTF16_BE;
#else
		encoding = unicode::EUTFE_UTF16_LE;
#endif

		loadDataStream(c, length);
	}
#endif


	//! Constructor for copying a UTF-8 string from a pointer.
	ustring16(const uchar8_t* const c)
	: array(0), allocated(0), used(0)
	{
#if __BYTE_ORDER == __BIG_ENDIAN
		encoding = unicode::EUTFE_UTF16_BE;
#else
		encoding = unicode::EUTFE_UTF16_LE;
#endif

		append(c);
	}


	//! Constructor for copying a UTF-8 string from a single char.
	ustring16(const char c)
	: array(0), allocated(0), used(0)
	{
#if __BYTE_ORDER == __BIG_ENDIAN
		encoding = unicode::EUTFE_UTF16_BE;
#else
		encoding = unicode::EUTFE_UTF16_LE;
#endif

		append((uchar32_t)c);
	}


	//! Constructor for copying a UTF-8 string from a pointer with a given length.
	ustring16(const uchar8_t* const c, u32 length)
	: array(0), allocated(0), used(0)
	{
#if __BYTE_ORDER == __BIG_ENDIAN
		encoding = unicode::EUTFE_UTF16_BE;
#else
		encoding = unicode::EUTFE_UTF16_LE;
#endif

		append(c, length);
	}


	//! Constructor for copying a UTF-16 string from a pointer.
	ustring16(const uchar16_t* const c)
	: array(0), allocated(0), used(0)
	{
#if __BYTE_ORDER == __BIG_ENDIAN
		encoding = unicode::EUTFE_UTF16_BE;
#else
		encoding = unicode::EUTFE_UTF16_LE;
#endif

		append(c);
	}


	//! Constructor for copying a UTF-16 string from a pointer with a given length
	ustring16(const uchar16_t* const c, u32 length)
	: array(0), allocated(0), used(0)
	{
#if __BYTE_ORDER == __BIG_ENDIAN
		encoding = unicode::EUTFE_UTF16_BE;
#else
		encoding = unicode::EUTFE_UTF16_LE;
#endif

		append(c, length);
	}


	//! Constructor for copying a UTF-32 string from a pointer.
	ustring16(const uchar32_t* const c)
	: array(0), allocated(0), used(0)
	{
#if __BYTE_ORDER == __BIG_ENDIAN
		encoding = unicode::EUTFE_UTF16_BE;
#else
		encoding = unicode::EUTFE_UTF16_LE;
#endif

		append(c);
	}


	//! Constructor for copying a UTF-32 from a pointer with a given length.
	ustring16(const uchar32_t* const c, u32 length)
	: array(0), allocated(0), used(0)
	{
#if __BYTE_ORDER == __BIG_ENDIAN
		encoding = unicode::EUTFE_UTF16_BE;
#else
		encoding = unicode::EUTFE_UTF16_LE;
#endif

		append(c, length);
	}


	//! Constructor for copying a wchar_t string from a pointer.
	ustring16(const wchar_t* const c)
	: array(0), allocated(0), used(0)
	{
#if __BYTE_ORDER == __BIG_ENDIAN
		encoding = unicode::EUTFE_UTF16_BE;
#else
		encoding = unicode::EUTFE_UTF16_LE;
#endif

		if (sizeof(wchar_t) == 4)
			append(reinterpret_cast<const uchar32_t* const>(c));
		else if (sizeof(wchar_t) == 2)
			append(reinterpret_cast<const uchar16_t* const>(c));
		else if (sizeof(wchar_t) == 1)
			append(reinterpret_cast<const uchar8_t* const>(c));
	}


	//! Constructor for copying a wchar_t string from a pointer with a given length.
	ustring16(const wchar_t* const c, u32 length)
	: array(0), allocated(0), used(0)
	{
#if __BYTE_ORDER == __BIG_ENDIAN
		encoding = unicode::EUTFE_UTF16_BE;
#else
		encoding = unicode::EUTFE_UTF16_LE;
#endif

		if (sizeof(wchar_t) == 4)
			append(reinterpret_cast<const uchar32_t* const>(c), length);
		else if (sizeof(wchar_t) == 2)
			append(reinterpret_cast<const uchar16_t* const>(c), length);
		else if (sizeof(wchar_t) == 1)
			append(reinterpret_cast<const uchar8_t* const>(c), length);
	}


#ifdef USTRING_CPP0X
	//! Constructor for moving a ustring16
	ustring16(ustring16<TAlloc>&& other)
	: array(other.array), encoding(other.encoding), allocated(other.allocated), used(other.used)
	{
		//std::cout << "MOVE constructor" << std::endl;
		other.array = 0;
		other.allocated = 0;
		other.used = 0;
	}
#endif


	//! Destructor
	~ustring16()
	{
		allocator.deallocate(array); // delete [] array;
	}


	//! Assignment operator
	ustring16& operator=(const ustring16<TAlloc>& other)
	{
		if (this == &other)
			return *this;

		used = other.size_raw();
		if (used >= allocated)
		{
			allocator.deallocate(array); // delete [] array;
			allocated = used + 1;
			array = allocator.allocate(used + 1); //new u16[used];
		}

		const uchar16_t* p = other.c_str();
		for (u32 i=0; i<=used; ++i, ++p)
			array[i] = *p;

		array[used] = 0;

		// Validate our new UTF-16 string.
		validate();

		return *this;
	}


#ifdef USTRING_CPP0X
	//! Move assignment operator
	ustring16& operator=(ustring16<TAlloc>&& other)
	{
		if (this != &other)
		{
			//std::cout << "MOVE operator=" << std::endl;
			allocator.deallocate(array);

			array = other.array;
			allocated = other.allocated;
			encoding = other.encoding;
			used = other.used;
			other.array = 0;
			other.used = 0;
		}
		return *this;
	}
#endif


	//! Assignment operator for other string types
	template <class B, class A>
	ustring16<TAlloc>& operator=(const string<B, A>& other)
	{
		*this = other.c_str();
		return *this;
	}


	//! Assignment operator for UTF-8 strings
	ustring16<TAlloc>& operator=(const uchar8_t* const c)
	{
		if (!array)
		{
			array = allocator.allocate(1); //new u16[1];
			allocated = 1;
		}
		used = 0;
		array[used] = 0x0;
		if (!c) return *this;

		//! Append our string now.
		append(c);
		return *this;
	}


	//! Assignment operator for UTF-16 strings
	ustring16<TAlloc>& operator=(const uchar16_t* const c)
	{
		if (!array)
		{
			array = allocator.allocate(1); //new u16[1];
			allocated = 1;
		}
		used = 0;
		array[used] = 0x0;
		if (!c) return *this;

		//! Append our string now.
		append(c);
		return *this;
	}


	//! Assignment operator for UTF-32 strings
	ustring16<TAlloc>& operator=(const uchar32_t* const c)
	{
		if (!array)
		{
			array = allocator.allocate(1); //new u16[1];
			allocated = 1;
		}
		used = 0;
		array[used] = 0x0;
		if (!c) return *this;

		//! Append our string now.
		append(c);
		return *this;
	}


	//! Assignment operator for wchar_t strings.
	/** Note that this assumes that a correct unicode string is stored in the wchar_t string.
		Since wchar_t changes depending on its platform, it could either be a UTF-8, -16, or -32 string.
		This function assumes you are storing the correct unicode encoding inside the wchar_t string. **/
	ustring16<TAlloc>& operator=(const wchar_t* const c)
	{
		if (sizeof(wchar_t) == 4)
			*this = reinterpret_cast<const uchar32_t* const>(c);
		else if (sizeof(wchar_t) == 2)
			*this = reinterpret_cast<const uchar16_t* const>(c);
		else if (sizeof(wchar_t) == 1)
			*this = reinterpret_cast<const uchar8_t* const>(c);

		return *this;
	}


	//! Assignment operator for other strings.
	/** Note that this assumes that a correct unicode string is stored in the string. **/
	template <class B>
	ustring16<TAlloc>& operator=(const B* const c)
	{
		if (sizeof(B) == 4)
			*this = reinterpret_cast<const uchar32_t* const>(c);
		else if (sizeof(B) == 2)
			*this = reinterpret_cast<const uchar16_t* const>(c);
		else if (sizeof(B) == 1)
			*this = reinterpret_cast<const uchar8_t* const>(c);

		return *this;
	}


	//! Direct access operator
	access operator [](const u32 index)
	{
		_IRR_DEBUG_BREAK_IF(index>=size()) // bad index
		iterator iter(*this, index);
		return iter.operator*();
	}


	//! Direct access operator
	const access operator [](const u32 index) const
	{
		_IRR_DEBUG_BREAK_IF(index>=size()) // bad index
		const_iterator iter(*this, index);
		return iter.operator*();
	}


	//! Equality operator
	bool operator ==(const uchar16_t* const str) const
	{
		if (!str)
			return false;

		u32 i;
		for(i=0; array[i] && str[i]; ++i)
			if (array[i] != str[i])
				return false;

		return !array[i] && !str[i];
	}


	//! Equality operator
	bool operator ==(const ustring16<TAlloc>& other) const
	{
		for(u32 i=0; array[i] && other.array[i]; ++i)
			if (array[i] != other.array[i])
				return false;

		return used == other.used;
	}


	//! Is smaller comparator
	bool operator <(const ustring16<TAlloc>& other) const
	{
		for(u32 i=0; array[i] && other.array[i]; ++i)
		{
			s32 diff = array[i] - other.array[i];
			if ( diff )
				return diff < 0;
		}

		return used < other.used;
	}


	//! Inequality operator
	bool operator !=(const uchar16_t* const str) const
	{
		return !(*this == str);
	}


	//! Inequality operator
	bool operator !=(const ustring16<TAlloc>& other) const
	{
		return !(*this == other);
	}


	//! Returns the length of a ustring16 in full characters.
	//! \return Length of a ustring16 in full characters.
	u32 size() const
	{
		const_iterator i(*this, 0);
		u32 pos = 0;
		while (!i.atEnd())
		{
			++i;
			++pos;
		}
		return pos;
	}


	//! Informs if the ustring is empty or not.
	//! \return True if the ustring is empty, false if not.
	bool empty() const
	{
		return (size_raw() == 0);
	}


	//! Returns a pointer to the raw UTF-16 string data.
	//! \return pointer to C-style NUL terminated array of UTF-16 code points.
	const uchar16_t* c_str() const
	{
		return array;
	}


	//! Compares the first n characters of this string with another.
	//! \param other Other string to compare to.
	//! \param n Number of characters to compare.
	//! \return True if the n first characters of both strings are equal.
	bool equalsn(const ustring16<TAlloc>& other, u32 n) const
	{
		u32 i;
		const uchar16_t* oa = other.c_str();
		for(i=0; array[i] && oa[i] && i < n; ++i)
			if (array[i] != oa[i])
				return false;

		// if one (or both) of the strings was smaller then they
		// are only equal if they have the same length
		return (i == n) || (used == other.used);
	}


	//! Compares the first n characters of this string with another.
	//! \param str Other string to compare to.
	//! \param n Number of characters to compare.
	//! \return True if the n first characters of both strings are equal.
	bool equalsn(const uchar16_t* const str, u32 n) const
	{
		if (!str)
			return false;
		u32 i;
		for(i=0; array[i] && str[i] && i < n; ++i)
			if (array[i] != str[i])
				return false;

		// if one (or both) of the strings was smaller then they
		// are only equal if they have the same length
		return (i == n) || (array[i] == 0 && str[i] == 0);
	}


	//! Appends a character to this ustring16
	//! \param character The character to append.
	//! \return A reference to our current string.
	ustring16<TAlloc>& append(uchar32_t character)
	{
		if (used + 2 >= allocated)
			reallocate(used + 2);

		if (character > 0xFFFF)
		{
			used += 2;

			// character will be multibyte, so split it up into a surrogate pair.
			uchar16_t x = static_cast<uchar16_t>(character);
			uchar16_t vh = UTF16_HI_SURROGATE | ((((character >> 16) & ((1 << 5) - 1)) - 1) << 6) | (x >> 10);
			uchar16_t vl = UTF16_LO_SURROGATE | (x & ((1 << 10) - 1));
			array[used-2] = vh;
			array[used-1] = vl;
		}
		else
		{
			++used;
			array[used-1] = character;
		}
		array[used] = 0;

		return *this;
	}


	//! Appends a UTF-8 string to this ustring16
	//! \param other The UTF-8 string to append.
	//! \param length The length of the string to append.
	//! \return A reference to our current string.
	ustring16<TAlloc>& append(const uchar8_t* const other, u32 length=0xffffffff)
	{
		if (!other)
			return *this;

		// Determine if the string is long enough for a BOM.
		u32 len = 0;
		const uchar8_t* p = other;
		do
		{
			++len;
		} while (*p++ && len < unicode::BOM_ENCODE_UTF8_LEN);

		// Check for BOM.
		unicode::EUTF_ENCODE c_bom = unicode::EUTFE_NONE;
		if (len == unicode::BOM_ENCODE_UTF8_LEN)
		{
			if (memcmp(other, unicode::BOM_ENCODE_UTF8, unicode::BOM_ENCODE_UTF8_LEN) == 0)
				c_bom = unicode::EUTFE_UTF8;
		}

		// If a BOM was found, don't include it in the string.
		const uchar8_t* c2 = other;
		if (c_bom != unicode::EUTFE_NONE)
		{
			c2 = other + unicode::BOM_UTF8_LEN;
			length -= unicode::BOM_UTF8_LEN;
		}

		// Calculate the size of the string to read in.
		len = 0;
		p = c2;
		do
		{
			++len;
		} while(*p++ && len < length);
		if (len > length)
			len = length;

		// If we need to grow the array, do it now.
		if (used + len >= allocated)
			reallocate(used + (len * 2));
		u32 start = used;

		// Convert UTF-8 to UTF-16.
		u32 pos = start;
		for (u32 l = 0; l<len;)
		{
			++used;
			if (((c2[l] >> 6) & 0x03) == 0x02)
			{	// Invalid continuation byte.
				array[pos++] = unicode::UTF_REPLACEMENT_CHARACTER;
				++l;
			}
			else if (c2[l] == 0xC0 || c2[l] == 0xC1)
			{	// Invalid byte - overlong encoding.
				array[pos++] = unicode::UTF_REPLACEMENT_CHARACTER;
				++l;
			}
			else if ((c2[l] & 0xF8) == 0xF0)
			{	// 4 bytes UTF-8, 2 bytes UTF-16.
				// Check for a full string.
				if ((l + 3) >= len)
				{
					array[pos++] = unicode::UTF_REPLACEMENT_CHARACTER;
					l += 3;
					break;
				}

				// Validate.
				bool valid = true;
				u8 l2 = 0;
				if (valid && (((c2[l+1] >> 6) & 0x03) == 0x02)) ++l2; else valid = false;
				if (valid && (((c2[l+2] >> 6) & 0x03) == 0x02)) ++l2; else valid = false;
				if (valid && (((c2[l+3] >> 6) & 0x03) == 0x02)) ++l2; else valid = false;
				if (!valid)
				{
					array[pos++] = unicode::UTF_REPLACEMENT_CHARACTER;
					l += l2;
					continue;
				}

				// Decode.
				uchar8_t b1 = ((c2[l] & 0x7) << 2) | ((c2[l+1] >> 4) & 0x3);
				uchar8_t b2 = ((c2[l+1] & 0xF) << 4) | ((c2[l+2] >> 2) & 0xF);
				uchar8_t b3 = ((c2[l+2] & 0x3) << 6) | (c2[l+3] & 0x3F);
				uchar32_t v = b3 | ((uchar32_t)b2 << 8) | ((uchar32_t)b1 << 16);

				// Split v up into a surrogate pair.
				uchar16_t x = static_cast<uchar16_t>(v);
				uchar16_t vh = UTF16_HI_SURROGATE | ((((v >> 16) & ((1 << 5) - 1)) - 1) << 6) | (x >> 10);
				uchar16_t vl = UTF16_LO_SURROGATE | (x & ((1 << 10) - 1));

				array[pos++] = vh;
				array[pos++] = vl;
				l += 4;
				++used;		// Using two shorts this time, so increase used by 1.
			}
			else if ((c2[l] & 0xF0) == 0xE0)
			{	// 3 bytes UTF-8, 1 byte UTF-16.
				// Check for a full string.
				if ((l + 2) >= len)
				{
					array[pos++] = unicode::UTF_REPLACEMENT_CHARACTER;
					l += 2;
					break;
				}

				// Validate.
				bool valid = true;
				u8 l2 = 0;
				if (valid && (((c2[l+1] >> 6) & 0x03) == 0x02)) ++l2; else valid = false;
				if (valid && (((c2[l+2] >> 6) & 0x03) == 0x02)) ++l2; else valid = false;
				if (!valid)
				{
					array[pos++] = unicode::UTF_REPLACEMENT_CHARACTER;
					l += l2;
					continue;
				}

				// Decode.
				uchar8_t b1 = ((c2[l] & 0xF) << 4) | ((c2[l+1] >> 2) & 0xF);
				uchar8_t b2 = ((c2[l+1] & 0x3) << 6) | (c2[l+2] & 0x3F);
				uchar16_t ch = b2 | ((uchar16_t)b1 << 8);
				array[pos++] = ch;
				l += 3;
			}
			else if ((c2[l] & 0xE0) == 0xC0)
			{	// 2 bytes UTF-8, 1 byte UTF-16.
				// Check for a full string.
				if ((l + 1) >= len)
				{
					array[pos++] = unicode::UTF_REPLACEMENT_CHARACTER;
					l += 1;
					break;
				}

				// Validate.
				if (((c2[l+1] >> 6) & 0x03) != 0x02)
				{
					array[pos++] = unicode::UTF_REPLACEMENT_CHARACTER;
					++l;
					continue;
				}

				// Decode.
				uchar8_t b1 = (c2[l] >> 2) & 0x7;
				uchar8_t b2 = ((c2[l] & 0x3) << 6) | (c2[l+1] & 0x3F);
				uchar16_t ch = b2 | ((uchar16_t)b1 << 8);
				array[pos++] = ch;
				l += 2;
			}
			else
			{	// 1 byte UTF-8, 1 byte UTF-16.
				// Validate.
				if (c2[l] > 0x7F)
				{	// Values above 0xF4 are restricted and aren't used.  By now, anything above 0x7F is invalid.
					array[pos++] = unicode::UTF_REPLACEMENT_CHARACTER;
				}
				else array[pos++] = static_cast<uchar16_t>(c2[l]);
				++l;
			}
		}
		array[used] = 0;

		// Validate our new UTF-16 string.
		validate();

		return *this;
	}


	//! Appends a UTF-16 string to this ustring16
	//! \param other The UTF-16 string to append.
	//! \param length The length of the string to append.
	//! \return A reference to our current string.
	ustring16<TAlloc>& append(const uchar16_t* const other, u32 length=0xffffffff)
	{
		if (!other)
			return *this;

		// Determine if the string is long enough for a BOM.
		u32 len = 0;
		const uchar16_t* p = other;
		do
		{
			++len;
		} while (*p++ && len < unicode::BOM_ENCODE_UTF16_LEN);

		// Check for the BOM to determine the string's endianness.
		unicode::EUTF_ENDIAN c_end = unicode::EUTFEE_NATIVE;
		if (memcmp(other, unicode::BOM_ENCODE_UTF16_LE, unicode::BOM_ENCODE_UTF16_LEN) == 0)
			c_end = unicode::EUTFEE_LITTLE;
		else if (memcmp(other, unicode::BOM_ENCODE_UTF16_BE, unicode::BOM_ENCODE_UTF16_LEN) == 0)
			c_end = unicode::EUTFEE_BIG;

		// If a BOM was found, don't include it in the string.
		const uchar16_t* c2 = other;
		if (c_end != unicode::EUTFEE_NATIVE)
		{
			c2 = other + unicode::BOM_UTF16_LEN;
			length -= unicode::BOM_UTF16_LEN;
		}

		// Calculate the size of the string to read in.
		len = 0;
		p = c2;
		do
		{
			++len;
		} while(*p++ && len < length);
		if (len > length)
			len = length;

		// If we need to grow the size of the array, do it now.
		if (used + len >= allocated)
			reallocate(used + (len * 2));
		u32 start = used;
		used += len;

		// Copy the string now.
		unicode::EUTF_ENDIAN m_end = getEndianness();
		for (u32 l = start; l < start + len; ++l)
		{
			array[l] = (uchar16_t)c2[l];
			if (c_end != unicode::EUTFEE_NATIVE && c_end != m_end)
				array[l] = unicode::swapEndian16(array[l]);
		}

		array[used] = 0;

		// Validate our new UTF-16 string.
		validate();
		return *this;
	}


	//! Appends a UTF-32 string to this ustring16
	//! \param other The UTF-32 string to append.
	//! \param length The length of the string to append.
	//! \return A reference to our current string.
	ustring16<TAlloc>& append(const uchar32_t* const other, u32 length=0xffffffff)
	{
		if (!other)
			return *this;

		// Check for the BOM to determine the string's endianness.
		unicode::EUTF_ENDIAN c_end = unicode::EUTFEE_NATIVE;
		if (memcmp(other, unicode::BOM_ENCODE_UTF32_LE, unicode::BOM_ENCODE_UTF32_LEN) == 0)
			c_end = unicode::EUTFEE_LITTLE;
		else if (memcmp(other, unicode::BOM_ENCODE_UTF32_BE, unicode::BOM_ENCODE_UTF32_LEN) == 0)
			c_end = unicode::EUTFEE_BIG;

		// If a BOM was found, don't include it in the string.
		const uchar32_t* c2 = other;
		if (c_end != unicode::EUTFEE_NATIVE)
		{
			c2 = other + unicode::BOM_UTF32_LEN;
			length -= unicode::BOM_UTF32_LEN;
		}

		// Calculate the size of the string to read in.
		u32 len = 0;
		const uchar32_t* p = c2;
		do
		{
			++len;
		} while(*p++ && len < length);
		if (len > length)
			len = length;

		// If we need to grow the size of the array, do it now.
		// In case all of the UTF-32 string is split into surrogate pairs, do len * 2.
		if (used + (len * 2) >= allocated)
			reallocate(used + ((len * 2) * 2));
		u32 start = used;

		// Convert UTF-32 to UTF-16.
		unicode::EUTF_ENDIAN m_end = getEndianness();
		u32 pos = start;
		for (u32 l = 0; l<len; ++l)
		{
			++used;

			uchar32_t ch = c2[l];
			if (c_end != unicode::EUTFEE_NATIVE && c_end != m_end)
				ch = unicode::swapEndian32(ch);

			if (ch > 0xFFFF)
			{
				// Split ch up into a surrogate pair as it is over 16 bits long.
				uchar16_t x = static_cast<uchar16_t>(ch);
				uchar16_t vh = UTF16_HI_SURROGATE | ((((ch >> 16) & ((1 << 5) - 1)) - 1) << 6) | (x >> 10);
				uchar16_t vl = UTF16_LO_SURROGATE | (x & ((1 << 10) - 1));
				array[pos++] = vh;
				array[pos++] = vl;
				++used;		// Using two shorts, so increased used again.
			}
			else if (ch >= 0xD800 && ch <= 0xDFFF)
			{
				// Between possible UTF-16 surrogates (invalid!)
				array[pos++] = unicode::UTF_REPLACEMENT_CHARACTER;
			}
			else array[pos++] = static_cast<uchar16_t>(ch);
		}
		array[used] = 0;

		// Validate our new UTF-16 string.
		validate();

		return *this;
	}


	//! Appends a ustring16 to this ustring16
	//! \param other The string to append to this one.
	//! \return A reference to our current string.
	ustring16<TAlloc>& append(const ustring16<TAlloc>& other)
	{
		const uchar16_t* oa = other.c_str();

		u32 len = other.size_raw();

		if (used + len >= allocated)
			reallocate(used + len);

		for (u32 l=0; l<len; ++l)
			array[used+l] = oa[l];

		used += len;
		array[used] = 0;

		return *this;
	}


	//! Appends a certain amount of characters of a ustring16 to this ustring16.
	//! \param other The string to append to this one.
	//! \param length How many characters of the other string to add to this one.
	//! \return A reference to our current string.
	ustring16<TAlloc>& append(const ustring16<TAlloc>& other, u32 length)
	{
		if (other.size() == 0)
			return *this;

		if (other.size() < length)
		{
			append(other);
			return *this;
		}

		if (used + length * 2 >= allocated)
			reallocate(used + length * 2);

		const_iterator iter(other, 0);
		u32 l = length;
		while (!iter.atEnd() && l)
		{
			uchar32_t c = *iter;
			append(c);
			++iter;
			--l;
		}

		return *this;
	}


	//! Reserves some memory.
	//! \param count The amount of characters to reserve.
	void reserve(u32 count)
	{
		if (count < allocated)
			return;

		reallocate(count);
	}


	//! Finds first occurrence of character.
	//! \param c The character to search for.
	//! \return Position where the character has been found, or -1 if not found.
	s32 findFirst(uchar32_t c) const
	{
		const_iterator i(*this, 0);

		s32 pos = 0;
		while (!i.atEnd())
		{
			uchar32_t t = *i;
			if (c == t)
				return pos;
			++pos;
			++i;
		}

		return -1;
	}

	//! Finds first occurrence of a character of a list.
	//! \param c A list of characters to find. For example if the method should find the first occurrence of 'a' or 'b', this parameter should be "ab".
	//! \param count The amount of characters in the list. Usually, this should be strlen(c).
	//! \return Position where one of the characters has been found, or -1 if not found.
	s32 findFirstChar(const uchar32_t* const c, u32 count=1) const
	{
		if (!c || !count)
			return -1;

		const_iterator i(*this, 0);

		s32 pos = 0;
		while (!i.atEnd())
		{
			uchar32_t t = *i;
			for (u32 j=0; j<count; ++j)
				if (t == c[j])
					return pos;
			++pos;
			++i;
		}

		return -1;
	}


	//! Finds first position of a character not in a given list.
	//! \param c A list of characters to NOT find. For example if the method should find the first occurrence of a character not 'a' or 'b', this parameter should be "ab".
	//! \param count The amount of characters in the list. Usually, this should be strlen(c).
	//! \return Position where the character has been found, or -1 if not found.
	s32 findFirstCharNotInList(const uchar32_t* const c, u32 count=1) const
	{
		if (!c || !count)
			return -1;

		const_iterator i(*this, 0);

		s32 pos = 0;
		while (!i.atEnd())
		{
			uchar32_t t = *i;
			u32 j;
			for (j=0; j<count; ++j)
				if (t == c[j])
					break;

			if (j==count)
				return pos;
			++pos;
			++i;
		}

		return -1;
	}

	//! Finds last position of a character not in a given list.
	//! \param c A list of characters to NOT find. For example if the method should find the first occurrence of a character not 'a' or 'b', this parameter should be "ab".
	//! \param count The amount of characters in the list. Usually, this should be strlen(c).
	//! \return Position where the character has been found, or -1 if not found.
	s32 findLastCharNotInList(const uchar32_t* const c, u32 count=1) const
	{
		if (!c || !count)
			return -1;

		const_iterator i(end());
		--i;

		s32 pos = size() - 1;
		while (!i.atStart())
		{
			uchar32_t t = *i;
			u32 j;
			for (j=0; j<count; ++j)
				if (t == c[j])
					break;

			if (j==count)
				return pos;
			--pos;
			--i;
		}

		return -1;
	}

	//! Finds next occurrence of character.
	//! \param c The character to search for.
	//! \param startPos The position in the string to start searching.
	//! \return Position where the character has been found, or -1 if not found.
	s32 findNext(uchar32_t c, u32 startPos) const
	{
		const_iterator i(*this, startPos);

		s32 pos = startPos;
		while (!i.atEnd())
		{
			uchar32_t t = *i;
			if (t == c)
				return pos;
			++pos;
			++i;
		}

		return -1;
	}


	//! Finds last occurrence of character.
	//! \param c The character to search for.
	//! \param start The start position of the reverse search ( default = -1, on end ).
	//! \return Position where the character has been found, or -1 if not found.
	s32 findLast(uchar32_t c, s32 start = -1) const
	{
		u32 s = size();
		start = core::clamp ( start < 0 ? (s32)s : start, 0, (s32)s ) - 1;

		const_iterator i(*this, start);
		u32 pos = start;
		while (!i.atStart())
		{
			uchar32_t t = *i;
			if (t == c)
				return pos;
			--pos;
			--i;
		}

		return -1;
	}

	//! Finds last occurrence of a character in a list.
	//! \param c A list of strings to find. For example if the method should find the last occurrence of 'a' or 'b', this parameter should be "ab".
	//! \param count The amount of characters in the list. Usually, this should be strlen(c).
	//! \return Position where one of the characters has been found, or -1 if not found.
	s32 findLastChar(const uchar32_t* const c, u32 count=1) const
	{
		if (!c || !count)
			return -1;

		const_iterator i(end());
		--i;

		s32 pos = size();
		while (!i.atStart())
		{
			uchar32_t t = *i;
			for (u32 j=0; j<count; ++j)
				if (t == c[j])
					return pos;
			--pos;
			--i;
		}

		return -1;
	}


	//! Finds another ustring16 in this ustring16.
	//! \param str The string to find.
	//! \param start The start position of the search.
	//! \return Positions where the ustring16 has been found, or -1 if not found.
	s32 find(const ustring16<TAlloc>& str, const u32 start = 0) const
	{
		u32 my_size = size();
		u32 their_size = str.size();

		if (their_size == 0 || my_size - start < their_size)
			return -1;

		const_iterator i(*this, start);

		s32 pos = start;
		while (!i.atEnd())
		{
			const_iterator i2(i);
			const_iterator j(str, 0);
			uchar32_t t1 = (uchar32_t)*i2;
			uchar32_t t2 = (uchar32_t)*j;
			while (t1 == t2)
			{
				++i2;
				++j;
				if (j.atEnd())
					return pos;
				t1 = (uchar32_t)*i2;
				t2 = (uchar32_t)*j;
			}
			++i;
			++pos;
		}

		return -1;
	}


	//! Finds another ustring16 in this ustring16.
	//! \param str The string to find.
	//! \param start The start position of the search.
	//! \return Positions where the string has been found, or -1 if not found.
	s32 find_raw(const ustring16<TAlloc>& str, const u32 start = 0) const
	{
		const uchar16_t* data = str.c_str();
		if (data && *data)
		{
			u32 len = 0;

			while (data[len])
				++len;

			if (len > used)
				return -1;

			for (u32 i=start; i<=used-len; ++i)
			{
				u32 j=0;

				while(data[j] && array[i+j] == data[j])
					++j;

				if (!data[j])
					return i;
			}
		}

		return -1;
	}


	//! Returns a substring.
	//! \param begin: Start of substring.
	//! \param length: Length of substring.
	//! \return A reference to our current string.
	ustring16<TAlloc> subString(u32 begin, s32 length) const
	{
		u32 len = size();
		// if start after ustring16
		// or no proper substring length
		if ((length <= 0) || (begin>=len))
			return ustring16<TAlloc>("");
		// clamp length to maximal value
		if ((length+begin) > len)
			length = len-begin;

		ustring16<TAlloc> o;
		o.reserve((length+1) * 2);

		const_iterator i(*this, begin);
		while (!i.atEnd() && length)
		{
			o.append(*i);
			++i;
			--length;
		}

		return o;
	}


	//! Appends a character to this ustring16.
	//! \param c Character to append.
	//! \return A reference to our current string.
	ustring16<TAlloc>& operator += (char c)
	{
		append((uchar32_t)c);
		return *this;
	}


	//! Appends a character to this ustring16.
	//! \param c Character to append.
	//! \return A reference to our current string.
	ustring16<TAlloc>& operator += (uchar32_t c)
	{
		append(c);
		return *this;
	}


	//! Appends a number to this ustring16.
	//! \param c Number to append.
	//! \return A reference to our current string.
	ustring16<TAlloc>& operator += (short c)
	{
		append(core::stringc(c));
		return *this;
	}


	//! Appends a number to this ustring16.
	//! \param c Number to append.
	//! \return A reference to our current string.
	ustring16<TAlloc>& operator += (unsigned short c)
	{
		append(core::stringc(c));
		return *this;
	}


#ifdef USTRING_CPP0X_NEWLITERALS
	//! Appends a number to this ustring16.
	//! \param c Number to append.
	//! \return A reference to our current string.
	ustring16<TAlloc>& operator += (int c)
	{
		append(core::stringc(c));
		return *this;
	}


	//! Appends a number to this ustring16.
	//! \param c Number to append.
	//! \return A reference to our current string.
	ustring16<TAlloc>& operator += (unsigned int c)
	{
		append(core::stringc(c));
		return *this;
	}
#endif


	//! Appends a number to this ustring16.
	//! \param c Number to append.
	//! \return A reference to our current string.
	ustring16<TAlloc>& operator += (long c)
	{
		append(core::stringc(c));
		return *this;
	}


	//! Appends a number to this ustring16.
	//! \param c Number to append.
	//! \return A reference to our current string.
	ustring16<TAlloc>& operator += (unsigned long c)
	{
		append(core::stringc(c));
		return *this;
	}


	//! Appends a number to this ustring16.
	//! \param c Number to append.
	//! \return A reference to our current string.
	ustring16<TAlloc>& operator += (double c)
	{
		append(core::stringc(c));
		return *this;
	}


	//! Appends a char ustring16 to this ustring16.
	//! \param c Char ustring16 to append.
	//! \return A reference to our current string.
	ustring16<TAlloc>& operator += (const uchar16_t* const c)
	{
		append(c);
		return *this;
	}


	//! Appends a ustring16 to this ustring16.
	//! \param other ustring16 to append.
	//! \return A reference to our current string.
	ustring16<TAlloc>& operator += (const ustring16<TAlloc>& other)
	{
		append(other);
		return *this;
	}


	//! Replaces all characters of a given type with another one.
	//! \param toReplace Character to replace.
	//! \param replaceWith Character replacing the old one.
	//! \return A reference to our current string.
	ustring16<TAlloc>& replace(uchar32_t toReplace, uchar32_t replaceWith)
	{
		iterator i(*this, 0);
		while (!i.atEnd())
		{
			typename ustring16<TAlloc>::access a = *i;
			if ((uchar32_t)a == toReplace)
				a = replaceWith;
			++i;
		}
		return *this;
	}


	//! Replaces all instances of a string with another one.
	//! \param toReplace The string to replace.
	//! \param replaceWith The string replacing the old one.
	//! \return A reference to our current string.
	ustring16<TAlloc>& replace(const ustring16<TAlloc>& toReplace, const ustring16<TAlloc>& replaceWith)
	{
		if (toReplace.size() == 0)
			return *this;

		const uchar16_t* other = toReplace.c_str();
		const uchar16_t* replace = replaceWith.c_str();
		const u32 other_size = toReplace.size_raw();
		const u32 replace_size = replaceWith.size_raw();

		// Determine the delta.  The algorithm will change depending on the delta.
		s32 delta = replace_size - other_size;

		// A character for character replace.  The string will not shrink or grow.
		if (delta == 0)
		{
			s32 pos = 0;
			while ((pos = find_raw(other, pos)) != -1)
			{
				for (u32 i = 0; i < replace_size; ++i)
					array[pos + i] = replace[i];
				++pos;
			}
			return *this;
		}

		// We are going to be removing some characters.  The string will shrink.
		if (delta < 0)
		{
			u32 i = 0;
			for (u32 pos = 0; pos <= used; ++i, ++pos)
			{
				// Is this potentially a match?
				if (array[pos] == *other)
				{
					// Check to see if we have a match.
					u32 j;
					for (j = 0; j < other_size; ++j)
					{
						if (array[pos + j] != other[j])
							break;
					}

					// If we have a match, replace characters.
					if (j == other_size)
					{
						for (j = 0; j < replace_size; ++j)
							array[i + j] = replace[j];
						i += replace_size - 1;
						pos += other_size - 1;
						continue;
					}
				}

				// No match found, just copy characters.
				array[i - 1] = array[pos];
			}
			array[i] = 0;
			used = i;

			return *this;
		}

		// We are going to be adding characters, so the string size will increase.
		// Count the number of times toReplace exists in the string so we can allocate the new size.
		u32 find_count = 0;
		s32 pos = 0;
		while ((pos = find_raw(other, pos)) != -1)
		{
			++find_count;
			++pos;
		}

		// Re-allocate the string now, if needed.
		u32 len = delta * find_count;
		if (used + len >= allocated)
			reallocate(used + len);

		// Start replacing.
		pos = 0;
		while ((pos = find_raw(other, pos)) != -1)
		{
			uchar16_t* start = array + pos + other_size - 1;
			uchar16_t* ptr   = array + used;
			uchar16_t* end   = array + used + delta;

			// Shift characters to make room for the string.
			while (ptr != start)
			{
				*end = *ptr;
				--ptr;
				--end;
			}

			// Add the new string now.
			for (u32 i = 0; i < replace_size; ++i)
				array[pos + i] = replace[i];

			pos += replace_size;
			used += delta;
		}

		// Terminate the string and return ourself.
		array[used] = 0;
		return *this;
	}


	//! Removes characters from a ustring16..
	//! \param c The character to remove.
	//! \return A reference to our current string.
	ustring16<TAlloc>& remove(uchar32_t c)
	{
		u32 pos = 0;
		u32 found = 0;
		u32 len = (c > 0xFFFF ? 2 : 1);		// Remove characters equal to the size of c as a UTF-16 character.
		for (u32 i=0; i<=used; ++i)
		{
			uchar32_t uc32 = 0;
			if (!UTF16_IS_SURROGATE_HI(array[i]))
				uc32 |= array[i];
			else if (i + 1 <= used)
			{
				// Convert the surrogate pair into a single UTF-32 character.
				uc32 = unicode::toUTF32(array[i], array[i + 1]);
			}
			u32 len2 = (uc32 > 0xFFFF ? 2 : 1);

			if (uc32 == c)
			{
				found += len;
				continue;
			}

			array[pos++] = array[i];
			if (len2 == 2)
				array[pos++] = array[++i];
		}
		used -= found;
		array[used] = 0;
		return *this;
	}


	//! Removes a ustring16 from the ustring16.
	//! \param toRemove The string to remove.
	//! \return A reference to our current string.
	ustring16<TAlloc>& remove(const ustring16<TAlloc>& toRemove)
	{
		u32 size = toRemove.size_raw();
		if (size == 0) return *this;

		const uchar16_t* tra = toRemove.c_str();
		u32 pos = 0;
		u32 found = 0;
		for (u32 i=0; i<=used; ++i)
		{
			u32 j = 0;
			while (j < size)
			{
				if (array[i + j] != tra[j])
					break;
				++j;
			}
			if (j == size)
			{
				found += size;
				i += size - 1;
				continue;
			}

			array[pos++] = array[i];
		}
		used -= found;
		array[used] = 0;
		return *this;
	}


	//! Removes characters from the ustring16.
	//! \param characters The characters to remove.
	//! \return A reference to our current string.
	ustring16<TAlloc>& removeChars(const ustring16<TAlloc>& characters)
	{
		if (characters.size_raw() == 0)
			return *this;

		u32 pos = 0;
		u32 found = 0;
		const_iterator iter(characters);
		for (u32 i=0; i<=used; ++i)
		{
			uchar32_t uc32 = 0;
			if (!UTF16_IS_SURROGATE_HI(array[i]))
				uc32 |= array[i];
			else if (i + 1 <= used)
			{
				// Convert the surrogate pair into a single UTF-32 character.
				uc32 = unicode::toUTF32(array[i], array[i+1]);
			}
			u32 len2 = (uc32 > 0xFFFF ? 2 : 1);

			bool cont = false;
			iter.toStart();
			while (!iter.atEnd())
			{
				uchar32_t c = *iter;
				if (uc32 == c)
				{
					found += (c > 0xFFFF ? 2 : 1);		// Remove characters equal to the size of c as a UTF-16 character.
					++i;
					cont = true;
					break;
				}
				++iter;
			}
			if (cont) continue;

			array[pos++] = array[i];
			if (len2 == 2)
				array[pos++] = array[++i];
		}
		used -= found;
		array[used] = 0;
		return *this;
	}


	//! Trims the ustring16.
	//! Removes the specified characters (by default, Latin-1 whitespace) from the begining and the end of the ustring16.
	//! \param whitespace The characters that are to be considered as whitespace.
	//! \return A reference to our current string.
	ustring16<TAlloc>& trim(const ustring16<TAlloc>& whitespace = " \t\n\r")
	{
		core::array<uchar32_t> utf32white = whitespace.toUTF32();

		// find start and end of the substring without the specified characters
		const s32 begin = findFirstCharNotInList(utf32white.const_pointer(), whitespace.used + 1);
		if (begin == -1)
			return (*this="");

		const s32 end = findLastCharNotInList(utf32white.const_pointer(), whitespace.used + 1);

		return (*this = subString(begin, (end +1) - begin));
	}


	//! Erases a character from the ustring16.
	//! May be slow, because all elements following after the erased element have to be copied.
	//! \param index Index of element to be erased.
	//! \return A reference to our current string.
	ustring16<TAlloc>& erase(u32 index)
	{
		_IRR_DEBUG_BREAK_IF(index>used) // access violation

		iterator i(*this, index);

		uchar32_t t = *i;
		u32 len = (t > 0xFFFF ? 2 : 1);

		for (u32 j = static_cast<u32>(i.getPos()) + len; j <= used; ++j)
			array[j - len] = array[j];

		used -= len;
		array[used] = 0;

		return *this;
	}


	//! Validate the existing ustring16, checking for valid surrogate pairs and checking for proper termination.
	//! \return A reference to our current string.
	ustring16<TAlloc>& validate()
	{
		// Validate all unicode characters.
		for (u32 i=0; i<allocated; ++i)
		{
			// Terminate on existing null.
			if (array[i] == 0)
			{
				used = i;
				return *this;
			}
			if (UTF16_IS_SURROGATE(array[i]))
			{
				if (((i+1) >= allocated) || UTF16_IS_SURROGATE_LO(array[i]))
					array[i] = unicode::UTF_REPLACEMENT_CHARACTER;
				else if (UTF16_IS_SURROGATE_HI(array[i]) && !UTF16_IS_SURROGATE_LO(array[i+1]))
					array[i] = unicode::UTF_REPLACEMENT_CHARACTER;
				++i;
			}
			if (array[i] >= 0xFDD0 && array[i] <= 0xFDEF)
				array[i] = unicode::UTF_REPLACEMENT_CHARACTER;
		}

		// terminate
		used = 0;
		if (allocated > 0)
		{
			used = allocated - 1;
			array[used] = 0;
		}
		return *this;
	}


	//! Gets the last char of the ustring16, or 0.
	//! \return The last char of the ustring16, or 0.
	uchar32_t lastChar() const
	{
		if (used < 1)
			return 0;

		if (UTF16_IS_SURROGATE_LO(array[used-1]))
		{
			// Make sure we have a paired surrogate.
			if (used < 2)
				return 0;

			// Check for an invalid surrogate.
			if (!UTF16_IS_SURROGATE_HI(array[used-2]))
				return 0;

			// Convert the surrogate pair into a single UTF-32 character.
			return unicode::toUTF32(array[used-2], array[used-1]);
		}
		else
		{
			return array[used-1];
		}
	}


	//! Split the ustring16 into parts.
	/** This method will split a ustring16 at certain delimiter characters
	into the container passed in as reference. The type of the container
	has to be given as template parameter. It must provide a push_back and
	a size method.
	\param ret The result container
	\param c C-style ustring16 of delimiter characters
	\param count Number of delimiter characters
	\param ignoreEmptyTokens Flag to avoid empty substrings in the result
	container. If two delimiters occur without a character in between, an
	empty substring would be placed in the result. If this flag is set,
	only non-empty strings are stored.
	\param keepSeparators Flag which allows to add the separator to the
	result ustring16. If this flag is true, the concatenation of the
	substrings results in the original ustring16. Otherwise, only the
	characters between the delimiters are returned.
	\return The number of resulting substrings
	*/
	template<class container>
	u32 split(container& ret, const uchar32_t* const c, u32 count=1, bool ignoreEmptyTokens=true, bool keepSeparators=false) const
	{
		if (!c)
			return 0;

		const_iterator i(*this);
		const u32 oldSize=ret.size();
		u32 pos = 0;
		u32 lastpos = 0;
		u32 lastpospos = 0;
		bool lastWasSeparator = false;
		while (!i.atEnd())
		{
			uchar32_t ch = *i;
			bool foundSeparator = false;
			for (u32 j=0; j<count; ++j)
			{
				if (ch == c[j])
				{
					if ((!ignoreEmptyTokens || pos - lastpos != 0) &&
							!lastWasSeparator)
					ret.push_back(ustring16<TAlloc>(&array[lastpospos], pos - lastpos));
					foundSeparator = true;
					lastpos = (keepSeparators ? pos : pos + 1);
					lastpospos = (keepSeparators ? i.getPos() : i.getPos() + 1);
					break;
				}
			}
			lastWasSeparator = foundSeparator;
			++pos;
			++i;
		}
		u32 s = size() + 1;
		if (s > lastpos)
			ret.push_back(ustring16<TAlloc>(&array[lastpospos], s - lastpos));
		return ret.size()-oldSize;
	}


	//! Split the ustring16 into parts.
	/** This method will split a ustring16 at certain delimiter characters
	into the container passed in as reference. The type of the container
	has to be given as template parameter. It must provide a push_back and
	a size method.
	\param ret The result container
	\param c A unicode string of delimiter characters
	\param ignoreEmptyTokens Flag to avoid empty substrings in the result
	container. If two delimiters occur without a character in between, an
	empty substring would be placed in the result. If this flag is set,
	only non-empty strings are stored.
	\param keepSeparators Flag which allows to add the separator to the
	result ustring16. If this flag is true, the concatenation of the
	substrings results in the original ustring16. Otherwise, only the
	characters between the delimiters are returned.
	\return The number of resulting substrings
	*/
	template<class container>
	u32 split(container& ret, const ustring16<TAlloc>& c, bool ignoreEmptyTokens=true, bool keepSeparators=false) const
	{
		core::array<uchar32_t> v = c.toUTF32();
		return split(ret, v.pointer(), v.size(), ignoreEmptyTokens, keepSeparators);
	}


	//! Gets the size of the allocated memory buffer for the string.
	//! \return The size of the allocated memory buffer.
	u32 capacity() const
	{
		return allocated;
	}


	//! Returns the raw number of UTF-16 code points in the string which includes the individual surrogates.
	//! \return The raw number of UTF-16 code points, excluding the trialing NUL.
	u32 size_raw() const
	{
		return used;
	}


	//! Inserts a character into the string.
	//! \param c The character to insert.
	//! \param pos The position to insert the character.
	//! \return A reference to our current string.
	ustring16<TAlloc>& insert(uchar32_t c, u32 pos)
	{
		u8 len = (c > 0xFFFF ? 2 : 1);

		if (used + len >= allocated)
			reallocate(used + len);

		used += len;

		iterator iter(*this, pos);
		for (u32 i = used - 2; i > iter.getPos(); --i)
			array[i] = array[i - len];

		if (c > 0xFFFF)
		{
			// c will be multibyte, so split it up into a surrogate pair.
			uchar16_t x = static_cast<uchar16_t>(c);
			uchar16_t vh = UTF16_HI_SURROGATE | ((((c >> 16) & ((1 << 5) - 1)) - 1) << 6) | (x >> 10);
			uchar16_t vl = UTF16_LO_SURROGATE | (x & ((1 << 10) - 1));
			array[iter.getPos()] = vh;
			array[iter.getPos()+1] = vl;
		}
		else
		{
			array[iter.getPos()] = static_cast<uchar16_t>(c);
		}
		array[used] = 0;
		return *this;
	}


	//! Inserts a string into the string.
	//! \param c The string to insert.
	//! \param pos The position to insert the string.
	//! \return A reference to our current string.
	ustring16<TAlloc>& insert(const ustring16<TAlloc>& c, u32 pos)
	{
		u32 len = c.size_raw();
		if (len == 0) return *this;

		if (used + len >= allocated)
			reallocate(used + len);

		used += len;

		iterator iter(*this, pos);
		for (u32 i = used - 2; i > iter.getPos() + len; --i)
			array[i] = array[i - len];

		const uchar16_t* s = c.c_str();
		for (u32 i = 0; i < len; ++i)
		{
			array[pos++] = *s;
			++s;
		}

		array[used] = 0;
		return *this;
	}


	//! Inserts a character into the string.
	//! \param c The character to insert.
	//! \param pos The position to insert the character.
	//! \return A reference to our current string.
	ustring16<TAlloc>& insert_raw(uchar16_t c, u32 pos)
	{
		if (used + 1 >= allocated)
			reallocate(used + 1);

		++used;

		for (u32 i = used - 1; i > pos; --i)
			array[i] = array[i - 1];

		array[pos] = c;
		array[used] = 0;
		return *this;
	}


	//! Removes a character from string.
	//! \param pos Position of the character to remove.
	//! \return A reference to our current string.
	ustring16<TAlloc>& erase_raw(u32 pos)
	{
		for (u32 i=pos; i<=used; ++i)
		{
			array[i] = array[i + 1];
		}
		--used;
		array[used] = 0;
		return *this;
	}


	//! Replaces a character in the string.
	//! \param c The new character.
	//! \param pos The position of the character to replace.
	//! \return A reference to our current string.
	ustring16<TAlloc>& replace_raw(uchar16_t c, u32 pos)
	{
		array[pos] = c;
		return *this;
	}


	//! Returns an iterator to the beginning of the string.
	//! \return An iterator to the beginning of the string.
	iterator begin()
	{
		iterator i(*this, 0);
		return i;
	}


	//! Returns an iterator to the beginning of the string.
	//! \return An iterator to the beginning of the string.
	const_iterator begin() const
	{
		const_iterator i(*this, 0);
		return i;
	}


	//! Returns an iterator to the beginning of the string.
	//! \return An iterator to the beginning of the string.
	const_iterator cbegin() const
	{
		const_iterator i(*this, 0);
		return i;
	}


	//! Returns an iterator to the end of the string.
	//! \return An iterator to the end of the string.
	iterator end()
	{
		iterator i(*this, 0);
		i.toEnd();
		return i;
	}


	//! Returns an iterator to the end of the string.
	//! \return An iterator to the end of the string.
	const_iterator end() const
	{
		const_iterator i(*this, 0);
		i.toEnd();
		return i;
	}


	//! Returns an iterator to the end of the string.
	//! \return An iterator to the end of the string.
	const_iterator cend() const
	{
		const_iterator i(*this, 0);
		i.toEnd();
		return i;
	}


	//! Converts the string to a UTF-8 encoded string.
	//! \param addBOM If true, the proper unicode byte-order mark will be prefixed to the string.
	//! \return A string containing the UTF-8 encoded string.
	core::string<uchar8_t> toUTF8_s(const bool addBOM = false) const
	{
		core::string<uchar8_t> ret;
		ret.reserve(used * 4 + (addBOM ? unicode::BOM_UTF8_LEN : 0) + 1);
		const_iterator iter(*this, 0);

		// Add the byte order mark if the user wants it.
		if (addBOM)
		{
			ret.append(unicode::BOM_ENCODE_UTF8[0]);
			ret.append(unicode::BOM_ENCODE_UTF8[1]);
			ret.append(unicode::BOM_ENCODE_UTF8[2]);
		}

		while (!iter.atEnd())
		{
			uchar32_t c = *iter;
			if (c > 0xFFFF)
			{	// 4 bytes
				uchar8_t b1 = (0x1E << 3) | ((c >> 18) & 0x7);
				uchar8_t b2 = (0x2 << 6) | ((c >> 12) & 0x3F);
				uchar8_t b3 = (0x2 << 6) | ((c >> 6) & 0x3F);
				uchar8_t b4 = (0x2 << 6) | (c & 0x3F);
				ret.append(b1);
				ret.append(b2);
				ret.append(b3);
				ret.append(b4);
			}
			else if (c > 0x7FF)
			{	// 3 bytes
				uchar8_t b1 = (0xE << 4) | ((c >> 12) & 0xF);
				uchar8_t b2 = (0x2 << 6) | ((c >> 6) & 0x3F);
				uchar8_t b3 = (0x2 << 6) | (c & 0x3F);
				ret.append(b1);
				ret.append(b2);
				ret.append(b3);
			}
			else if (c > 0x7F)
			{	// 2 bytes
				uchar8_t b1 = (0x6 << 5) | ((c >> 6) & 0x1F);
				uchar8_t b2 = (0x2 << 6) | (c & 0x3F);
				ret.append(b1);
				ret.append(b2);
			}
			else
			{	// 1 byte
				ret.append(static_cast<uchar8_t>(c));
			}
			++iter;
		}
		return ret;
	}


	//! Converts the string to a UTF-8 encoded string array.
	//! \param addBOM If true, the proper unicode byte-order mark will be prefixed to the string.
	//! \return An array containing the UTF-8 encoded string.
	core::array<uchar8_t> toUTF8(const bool addBOM = false) const
	{
		core::array<uchar8_t> ret(used * 4 + (addBOM ? unicode::BOM_UTF8_LEN : 0) + 1);
		const_iterator iter(*this, 0);

		// Add the byte order mark if the user wants it.
		if (addBOM)
		{
			ret.push_back(unicode::BOM_ENCODE_UTF8[0]);
			ret.push_back(unicode::BOM_ENCODE_UTF8[1]);
			ret.push_back(unicode::BOM_ENCODE_UTF8[2]);
		}

		while (!iter.atEnd())
		{
			uchar32_t c = *iter;
			if (c > 0xFFFF)
			{	// 4 bytes
				uchar8_t b1 = (0x1E << 3) | ((c >> 18) & 0x7);
				uchar8_t b2 = (0x2 << 6) | ((c >> 12) & 0x3F);
				uchar8_t b3 = (0x2 << 6) | ((c >> 6) & 0x3F);
				uchar8_t b4 = (0x2 << 6) | (c & 0x3F);
				ret.push_back(b1);
				ret.push_back(b2);
				ret.push_back(b3);
				ret.push_back(b4);
			}
			else if (c > 0x7FF)
			{	// 3 bytes
				uchar8_t b1 = (0xE << 4) | ((c >> 12) & 0xF);
				uchar8_t b2 = (0x2 << 6) | ((c >> 6) & 0x3F);
				uchar8_t b3 = (0x2 << 6) | (c & 0x3F);
				ret.push_back(b1);
				ret.push_back(b2);
				ret.push_back(b3);
			}
			else if (c > 0x7F)
			{	// 2 bytes
				uchar8_t b1 = (0x6 << 5) | ((c >> 6) & 0x1F);
				uchar8_t b2 = (0x2 << 6) | (c & 0x3F);
				ret.push_back(b1);
				ret.push_back(b2);
			}
			else
			{	// 1 byte
				ret.push_back(static_cast<uchar8_t>(c));
			}
			++iter;
		}
		ret.push_back(0);
		return ret;
	}


#ifdef USTRING_CPP0X_NEWLITERALS	// C++0x
	//! Converts the string to a UTF-16 encoded string.
	//! \param endian The desired endianness of the string.
	//! \param addBOM If true, the proper unicode byte-order mark will be prefixed to the string.
	//! \return A string containing the UTF-16 encoded string.
	core::string<char16_t> toUTF16_s(const unicode::EUTF_ENDIAN endian = unicode::EUTFEE_NATIVE, const bool addBOM = false) const
	{
		core::string<char16_t> ret;
		ret.reserve(used + (addBOM ? unicode::BOM_UTF16_LEN : 0) + 1);

		// Add the BOM if specified.
		if (addBOM)
		{
			if (endian == unicode::EUTFEE_NATIVE)
				ret[0] = unicode::BOM;
			else if (endian == unicode::EUTFEE_LITTLE)
			{
				uchar8_t* ptr8 = reinterpret_cast<uchar8_t*>(ret.c_str());
				*ptr8++ = unicode::BOM_ENCODE_UTF16_LE[0];
				*ptr8 = unicode::BOM_ENCODE_UTF16_LE[1];
			}
			else
			{
				uchar8_t* ptr8 = reinterpret_cast<uchar8_t*>(ret.c_str());
				*ptr8++ = unicode::BOM_ENCODE_UTF16_BE[0];
				*ptr8 = unicode::BOM_ENCODE_UTF16_BE[1];
			}
		}

		ret.append(array);
		if (endian != unicode::EUTFEE_NATIVE && getEndianness() != endian)
		{
			char16_t* ptr = ret.c_str();
			for (u32 i = 0; i < ret.size(); ++i)
				*ptr++ = unicode::swapEndian16(*ptr);
		}
		return ret;
	}
#endif


	//! Converts the string to a UTF-16 encoded string array.
	//! Unfortunately, no toUTF16_s() version exists due to limitations with Irrlicht's string class.
	//! \param endian The desired endianness of the string.
	//! \param addBOM If true, the proper unicode byte-order mark will be prefixed to the string.
	//! \return An array containing the UTF-16 encoded string.
	core::array<uchar16_t> toUTF16(const unicode::EUTF_ENDIAN endian = unicode::EUTFEE_NATIVE, const bool addBOM = false) const
	{
		core::array<uchar16_t> ret(used + (addBOM ? unicode::BOM_UTF16_LEN : 0) + 1);
		uchar16_t* ptr = ret.pointer();

		// Add the BOM if specified.
		if (addBOM)
		{
			if (endian == unicode::EUTFEE_NATIVE)
				*ptr = unicode::BOM;
			else if (endian == unicode::EUTFEE_LITTLE)
			{
				uchar8_t* ptr8 = reinterpret_cast<uchar8_t*>(ptr);
				*ptr8++ = unicode::BOM_ENCODE_UTF16_LE[0];
				*ptr8 = unicode::BOM_ENCODE_UTF16_LE[1];
			}
			else
			{
				uchar8_t* ptr8 = reinterpret_cast<uchar8_t*>(ptr);
				*ptr8++ = unicode::BOM_ENCODE_UTF16_BE[0];
				*ptr8 = unicode::BOM_ENCODE_UTF16_BE[1];
			}
			++ptr;
		}

		memcpy((void*)ptr, (void*)array, used * sizeof(uchar16_t));
		if (endian != unicode::EUTFEE_NATIVE && getEndianness() != endian)
		{
			for (u32 i = 0; i <= used; ++i)
				ptr[i] = unicode::swapEndian16(ptr[i]);
		}
		ret.set_used(used + (addBOM ? unicode::BOM_UTF16_LEN : 0));
		ret.push_back(0);
		return ret;
	}


#ifdef USTRING_CPP0X_NEWLITERALS	// C++0x
	//! Converts the string to a UTF-32 encoded string.
	//! \param endian The desired endianness of the string.
	//! \param addBOM If true, the proper unicode byte-order mark will be prefixed to the string.
	//! \return A string containing the UTF-32 encoded string.
	core::string<char32_t> toUTF32_s(const unicode::EUTF_ENDIAN endian = unicode::EUTFEE_NATIVE, const bool addBOM = false) const
	{
		core::string<char32_t> ret;
		ret.reserve(size() + 1 + (addBOM ? unicode::BOM_UTF32_LEN : 0));
		const_iterator iter(*this, 0);

		// Add the BOM if specified.
		if (addBOM)
		{
			if (endian == unicode::EUTFEE_NATIVE)
				ret.append(unicode::BOM);
			else
			{
				union
				{
					uchar32_t full;
					u8 chunk[4];
				} t;

				if (endian == unicode::EUTFEE_LITTLE)
				{
					t.chunk[0] = unicode::BOM_ENCODE_UTF32_LE[0];
					t.chunk[1] = unicode::BOM_ENCODE_UTF32_LE[1];
					t.chunk[2] = unicode::BOM_ENCODE_UTF32_LE[2];
					t.chunk[3] = unicode::BOM_ENCODE_UTF32_LE[3];
				}
				else
				{
					t.chunk[0] = unicode::BOM_ENCODE_UTF32_BE[0];
					t.chunk[1] = unicode::BOM_ENCODE_UTF32_BE[1];
					t.chunk[2] = unicode::BOM_ENCODE_UTF32_BE[2];
					t.chunk[3] = unicode::BOM_ENCODE_UTF32_BE[3];
				}
				ret.append(t.full);
			}
		}

		while (!iter.atEnd())
		{
			uchar32_t c = *iter;
			if (endian != unicode::EUTFEE_NATIVE && getEndianness() != endian)
				c = unicode::swapEndian32(c);
			ret.append(c);
			++iter;
		}
		return ret;
	}
#endif


	//! Converts the string to a UTF-32 encoded string array.
	//! Unfortunately, no toUTF32_s() version exists due to limitations with Irrlicht's string class.
	//! \param endian The desired endianness of the string.
	//! \param addBOM If true, the proper unicode byte-order mark will be prefixed to the string.
	//! \return An array containing the UTF-32 encoded string.
	core::array<uchar32_t> toUTF32(const unicode::EUTF_ENDIAN endian = unicode::EUTFEE_NATIVE, const bool addBOM = false) const
	{
		core::array<uchar32_t> ret(size() + (addBOM ? unicode::BOM_UTF32_LEN : 0) + 1);
		const_iterator iter(*this, 0);

		// Add the BOM if specified.
		if (addBOM)
		{
			if (endian == unicode::EUTFEE_NATIVE)
				ret.push_back(unicode::BOM);
			else
			{
				union
				{
					uchar32_t full;
					u8 chunk[4];
				} t;

				if (endian == unicode::EUTFEE_LITTLE)
				{
					t.chunk[0] = unicode::BOM_ENCODE_UTF32_LE[0];
					t.chunk[1] = unicode::BOM_ENCODE_UTF32_LE[1];
					t.chunk[2] = unicode::BOM_ENCODE_UTF32_LE[2];
					t.chunk[3] = unicode::BOM_ENCODE_UTF32_LE[3];
				}
				else
				{
					t.chunk[0] = unicode::BOM_ENCODE_UTF32_BE[0];
					t.chunk[1] = unicode::BOM_ENCODE_UTF32_BE[1];
					t.chunk[2] = unicode::BOM_ENCODE_UTF32_BE[2];
					t.chunk[3] = unicode::BOM_ENCODE_UTF32_BE[3];
				}
				ret.push_back(t.full);
			}
		}
		ret.push_back(0);

		while (!iter.atEnd())
		{
			uchar32_t c = *iter;
			if (endian != unicode::EUTFEE_NATIVE && getEndianness() != endian)
				c = unicode::swapEndian32(c);
			ret.push_back(c);
			++iter;
		}
		return ret;
	}


	//! Converts the string to a wchar_t encoded string.
	/** The size of a wchar_t changes depending on the platform.  This function will store a
	correct UTF-8, -16, or -32 encoded string depending on the size of a wchar_t. **/
	//! \param endian The desired endianness of the string.
	//! \param addBOM If true, the proper unicode byte-order mark will be prefixed to the string.
	//! \return A string containing the wchar_t encoded string.
	core::string<wchar_t> toWCHAR_s(const unicode::EUTF_ENDIAN endian = unicode::EUTFEE_NATIVE, const bool addBOM = false) const
	{
		if (sizeof(wchar_t) == 4)
		{
			core::array<uchar32_t> a(toUTF32(endian, addBOM));
			core::stringw ret(a.pointer());
			return ret;
		}
		else if (sizeof(wchar_t) == 2)
		{
			if (endian == unicode::EUTFEE_NATIVE && addBOM == false)
			{
				core::stringw ret(array);
				return ret;
			}
			else
			{
				core::array<uchar16_t> a(toUTF16(endian, addBOM));
				core::stringw ret(a.pointer());
				return ret;
			}
		}
		else if (sizeof(wchar_t) == 1)
		{
			core::array<uchar8_t> a(toUTF8(addBOM));
			core::stringw ret(a.pointer());
			return ret;
		}

		// Shouldn't happen.
		return core::stringw();
	}


	//! Converts the string to a wchar_t encoded string array.
	/** The size of a wchar_t changes depending on the platform.  This function will store a
	correct UTF-8, -16, or -32 encoded string depending on the size of a wchar_t. **/
	//! \param endian The desired endianness of the string.
	//! \param addBOM If true, the proper unicode byte-order mark will be prefixed to the string.
	//! \return An array containing the wchar_t encoded string.
	core::array<wchar_t> toWCHAR(const unicode::EUTF_ENDIAN endian = unicode::EUTFEE_NATIVE, const bool addBOM = false) const
	{
		if (sizeof(wchar_t) == 4)
		{
			core::array<uchar32_t> a(toUTF32(endian, addBOM));
			core::array<wchar_t> ret(a.size());
			ret.set_used(a.size());
			memcpy((void*)ret.pointer(), (void*)a.pointer(), a.size() * sizeof(uchar32_t));
			return ret;
		}
		if (sizeof(wchar_t) == 2)
		{
			if (endian == unicode::EUTFEE_NATIVE && addBOM == false)
			{
				core::array<wchar_t> ret(used);
				ret.set_used(used);
				memcpy((void*)ret.pointer(), (void*)array, used * sizeof(uchar16_t));
				return ret;
			}
			else
			{
				core::array<uchar16_t> a(toUTF16(endian, addBOM));
				core::array<wchar_t> ret(a.size());
				ret.set_used(a.size());
				memcpy((void*)ret.pointer(), (void*)a.pointer(), a.size() * sizeof(uchar16_t));
				return ret;
			}
		}
		if (sizeof(wchar_t) == 1)
		{
			core::array<uchar8_t> a(toUTF8(addBOM));
			core::array<wchar_t> ret(a.size());
			ret.set_used(a.size());
			memcpy((void*)ret.pointer(), (void*)a.pointer(), a.size() * sizeof(uchar8_t));
			return ret;
		}

		// Shouldn't happen.
		return core::array<wchar_t>();
	}

	//! Converts the string to a properly encoded io::path string.
	//! \param endian The desired endianness of the string.
	//! \param addBOM If true, the proper unicode byte-order mark will be prefixed to the string.
	//! \return An io::path string containing the properly encoded string.
	io::path toPATH_s(const unicode::EUTF_ENDIAN endian = unicode::EUTFEE_NATIVE, const bool addBOM = false) const
	{
#if defined(_IRR_WCHAR_FILESYSTEM)
		return toWCHAR_s(endian, addBOM);
#else
		return toUTF8_s(addBOM);
#endif
	}

	//! Loads an unknown stream of data.
	//! Will attempt to determine if the stream is unicode data.  Useful for loading from files.
	//! \param data The data stream to load from.
	//! \param data_size The length of the data string.
	//! \return A reference to our current string.
	ustring16<TAlloc>& loadDataStream(const char* data, size_t data_size)
	{
		// Clear our string.
		*this = "";
		if (!data)
			return *this;

		unicode::EUTF_ENCODE e = unicode::determineUnicodeBOM(data);
		switch (e)
		{
			default:
			case unicode::EUTFE_UTF8:
				append((uchar8_t*)data, data_size);
				break;

			case unicode::EUTFE_UTF16:
			case unicode::EUTFE_UTF16_BE:
			case unicode::EUTFE_UTF16_LE:
				append((uchar16_t*)data, data_size / 2);
				break;

			case unicode::EUTFE_UTF32:
			case unicode::EUTFE_UTF32_BE:
			case unicode::EUTFE_UTF32_LE:
				append((uchar32_t*)data, data_size / 4);
				break;
		}

		return *this;
	}

	//! Gets the encoding of the Unicode string this class contains.
	//! \return An enum describing the current encoding of this string.
	const unicode::EUTF_ENCODE getEncoding() const
	{
		return encoding;
	}

	//! Gets the endianness of the Unicode string this class contains.
	//! \return An enum describing the endianness of this string.
	const unicode::EUTF_ENDIAN getEndianness() const
	{
		if (encoding == unicode::EUTFE_UTF16_LE ||
			encoding == unicode::EUTFE_UTF32_LE)
			return unicode::EUTFEE_LITTLE;
		else return unicode::EUTFEE_BIG;
	}

private:

	//! Reallocate the string, making it bigger or smaller.
	//! \param new_size The new size of the string.
	void reallocate(u32 new_size)
	{
		uchar16_t* old_array = array;

		array = allocator.allocate(new_size + 1); //new u16[new_size];
		allocated = new_size + 1;
		if (old_array == 0) return;

		u32 amount = used < new_size ? used : new_size;
		for (u32 i=0; i<=amount; ++i)
			array[i] = old_array[i];

		if (allocated <= used)
			used = allocated - 1;

		array[used] = 0;

		allocator.deallocate(old_array); // delete [] old_array;
	}

	//--- member variables

	uchar16_t* array;
	unicode::EUTF_ENCODE encoding;
	u32 allocated;
	u32 used;
	TAlloc allocator;
	//irrAllocator<uchar16_t> allocator;
};

typedef ustring16<irrAllocator<uchar16_t> > ustring;


//! Appends two ustring16s.
template <typename TAlloc>
inline ustring16<TAlloc> operator+(const ustring16<TAlloc>& left, const ustring16<TAlloc>& right)
{
	ustring16<TAlloc> ret(left);
	ret += right;
	return ret;
}


//! Appends a ustring16 and a null-terminated unicode string.
template <typename TAlloc, class B>
inline ustring16<TAlloc> operator+(const ustring16<TAlloc>& left, const B* const right)
{
	ustring16<TAlloc> ret(left);
	ret += right;
	return ret;
}


//! Appends a ustring16 and a null-terminated unicode string.
template <class B, typename TAlloc>
inline ustring16<TAlloc> operator+(const B* const left, const ustring16<TAlloc>& right)
{
	ustring16<TAlloc> ret(left);
	ret += right;
	return ret;
}


//! Appends a ustring16 and an Irrlicht string.
template <typename TAlloc, typename B, typename BAlloc>
inline ustring16<TAlloc> operator+(const ustring16<TAlloc>& left, const string<B, BAlloc>& right)
{
	ustring16<TAlloc> ret(left);
	ret += right;
	return ret;
}


//! Appends a ustring16 and an Irrlicht string.
template <typename TAlloc, typename B, typename BAlloc>
inline ustring16<TAlloc> operator+(const string<B, BAlloc>& left, const ustring16<TAlloc>& right)
{
	ustring16<TAlloc> ret(left);
	ret += right;
	return ret;
}


//! Appends a ustring16 and a std::basic_string.
template <typename TAlloc, typename B, typename A, typename BAlloc>
inline ustring16<TAlloc> operator+(const ustring16<TAlloc>& left, const std::basic_string<B, A, BAlloc>& right)
{
	ustring16<TAlloc> ret(left);
	ret += right;
	return ret;
}


//! Appends a ustring16 and a std::basic_string.
template <typename TAlloc, typename B, typename A, typename BAlloc>
inline ustring16<TAlloc> operator+(const std::basic_string<B, A, BAlloc>& left, const ustring16<TAlloc>& right)
{
	ustring16<TAlloc> ret(left);
	ret += right;
	return ret;
}


//! Appends a ustring16 and a char.
template <typename TAlloc>
inline ustring16<TAlloc> operator+(const ustring16<TAlloc>& left, const char right)
{
	ustring16<TAlloc> ret(left);
	ret += right;
	return ret;
}


//! Appends a ustring16 and a char.
template <typename TAlloc>
inline ustring16<TAlloc> operator+(const char left, const ustring16<TAlloc>& right)
{
	ustring16<TAlloc> ret(left);
	ret += right;
	return ret;
}


#ifdef USTRING_CPP0X_NEWLITERALS
//! Appends a ustring16 and a uchar32_t.
template <typename TAlloc>
inline ustring16<TAlloc> operator+(const ustring16<TAlloc>& left, const uchar32_t right)
{
	ustring16<TAlloc> ret(left);
	ret += right;
	return ret;
}


//! Appends a ustring16 and a uchar32_t.
template <typename TAlloc>
inline ustring16<TAlloc> operator+(const uchar32_t left, const ustring16<TAlloc>& right)
{
	ustring16<TAlloc> ret(left);
	ret += right;
	return ret;
}
#endif


//! Appends a ustring16 and a short.
template <typename TAlloc>
inline ustring16<TAlloc> operator+(const ustring16<TAlloc>& left, const short right)
{
	ustring16<TAlloc> ret(left);
	ret += core::stringc(right);
	return ret;
}


//! Appends a ustring16 and a short.
template <typename TAlloc>
inline ustring16<TAlloc> operator+(const short left, const ustring16<TAlloc>& right)
{
	ustring16<TAlloc> ret((core::stringc(left)));
	ret += right;
	return ret;
}


//! Appends a ustring16 and an unsigned short.
template <typename TAlloc>
inline ustring16<TAlloc> operator+(const ustring16<TAlloc>& left, const unsigned short right)
{
	ustring16<TAlloc> ret(left);
	ret += core::stringc(right);
	return ret;
}


//! Appends a ustring16 and an unsigned short.
template <typename TAlloc>
inline ustring16<TAlloc> operator+(const unsigned short left, const ustring16<TAlloc>& right)
{
	ustring16<TAlloc> ret((core::stringc(left)));
	ret += right;
	return ret;
}


//! Appends a ustring16 and an int.
template <typename TAlloc>
inline ustring16<TAlloc> operator+(const ustring16<TAlloc>& left, const int right)
{
	ustring16<TAlloc> ret(left);
	ret += core::stringc(right);
	return ret;
}


//! Appends a ustring16 and an int.
template <typename TAlloc>
inline ustring16<TAlloc> operator+(const int left, const ustring16<TAlloc>& right)
{
	ustring16<TAlloc> ret((core::stringc(left)));
	ret += right;
	return ret;
}


//! Appends a ustring16 and an unsigned int.
template <typename TAlloc>
inline ustring16<TAlloc> operator+(const ustring16<TAlloc>& left, const unsigned int right)
{
	ustring16<TAlloc> ret(left);
	ret += core::stringc(right);
	return ret;
}


//! Appends a ustring16 and an unsigned int.
template <typename TAlloc>
inline ustring16<TAlloc> operator+(const unsigned int left, const ustring16<TAlloc>& right)
{
	ustring16<TAlloc> ret((core::stringc(left)));
	ret += right;
	return ret;
}


//! Appends a ustring16 and a long.
template <typename TAlloc>
inline ustring16<TAlloc> operator+(const ustring16<TAlloc>& left, const long right)
{
	ustring16<TAlloc> ret(left);
	ret += core::stringc(right);
	return ret;
}


//! Appends a ustring16 and a long.
template <typename TAlloc>
inline ustring16<TAlloc> operator+(const long left, const ustring16<TAlloc>& right)
{
	ustring16<TAlloc> ret((core::stringc(left)));
	ret += right;
	return ret;
}


//! Appends a ustring16 and an unsigned long.
template <typename TAlloc>
inline ustring16<TAlloc> operator+(const ustring16<TAlloc>& left, const unsigned long right)
{
	ustring16<TAlloc> ret(left);
	ret += core::stringc(right);
	return ret;
}


//! Appends a ustring16 and an unsigned long.
template <typename TAlloc>
inline ustring16<TAlloc> operator+(const unsigned long left, const ustring16<TAlloc>& right)
{
	ustring16<TAlloc> ret((core::stringc(left)));
	ret += right;
	return ret;
}


//! Appends a ustring16 and a float.
template <typename TAlloc>
inline ustring16<TAlloc> operator+(const ustring16<TAlloc>& left, const float right)
{
	ustring16<TAlloc> ret(left);
	ret += core::stringc(right);
	return ret;
}


//! Appends a ustring16 and a float.
template <typename TAlloc>
inline ustring16<TAlloc> operator+(const float left, const ustring16<TAlloc>& right)
{
	ustring16<TAlloc> ret((core::stringc(left)));
	ret += right;
	return ret;
}


//! Appends a ustring16 and a double.
template <typename TAlloc>
inline ustring16<TAlloc> operator+(const ustring16<TAlloc>& left, const double right)
{
	ustring16<TAlloc> ret(left);
	ret += core::stringc(right);
	return ret;
}


//! Appends a ustring16 and a double.
template <typename TAlloc>
inline ustring16<TAlloc> operator+(const double left, const ustring16<TAlloc>& right)
{
	ustring16<TAlloc> ret((core::stringc(left)));
	ret += right;
	return ret;
}


#ifdef USTRING_CPP0X
//! Appends two ustring16s.
template <typename TAlloc>
inline ustring16<TAlloc>&& operator+(const ustring16<TAlloc>& left, ustring16<TAlloc>&& right)
{
	//std::cout << "MOVE operator+(&, &&)" << std::endl;
	right.insert(left, 0);
	return std::move(right);
}


//! Appends two ustring16s.
template <typename TAlloc>
inline ustring16<TAlloc>&& operator+(ustring16<TAlloc>&& left, const ustring16<TAlloc>& right)
{
	//std::cout << "MOVE operator+(&&, &)" << std::endl;
	left.append(right);
	return std::move(left);
}


//! Appends two ustring16s.
template <typename TAlloc>
inline ustring16<TAlloc>&& operator+(ustring16<TAlloc>&& left, ustring16<TAlloc>&& right)
{
	//std::cout << "MOVE operator+(&&, &&)" << std::endl;
	if ((right.size_raw() <= left.capacity() - left.size_raw()) ||
		(right.capacity() - right.size_raw() < left.size_raw()))
	{
		left.append(right);
		return std::move(left);
	}
	else
	{
		right.insert(left, 0);
		return std::move(right);
	}
}


//! Appends a ustring16 and a null-terminated unicode string.
template <typename TAlloc, class B>
inline ustring16<TAlloc>&& operator+(ustring16<TAlloc>&& left, const B* const right)
{
	//std::cout << "MOVE operator+(&&, B*)" << std::endl;
	left.append(right);
	return std::move(left);
}


//! Appends a ustring16 and a null-terminated unicode string.
template <class B, typename TAlloc>
inline ustring16<TAlloc>&& operator+(const B* const left, ustring16<TAlloc>&& right)
{
	//std::cout << "MOVE operator+(B*, &&)" << std::endl;
	right.insert(left, 0);
	return std::move(right);
}


//! Appends a ustring16 and an Irrlicht string.
template <typename TAlloc, typename B, typename BAlloc>
inline ustring16<TAlloc>&& operator+(const string<B, BAlloc>& left, ustring16<TAlloc>&& right)
{
	//std::cout << "MOVE operator+(&, &&)" << std::endl;
	right.insert(left, 0);
	return std::move(right);
}


//! Appends a ustring16 and an Irrlicht string.
template <typename TAlloc, typename B, typename BAlloc>
inline ustring16<TAlloc>&& operator+(ustring16<TAlloc>&& left, const string<B, BAlloc>& right)
{
	//std::cout << "MOVE operator+(&&, &)" << std::endl;
	left.append(right);
	return std::move(left);
}


//! Appends a ustring16 and a std::basic_string.
template <typename TAlloc, typename B, typename A, typename BAlloc>
inline ustring16<TAlloc>&& operator+(const std::basic_string<B, A, BAlloc>& left, ustring16<TAlloc>&& right)
{
	//std::cout << "MOVE operator+(&, &&)" << std::endl;
	right.insert(core::ustring16<TAlloc>(left), 0);
	return std::move(right);
}


//! Appends a ustring16 and a std::basic_string.
template <typename TAlloc, typename B, typename A, typename BAlloc>
inline ustring16<TAlloc>&& operator+(ustring16<TAlloc>&& left, const std::basic_string<B, A, BAlloc>& right)
{
	//std::cout << "MOVE operator+(&&, &)" << std::endl;
	left.append(right);
	return std::move(left);
}


//! Appends a ustring16 and a char.
template <typename TAlloc>
inline ustring16<TAlloc> operator+(ustring16<TAlloc>&& left, const char right)
{
	left.append((uchar32_t)right);
	return std::move(left);
}


//! Appends a ustring16 and a char.
template <typename TAlloc>
inline ustring16<TAlloc> operator+(const char left, ustring16<TAlloc>&& right)
{
	right.insert((uchar32_t)left, 0);
	return std::move(right);
}


#ifdef USTRING_CPP0X_NEWLITERALS
//! Appends a ustring16 and a uchar32_t.
template <typename TAlloc>
inline ustring16<TAlloc> operator+(ustring16<TAlloc>&& left, const uchar32_t right)
{
	left.append(right);
	return std::move(left);
}


//! Appends a ustring16 and a uchar32_t.
template <typename TAlloc>
inline ustring16<TAlloc> operator+(const uchar32_t left, ustring16<TAlloc>&& right)
{
	right.insert(left, 0);
	return std::move(right);
}
#endif


//! Appends a ustring16 and a short.
template <typename TAlloc>
inline ustring16<TAlloc> operator+(ustring16<TAlloc>&& left, const short right)
{
	left.append(core::stringc(right));
	return std::move(left);
}


//! Appends a ustring16 and a short.
template <typename TAlloc>
inline ustring16<TAlloc> operator+(const short left, ustring16<TAlloc>&& right)
{
	right.insert(core::stringc(left), 0);
	return std::move(right);
}


//! Appends a ustring16 and an unsigned short.
template <typename TAlloc>
inline ustring16<TAlloc> operator+(ustring16<TAlloc>&& left, const unsigned short right)
{
	left.append(core::stringc(right));
	return std::move(left);
}


//! Appends a ustring16 and an unsigned short.
template <typename TAlloc>
inline ustring16<TAlloc> operator+(const unsigned short left, ustring16<TAlloc>&& right)
{
	right.insert(core::stringc(left), 0);
	return std::move(right);
}


//! Appends a ustring16 and an int.
template <typename TAlloc>
inline ustring16<TAlloc> operator+(ustring16<TAlloc>&& left, const int right)
{
	left.append(core::stringc(right));
	return std::move(left);
}


//! Appends a ustring16 and an int.
template <typename TAlloc>
inline ustring16<TAlloc> operator+(const int left, ustring16<TAlloc>&& right)
{
	right.insert(core::stringc(left), 0);
	return std::move(right);
}


//! Appends a ustring16 and an unsigned int.
template <typename TAlloc>
inline ustring16<TAlloc> operator+(ustring16<TAlloc>&& left, const unsigned int right)
{
	left.append(core::stringc(right));
	return std::move(left);
}


//! Appends a ustring16 and an unsigned int.
template <typename TAlloc>
inline ustring16<TAlloc> operator+(const unsigned int left, ustring16<TAlloc>&& right)
{
	right.insert(core::stringc(left), 0);
	return std::move(right);
}


//! Appends a ustring16 and a long.
template <typename TAlloc>
inline ustring16<TAlloc> operator+(ustring16<TAlloc>&& left, const long right)
{
	left.append(core::stringc(right));
	return std::move(left);
}


//! Appends a ustring16 and a long.
template <typename TAlloc>
inline ustring16<TAlloc> operator+(const long left, ustring16<TAlloc>&& right)
{
	right.insert(core::stringc(left), 0);
	return std::move(right);
}


//! Appends a ustring16 and an unsigned long.
template <typename TAlloc>
inline ustring16<TAlloc> operator+(ustring16<TAlloc>&& left, const unsigned long right)
{
	left.append(core::stringc(right));
	return std::move(left);
}


//! Appends a ustring16 and an unsigned long.
template <typename TAlloc>
inline ustring16<TAlloc> operator+(const unsigned long left, ustring16<TAlloc>&& right)
{
	right.insert(core::stringc(left), 0);
	return std::move(right);
}


//! Appends a ustring16 and a float.
template <typename TAlloc>
inline ustring16<TAlloc> operator+(ustring16<TAlloc>&& left, const float right)
{
	left.append(core::stringc(right));
	return std::move(left);
}


//! Appends a ustring16 and a float.
template <typename TAlloc>
inline ustring16<TAlloc> operator+(const float left, ustring16<TAlloc>&& right)
{
	right.insert(core::stringc(left), 0);
	return std::move(right);
}


//! Appends a ustring16 and a double.
template <typename TAlloc>
inline ustring16<TAlloc> operator+(ustring16<TAlloc>&& left, const double right)
{
	left.append(core::stringc(right));
	return std::move(left);
}


//! Appends a ustring16 and a double.
template <typename TAlloc>
inline ustring16<TAlloc> operator+(const double left, ustring16<TAlloc>&& right)
{
	right.insert(core::stringc(left), 0);
	return std::move(right);
}
#endif


#ifndef USTRING_NO_STL
//! Writes a ustring16 to an ostream.
template <typename TAlloc>
inline std::ostream& operator<<(std::ostream& out, const ustring16<TAlloc>& in)
{
	out << in.toUTF8_s().c_str();
	return out;
}

//! Writes a ustring16 to a wostream.
template <typename TAlloc>
inline std::wostream& operator<<(std::wostream& out, const ustring16<TAlloc>& in)
{
	out << in.toWCHAR_s().c_str();
	return out;
}
#endif


#ifndef USTRING_NO_STL

namespace unicode
{

//! Hashing algorithm for hashing a ustring.  Used for things like unordered_maps.
//! Algorithm taken from std::hash<std::string>.
class hash : public std::unary_function<core::ustring, size_t>
{
	public:
		size_t operator()(const core::ustring& s) const
		{
			size_t ret = 2166136261U;
			size_t index = 0;
			size_t stride = 1 + s.size_raw() / 10;

			core::ustring::const_iterator i = s.begin();
			while (i != s.end())
			{
				// TODO: Don't force u32 on an x64 OS.  Make it agnostic.
				ret = 16777619U * ret ^ (size_t)s[(u32)index];
				index += stride;
				i += stride;
			}
			return (ret);
		}
};

} // end namespace unicode

#endif

} // end namespace core
} // end namespace irr

#endif