unicode.inc

  1. 7.x drupal-7.x/includes/unicode.inc
  2. 6.x drupal-6.x/includes/unicode.inc

File

drupal-6.x/includes/unicode.inc
View source
  1. <?php
  2. /**
  3. * Indicates an error during check for PHP unicode support.
  4. */
  5. define('UNICODE_ERROR', -1);
  6. /**
  7. * Indicates that standard PHP (emulated) unicode support is being used.
  8. */
  9. define('UNICODE_SINGLEBYTE', 0);
  10. /**
  11. * Indicates that full unicode support with the PHP mbstring extension is being
  12. * used.
  13. */
  14. define('UNICODE_MULTIBYTE', 1);
  15. /**
  16. * Wrapper around _unicode_check().
  17. */
  18. function unicode_check() {
  19. list($GLOBALS['multibyte']) = _unicode_check();
  20. }
  21. /**
  22. * Perform checks about Unicode support in PHP, and set the right settings if
  23. * needed.
  24. *
  25. * Because Drupal needs to be able to handle text in various encodings, we do
  26. * not support mbstring function overloading. HTTP input/output conversion must
  27. * be disabled for similar reasons.
  28. *
  29. * @param $errors
  30. * Whether to report any fatal errors with form_set_error().
  31. */
  32. function _unicode_check() {
  33. // Ensure translations don't break at install time
  34. $t = get_t();
  35. // Set the standard C locale to ensure consistent, ASCII-only string handling.
  36. setlocale(LC_CTYPE, 'C');
  37. // Check for outdated PCRE library
  38. // Note: we check if U+E2 is in the range U+E0 - U+E1. This test returns TRUE on old PCRE versions.
  39. if (preg_match('/[à-á]/u', 'â')) {
  40. return array(UNICODE_ERROR, $t('The PCRE library in your PHP installation is outdated. This will cause problems when handling Unicode text. If you are running PHP 4.3.3 or higher, make sure you are using the PCRE library supplied by PHP. Please refer to the <a href="@url">PHP PCRE documentation</a> for more information.', array('@url' => 'http://www.php.net/pcre')));
  41. }
  42. // Check for mbstring extension
  43. if (!function_exists('mb_strlen')) {
  44. return array(UNICODE_SINGLEBYTE, $t('Operations on Unicode strings are emulated on a best-effort basis. Install the <a href="@url">PHP mbstring extension</a> for improved Unicode support.', array('@url' => 'http://www.php.net/mbstring')));
  45. }
  46. // Check mbstring configuration
  47. if (ini_get('mbstring.func_overload') != 0) {
  48. return array(UNICODE_ERROR, $t('Multibyte string function overloading in PHP is active and must be disabled. Check the php.ini <em>mbstring.func_overload</em> setting. Please refer to the <a href="@url">PHP mbstring documentation</a> for more information.', array('@url' => 'http://www.php.net/mbstring')));
  49. }
  50. if (ini_get('mbstring.encoding_translation') != 0) {
  51. return array(UNICODE_ERROR, $t('Multibyte string input conversion in PHP is active and must be disabled. Check the php.ini <em>mbstring.encoding_translation</em> setting. Please refer to the <a href="@url">PHP mbstring documentation</a> for more information.', array('@url' => 'http://www.php.net/mbstring')));
  52. }
  53. if (ini_get('mbstring.http_input') != 'pass') {
  54. return array(UNICODE_ERROR, $t('Multibyte string input conversion in PHP is active and must be disabled. Check the php.ini <em>mbstring.http_input</em> setting. Please refer to the <a href="@url">PHP mbstring documentation</a> for more information.', array('@url' => 'http://www.php.net/mbstring')));
  55. }
  56. if (ini_get('mbstring.http_output') != 'pass') {
  57. return array(UNICODE_ERROR, $t('Multibyte string output conversion in PHP is active and must be disabled. Check the php.ini <em>mbstring.http_output</em> setting. Please refer to the <a href="@url">PHP mbstring documentation</a> for more information.', array('@url' => 'http://www.php.net/mbstring')));
  58. }
  59. // Set appropriate configuration
  60. mb_internal_encoding('utf-8');
  61. mb_language('uni');
  62. return array(UNICODE_MULTIBYTE, '');
  63. }
  64. /**
  65. * Return Unicode library status and errors.
  66. */
  67. function unicode_requirements() {
  68. // Ensure translations don't break at install time
  69. $t = get_t();
  70. $libraries = array(
  71. UNICODE_SINGLEBYTE => $t('Standard PHP'),
  72. UNICODE_MULTIBYTE => $t('PHP Mbstring Extension'),
  73. UNICODE_ERROR => $t('Error'),
  74. );
  75. $severities = array(
  76. UNICODE_SINGLEBYTE => REQUIREMENT_WARNING,
  77. UNICODE_MULTIBYTE => REQUIREMENT_OK,
  78. UNICODE_ERROR => REQUIREMENT_ERROR,
  79. );
  80. list($library, $description) = _unicode_check();
  81. $requirements['unicode'] = array(
  82. 'title' => $t('Unicode library'),
  83. 'value' => $libraries[$library],
  84. );
  85. if ($description) {
  86. $requirements['unicode']['description'] = $description;
  87. }
  88. $requirements['unicode']['severity'] = $severities[$library];
  89. return $requirements;
  90. }
  91. /**
  92. * Prepare a new XML parser.
  93. *
  94. * This is a wrapper around xml_parser_create() which extracts the encoding from
  95. * the XML data first and sets the output encoding to UTF-8. This function should
  96. * be used instead of xml_parser_create(), because PHP 4's XML parser doesn't
  97. * check the input encoding itself. "Starting from PHP 5, the input encoding is
  98. * automatically detected, so that the encoding parameter specifies only the
  99. * output encoding."
  100. *
  101. * This is also where unsupported encodings will be converted. Callers should
  102. * take this into account: $data might have been changed after the call.
  103. *
  104. * @param &$data
  105. * The XML data which will be parsed later.
  106. * @return
  107. * An XML parser object.
  108. */
  109. function drupal_xml_parser_create(&$data) {
  110. // Default XML encoding is UTF-8
  111. $encoding = 'utf-8';
  112. $bom = FALSE;
  113. // Check for UTF-8 byte order mark (PHP5's XML parser doesn't handle it).
  114. if (!strncmp($data, "\xEF\xBB\xBF", 3)) {
  115. $bom = TRUE;
  116. $data = substr($data, 3);
  117. }
  118. // Check for an encoding declaration in the XML prolog if no BOM was found.
  119. if (!$bom && @ereg('^<\?xml[^>]+encoding="([^"]+)"', $data, $match)) {
  120. $encoding = $match[1];
  121. }
  122. // Unsupported encodings are converted here into UTF-8.
  123. $php_supported = array('utf-8', 'iso-8859-1', 'us-ascii');
  124. if (!in_array(strtolower($encoding), $php_supported)) {
  125. $out = drupal_convert_to_utf8($data, $encoding);
  126. if ($out !== FALSE) {
  127. $encoding = 'utf-8';
  128. $data = @ereg_replace('^(<\?xml[^>]+encoding)="([^"]+)"', '\\1="utf-8"', $out);
  129. }
  130. else {
  131. watchdog('php', 'Could not convert XML encoding %s to UTF-8.', array('%s' => $encoding), WATCHDOG_WARNING);
  132. return 0;
  133. }
  134. }
  135. $xml_parser = xml_parser_create($encoding);
  136. xml_parser_set_option($xml_parser, XML_OPTION_TARGET_ENCODING, 'utf-8');
  137. return $xml_parser;
  138. }
  139. /**
  140. * Convert data to UTF-8
  141. *
  142. * Requires the iconv, GNU recode or mbstring PHP extension.
  143. *
  144. * @param $data
  145. * The data to be converted.
  146. * @param $encoding
  147. * The encoding that the data is in
  148. * @return
  149. * Converted data or FALSE.
  150. */
  151. function drupal_convert_to_utf8($data, $encoding) {
  152. if (function_exists('iconv')) {
  153. $out = @iconv($encoding, 'utf-8', $data);
  154. }
  155. else if (function_exists('mb_convert_encoding')) {
  156. $out = @mb_convert_encoding($data, 'utf-8', $encoding);
  157. }
  158. else if (function_exists('recode_string')) {
  159. $out = @recode_string($encoding .'..utf-8', $data);
  160. }
  161. else {
  162. watchdog('php', 'Unsupported encoding %s. Please install iconv, GNU recode or mbstring for PHP.', array('%s' => $encoding), WATCHDOG_ERROR);
  163. return FALSE;
  164. }
  165. return $out;
  166. }
  167. /**
  168. * Truncate a UTF-8-encoded string safely to a number of bytes.
  169. *
  170. * If the end position is in the middle of a UTF-8 sequence, it scans backwards
  171. * until the beginning of the byte sequence.
  172. *
  173. * Use this function whenever you want to chop off a string at an unsure
  174. * location. On the other hand, if you're sure that you're splitting on a
  175. * character boundary (e.g. after using strpos() or similar), you can safely use
  176. * substr() instead.
  177. *
  178. * @param $string
  179. * The string to truncate.
  180. * @param $len
  181. * An upper limit on the returned string length.
  182. * @return
  183. * The truncated string.
  184. */
  185. function drupal_truncate_bytes($string, $len) {
  186. if (strlen($string) <= $len) {
  187. return $string;
  188. }
  189. if ((ord($string[$len]) < 0x80) || (ord($string[$len]) >= 0xC0)) {
  190. return substr($string, 0, $len);
  191. }
  192. while (--$len >= 0 && ord($string[$len]) >= 0x80 && ord($string[$len]) < 0xC0) {};
  193. return substr($string, 0, $len);
  194. }
  195. /**
  196. * Truncate a UTF-8-encoded string safely to a number of characters.
  197. *
  198. * @param $string
  199. * The string to truncate.
  200. * @param $len
  201. * An upper limit on the returned string length.
  202. * @param $wordsafe
  203. * Flag to truncate at last space within the upper limit. Defaults to FALSE.
  204. * @param $dots
  205. * Flag to add trailing dots. Defaults to FALSE.
  206. * @return
  207. * The truncated string.
  208. */
  209. function truncate_utf8($string, $len, $wordsafe = FALSE, $dots = FALSE) {
  210. if (drupal_strlen($string) <= $len) {
  211. return $string;
  212. }
  213. if ($dots) {
  214. $len -= 4;
  215. }
  216. if ($wordsafe) {
  217. $string = drupal_substr($string, 0, $len + 1); // leave one more character
  218. if ($last_space = strrpos($string, ' ')) { // space exists AND is not on position 0
  219. $string = substr($string, 0, $last_space);
  220. }
  221. else {
  222. $string = drupal_substr($string, 0, $len);
  223. }
  224. }
  225. else {
  226. $string = drupal_substr($string, 0, $len);
  227. }
  228. if ($dots) {
  229. $string .= ' ...';
  230. }
  231. return $string;
  232. }
  233. /**
  234. * Encodes MIME/HTTP header values that contain non-ASCII, UTF-8 encoded
  235. * characters.
  236. *
  237. * For example, mime_header_encode('tést.txt') returns "=?UTF-8?B?dMOpc3QudHh0?=".
  238. *
  239. * See http://www.rfc-editor.org/rfc/rfc2047.txt for more information.
  240. *
  241. * Notes:
  242. * - Only encode strings that contain non-ASCII characters.
  243. * - We progressively cut-off a chunk with truncate_utf8(). This is to ensure
  244. * each chunk starts and ends on a character boundary.
  245. * - Using \n as the chunk separator may cause problems on some systems and may
  246. * have to be changed to \r\n or \r.
  247. */
  248. function mime_header_encode($string) {
  249. if (preg_match('/[^\x20-\x7E]/', $string)) {
  250. $chunk_size = 47; // floor((75 - strlen("=?UTF-8?B??=")) * 0.75);
  251. $len = strlen($string);
  252. $output = '';
  253. while ($len > 0) {
  254. $chunk = drupal_truncate_bytes($string, $chunk_size);
  255. $output .= ' =?UTF-8?B?'. base64_encode($chunk) ."?=\n";
  256. $c = strlen($chunk);
  257. $string = substr($string, $c);
  258. $len -= $c;
  259. }
  260. return trim($output);
  261. }
  262. return $string;
  263. }
  264. /**
  265. * Complement to mime_header_encode
  266. */
  267. function mime_header_decode($header) {
  268. // First step: encoded chunks followed by other encoded chunks (need to collapse whitespace)
  269. $header = preg_replace_callback('/=\?([^?]+)\?(Q|B)\?([^?]+|\?(?!=))\?=\s+(?==\?)/', '_mime_header_decode', $header);
  270. // Second step: remaining chunks (do not collapse whitespace)
  271. return preg_replace_callback('/=\?([^?]+)\?(Q|B)\?([^?]+|\?(?!=))\?=/', '_mime_header_decode', $header);
  272. }
  273. /**
  274. * Helper function to mime_header_decode
  275. */
  276. function _mime_header_decode($matches) {
  277. // Regexp groups:
  278. // 1: Character set name
  279. // 2: Escaping method (Q or B)
  280. // 3: Encoded data
  281. $data = ($matches[2] == 'B') ? base64_decode($matches[3]) : str_replace('_', ' ', quoted_printable_decode($matches[3]));
  282. if (strtolower($matches[1]) != 'utf-8') {
  283. $data = drupal_convert_to_utf8($data, $matches[1]);
  284. }
  285. return $data;
  286. }
  287. /**
  288. * Decodes all HTML entities (including numerical ones) to regular UTF-8 bytes.
  289. *
  290. * Double-escaped entities will only be decoded once ("&amp;lt;" becomes "&lt;",
  291. * not "<"). Be careful when using this function, as decode_entities can revert
  292. * previous sanitization efforts (&lt;script&gt; will become <script>).
  293. *
  294. * @param $text
  295. * The text to decode entities in.
  296. * @param $exclude
  297. * An array of characters which should not be decoded. For example,
  298. * array('<', '&', '"'). This affects both named and numerical entities.
  299. *
  300. * @return
  301. * The input $text, with all HTML entities decoded once.
  302. */
  303. function decode_entities($text, $exclude = array()) {
  304. static $html_entities;
  305. if (!isset($html_entities)) {
  306. include_once './includes/unicode.entities.inc';
  307. }
  308. // Flip the exclude list so that we can do quick lookups later.
  309. $exclude = array_flip($exclude);
  310. // Use a regexp to select all entities in one pass, to avoid decoding
  311. // double-escaped entities twice. The PREG_REPLACE_EVAL modifier 'e' is
  312. // being used to allow for a callback (see
  313. // http://php.net/manual/en/reference.pcre.pattern.modifiers).
  314. return preg_replace('/&(#x?)?([A-Za-z0-9]+);/e', '_decode_entities("$1", "$2", "$0", $html_entities, $exclude)', $text);
  315. }
  316. /**
  317. * Helper function for decode_entities
  318. */
  319. function _decode_entities($prefix, $codepoint, $original, &$html_entities, &$exclude) {
  320. // Named entity
  321. if (!$prefix) {
  322. // A named entity not in the exclude list.
  323. if (isset($html_entities[$original]) && !isset($exclude[$html_entities[$original]])) {
  324. return $html_entities[$original];
  325. }
  326. else {
  327. return $original;
  328. }
  329. }
  330. // Hexadecimal numerical entity
  331. if ($prefix == '#x') {
  332. $codepoint = base_convert($codepoint, 16, 10);
  333. }
  334. // Decimal numerical entity (strip leading zeros to avoid PHP octal notation)
  335. else {
  336. $codepoint = preg_replace('/^0+/', '', $codepoint);
  337. }
  338. // Encode codepoint as UTF-8 bytes
  339. if ($codepoint < 0x80) {
  340. $str = chr($codepoint);
  341. }
  342. else if ($codepoint < 0x800) {
  343. $str = chr(0xC0 | ($codepoint >> 6))
  344. . chr(0x80 | ($codepoint & 0x3F));
  345. }
  346. else if ($codepoint < 0x10000) {
  347. $str = chr(0xE0 | ( $codepoint >> 12))
  348. . chr(0x80 | (($codepoint >> 6) & 0x3F))
  349. . chr(0x80 | ( $codepoint & 0x3F));
  350. }
  351. else if ($codepoint < 0x200000) {
  352. $str = chr(0xF0 | ( $codepoint >> 18))
  353. . chr(0x80 | (($codepoint >> 12) & 0x3F))
  354. . chr(0x80 | (($codepoint >> 6) & 0x3F))
  355. . chr(0x80 | ( $codepoint & 0x3F));
  356. }
  357. // Check for excluded characters
  358. if (isset($exclude[$str])) {
  359. return $original;
  360. }
  361. else {
  362. return $str;
  363. }
  364. }
  365. /**
  366. * Count the amount of characters in a UTF-8 string. This is less than or
  367. * equal to the byte count.
  368. */
  369. function drupal_strlen($text) {
  370. global $multibyte;
  371. if ($multibyte == UNICODE_MULTIBYTE) {
  372. return mb_strlen($text);
  373. }
  374. else {
  375. // Do not count UTF-8 continuation bytes.
  376. return strlen(preg_replace("/[\x80-\xBF]/", '', $text));
  377. }
  378. }
  379. /**
  380. * Uppercase a UTF-8 string.
  381. */
  382. function drupal_strtoupper($text) {
  383. global $multibyte;
  384. if ($multibyte == UNICODE_MULTIBYTE) {
  385. return mb_strtoupper($text);
  386. }
  387. else {
  388. // Use C-locale for ASCII-only uppercase
  389. $text = strtoupper($text);
  390. // Case flip Latin-1 accented letters
  391. $text = preg_replace_callback('/\xC3[\xA0-\xB6\xB8-\xBE]/', '_unicode_caseflip', $text);
  392. return $text;
  393. }
  394. }
  395. /**
  396. * Lowercase a UTF-8 string.
  397. */
  398. function drupal_strtolower($text) {
  399. global $multibyte;
  400. if ($multibyte == UNICODE_MULTIBYTE) {
  401. return mb_strtolower($text);
  402. }
  403. else {
  404. // Use C-locale for ASCII-only lowercase
  405. $text = strtolower($text);
  406. // Case flip Latin-1 accented letters
  407. $text = preg_replace_callback('/\xC3[\x80-\x96\x98-\x9E]/', '_unicode_caseflip', $text);
  408. return $text;
  409. }
  410. }
  411. /**
  412. * Helper function for case conversion of Latin-1.
  413. * Used for flipping U+C0-U+DE to U+E0-U+FD and back.
  414. */
  415. function _unicode_caseflip($matches) {
  416. return $matches[0][0] . chr(ord($matches[0][1]) ^ 32);
  417. }
  418. /**
  419. * Capitalize the first letter of a UTF-8 string.
  420. */
  421. function drupal_ucfirst($text) {
  422. // Note: no mbstring equivalent!
  423. return drupal_strtoupper(drupal_substr($text, 0, 1)) . drupal_substr($text, 1);
  424. }
  425. /**
  426. * Cut off a piece of a string based on character indices and counts. Follows
  427. * the same behavior as PHP's own substr() function.
  428. *
  429. * Note that for cutting off a string at a known character/substring
  430. * location, the usage of PHP's normal strpos/substr is safe and
  431. * much faster.
  432. */
  433. function drupal_substr($text, $start, $length = NULL) {
  434. global $multibyte;
  435. if ($multibyte == UNICODE_MULTIBYTE) {
  436. return $length === NULL ? mb_substr($text, $start) : mb_substr($text, $start, $length);
  437. }
  438. else {
  439. $strlen = strlen($text);
  440. // Find the starting byte offset
  441. $bytes = 0;
  442. if ($start > 0) {
  443. // Count all the continuation bytes from the start until we have found
  444. // $start characters
  445. $bytes = -1; $chars = -1;
  446. while ($bytes < $strlen && $chars < $start) {
  447. $bytes++;
  448. $c = ord($text[$bytes]);
  449. if ($c < 0x80 || $c >= 0xC0) {
  450. $chars++;
  451. }
  452. }
  453. }
  454. else if ($start < 0) {
  455. // Count all the continuation bytes from the end until we have found
  456. // abs($start) characters
  457. $start = abs($start);
  458. $bytes = $strlen; $chars = 0;
  459. while ($bytes > 0 && $chars < $start) {
  460. $bytes--;
  461. $c = ord($text[$bytes]);
  462. if ($c < 0x80 || $c >= 0xC0) {
  463. $chars++;
  464. }
  465. }
  466. }
  467. $istart = $bytes;
  468. // Find the ending byte offset
  469. if ($length === NULL) {
  470. $bytes = $strlen - 1;
  471. }
  472. else if ($length > 0) {
  473. // Count all the continuation bytes from the starting index until we have
  474. // found $length + 1 characters. Then backtrack one byte.
  475. $bytes = $istart; $chars = 0;
  476. while ($bytes < $strlen && $chars < $length) {
  477. $bytes++;
  478. $c = ord($text[$bytes]);
  479. if ($c < 0x80 || $c >= 0xC0) {
  480. $chars++;
  481. }
  482. }
  483. $bytes--;
  484. }
  485. else if ($length < 0) {
  486. // Count all the continuation bytes from the end until we have found
  487. // abs($length) characters
  488. $length = abs($length);
  489. $bytes = $strlen - 1; $chars = 0;
  490. while ($bytes >= 0 && $chars < $length) {
  491. $c = ord($text[$bytes]);
  492. if ($c < 0x80 || $c >= 0xC0) {
  493. $chars++;
  494. }
  495. $bytes--;
  496. }
  497. }
  498. $iend = $bytes;
  499. return substr($text, $istart, max(0, $iend - $istart + 1));
  500. }
  501. }