Класс для определения языка и кодировки - Работа с текстом - Интересные решения для программистов - Примеры кода

Работа с текстом → Класс для определения языка и кодировки → PHP

Этот скрипт определяет не только кодировку, но и язык (русский, белорусский, украинский и ещё 60 языков).
Выполнен в виде компонента для CakePHP.

<?php
/**
 * Language detection component
 *
 * Most of code was copied from http://boxoffice.ch/pseudo/code_expl/code_class.php
 * but a lot of things were simplified and beautified
 *
 * @link http://php.southpark.com.ua
 * @author Vladimir Luchaninov
 * @version 1.0 (3 Dec 2007)
 *
 */
class LangDetectComponent {
	protected $fingerprint = null;
	protected $ngrams = array();
	//reasonable defaults
	public $ngramCount = 350;	 //default nb of ngrams created from analyzed text
	public $maxDelta = 140000;	//stop evaluation deviate strongly
	function startup(&$controller) {
		$this->fingerprint = unserialize(file_get_contents(APP . 'vendors' . DS . 'fingerprint.dat'));
	}
	/**
	 * Main function
	 *
	 * @param string $text Text with unknown language
	 * @param bool $onlyBest
	 *   true - detect the best language
	 *   false - detect all languages with possibilities
	 * @return LangDetect
	 */
	function detect($text, $onlyBest = true) {
		if (empty($text)) {
			trigger_error('Text should not be empty');
			return false;
		}
		$this->createNGrams($text);
		if ($onlyBest){
			return $this->compareNGramsOne();
		} else {
			return $this->compareNGrams();
		}
	}
	/**
	 * Get list of the available languages
	 *
	 * @return array List of languages
	 */
	function listLanguages() {
		$languages = array_keys($this->fingerprint);
		sort($languages);
		return $languages;
	}
	/**
	 * Create ngram-array of given string
	 *
	 * @param string $text
	 *
	 */
	protected function createNGrams($text) {
		$array_words = explode(" ", $text);
		$ngrams = array();
		foreach($array_words as $word) {
			$word = "_". $word . "_";
			$wordLength = strlen($word);
			for ($i=0; $i <$wordLength; $i++) { //start position within word
				for ($s=1; $s <4+1; $s++) {  //length of ngram
					if (($i + $s) <$wordLength + 1) { //length depends on postion
						$ngrams[] = substr($word, $i, $s);
					}
				}
			}
		}
		//count-> value(frequency, int)... key(ngram, string)
		$blub = array_count_values($ngrams);
		//sort array by value(frequency) desc
		arsort($blub);
		//use only top frequent ngrams (def by $ng_number)
		$top = array_slice($blub, 0, $this->ngramCount);
		$this->ngrams = array();
		foreach ($top as $keyvar => $valvar){
			$this->ngrams[] = $keyvar;
		}
	}
	/**
	 * Compare ngrams: Textinput vs lm-files.
	 *
	 * @return array of languages with lowest deviation
	 */
	protected function compareNGrams() {
		$limit = $this->maxDelta;
		foreach ($this->fingerprint as $basename => $language) {
			$delta = 0;
			//compare each ngram of input text to current lm-array
			foreach ($this->ngrams as $key => $ngram){
				//match
				if(in_array($ngram, $language)) {
					$delta += abs($key - array_search($ngram, $language));
					//no match
				} else {
					$delta += 400;
				}
				//abort: this language already differs too much
				if ($delta> $this->maxDelta) {
					break;
				}
			} // End comparison with current language
			//include only non-aborted languages in result array
			if ($delta <($this->maxDelta)-400) {
				$result[$basename] = $delta;
			}
		} //End comparison all languages
		if(!isset($result)) {
			$result = array('unknown'=>0);
		} else {
			asort($result);
		}
		return $result;
	}
	/**
	 * Variation - COMPARE ng's - Return 1 LANGUAGE only
	 *
	 * @return string Most probable language
	 */
	protected function compareNGramsOne() {
		$limit = 160000;
		foreach ($this->fingerprint as $basename => $language) {
			$delta = 0;
			foreach ($this->ngrams as $key => $ngram){
				if (in_array($ngram, $language)) {
					$delta += abs($key - array_search($ngram, $language));
				} else {
					$delta += 400;
				}
				if ($delta> $limit) {
					break;
				}
			}
			if ($delta <$limit) {
				$result[$basename] = $delta;
				$limit = $delta; //lower limit
			}
		}
		if (!isset($result)) {
			return 'unknown';
		} else {
			asort($result);
			//basename of best matching lm file
			list($result_first, $ignore) = each($result);
		}
		return $result_first;
	}
}
?>

cakephp, charset, detect, language, php

KOS_MOS

unknown

25.01.2008

Меню

Подписка на RSS:

Подпишись на RSS-ленту

Поиск

Облако тегов

	Работа с текстом → Класс для определения языка и кодировки → PHP Этот скрипт определяет не только кодировку, но и язык (русский, белорусский, украинский и ещё 60 языков). Выполнен в виде компонента для CakePHP. <?php /** * Language detection component * * Most of code was copied from http://boxoffice.ch/pseudo/code_expl/code_class.php * but a lot of things were simplified and beautified * * @link http://php.southpark.com.ua * @author Vladimir Luchaninov * @version 1.0 (3 Dec 2007) * / class LangDetectComponent { protected $fingerprint = null; protected $ngrams = array(); //reasonable defaults public $ngramCount = 350; //default nb of ngrams created from analyzed text public $maxDelta = 140000; //stop evaluation deviate strongly function startup(&$controller) { $this->fingerprint = unserialize(file_get_contents(APP . 'vendors' . DS . 'fingerprint.dat')); } /* * Main function * * @param string $text Text with unknown language * @param bool $onlyBest * true - detect the best language * false - detect all languages with possibilities * @return LangDetect / function detect($text, $onlyBest = true) { if (empty($text)) { trigger_error('Text should not be empty'); return false; } $this->createNGrams($text); if ($onlyBest){ return $this->compareNGramsOne(); } else { return $this->compareNGrams(); } } /* * Get list of the available languages * * @return array List of languages / function listLanguages() { $languages = array_keys($this->fingerprint); sort($languages); return $languages; } /* * Create ngram-array of given string * * @param string $text * / protected function createNGrams($text) { $array_words = explode(" ", $text); $ngrams = array(); foreach($array_words as $word) { $word = "_". $word . "_"; $wordLength = strlen($word); for ($i=0; $i <$wordLength; $i++) { //start position within word for ($s=1; $s <4+1; $s++) { //length of ngram if (($i + $s) <$wordLength + 1) { //length depends on postion $ngrams[] = substr($word, $i, $s); } } } } //count-> value(frequency, int)... key(ngram, string) $blub = array_count_values($ngrams); //sort array by value(frequency) desc arsort($blub); //use only top frequent ngrams (def by $ng_number) $top = array_slice($blub, 0, $this->ngramCount); $this->ngrams = array(); foreach ($top as $keyvar => $valvar){ $this->ngrams[] = $keyvar; } } /* * Compare ngrams: Textinput vs lm-files. * * @return array of languages with lowest deviation / protected function compareNGrams() { $limit = $this->maxDelta; foreach ($this->fingerprint as $basename => $language) { $delta = 0; //compare each ngram of input text to current lm-array foreach ($this->ngrams as $key => $ngram){ //match if(in_array($ngram, $language)) { $delta += abs($key - array_search($ngram, $language)); //no match } else { $delta += 400; } //abort: this language already differs too much if ($delta> $this->maxDelta) { break; } } // End comparison with current language //include only non-aborted languages in result array if ($delta <($this->maxDelta)-400) { $result[$basename] = $delta; } } //End comparison all languages if(!isset($result)) { $result = array('unknown'=>0); } else { asort($result); } return $result; } /* * Variation - COMPARE ng's - Return 1 LANGUAGE only * * @return string Most probable language / protected function compareNGramsOne() { $limit = 160000; foreach ($this->fingerprint as $basename => $language) { $delta = 0; foreach ($this->ngrams as $key => $ngram){ if (in_array($ngram, $language)) { $delta += abs($key - array_search($ngram, $language)); } else { $delta += 400; } if ($delta> $limit) { break; } } if ($delta <$limit) { $result[$basename] = $delta; $limit = $delta; //lower limit } } if (!isset($result)) { return 'unknown'; } else { asort($result); //basename of best matching lm file list($result_first, $ignore) = each($result); } return $result_first; } } ?> cakephp, charset, detect, language, php* KOS_MOS unknown 25.01.2008 Ваше имя Ваш E-mail (если хотите отслеживать комментарии) Дoкажи, что ты не робот Текст комментария	Меню CodeIgniter Алгоритмы Базы данных Дата и Время Интернет Интерфейс Мультимедиа ОС и Железо Работа с текстом Разное Технологии Файлы Подписка на RSS: Подпишись на RSS-ленту Поиск Облако тегов .net array asp bash browser charset check class clean convert count create css curl database date delete directory download element email equivalent extension file find form format function get height html http ie image javascript jquery load mysql number page perl php random scriplet send shell size sql string text time url utf8 variable vb visual basic width window xml xslt