lib/CharsetDetector.class.php

Go to the documentation of this file.

<?php
class CharsetDetector {
  const DEFAULT_TARGET_CHARSET = "UTF-8";     
  const DEFAULT_MAX_ANALYZED_LENGTH = 65536;  
  const DEFAULT_MIN_RELEVANCE = 0.1;          

  public static function file_get_contents($filename) {
    return self::convert(file_get_contents($filename));
  }
  public static function convert($stringToConvert, $targetCharset = self::DEFAULT_TARGET_CHARSET) {
    $instance = new self;
    $instance->setTargetCharset($targetCharset);
    $instance->analyze($stringToConvert);
    return $instance->convertIfRelevant($stringToConvert);
  }
  public function __construct($charsetStreamAnalyzer = null) {
    if (null==$charsetStreamAnalyzer)
      $this->analyzer = CharsetStreamAnalyzerFactory::createDefault();
    $this->minRelevance = self::DEFAULT_MIN_RELEVANCE;
    $this->targetCharset = self::DEFAULT_TARGET_CHARSET;
  }
  public function getCharsetStreamAnalyzer() {
    return $this->analyzer;
  }
  public function analyze($stringToAnalyze, $maxLengthAnalyzed = self::DEFAULT_MAX_ANALYZED_LENGTH) {
    $length = strlen($stringToAnalyze);
    if ($length > $maxLengthAnalyzed)
      // Limit analyzed length
      $length = $maxLengthAnalyzed;
      
      
    for($i=0; $i< $length; $i++) {
      $char = $stringToAnalyze[$i];
      if (0x80 <= ord($char))
        // Non-ASCII character
        $this->analyzer->putchar($char);
    }
  }
  public function convertIfRelevant($stringToConvert) {
    $weightMap = $this->analyzer->getCharsetWeightMap();
    $iterator = $weightMap->createSortedIterator();
    $item = $iterator->next();
    if (null == $item)
      return $stringToConvert;
    $relWeight = $item->getRelWeight();
    $charset = $item->getCharset();
    if ($relWeight < $this->minRelevance)
      return $stringToConvert;
    else
      return CharsetConverter::convertCharset($charset, $this->targetCharset, $stringToConvert);
  }
  public function getTargetCharset() {
    return $this->targetCharset;
  }
  public function setTargetCharset($targetCharset) {
    $this->targetCharset = $targetCharset;
  }
  public function getMinRelevance() {
    return $this->minRelevance;
  }
  public function setMinRelevance($minRelevance) {
    $this->minRelevance = $minRelevance;
  }
  private $analyzer;
  private $minRelevance;
  private $targetCharset;
};

class CharsetConverter {
  public static function convertCharset($sourceEncoding, $targetEncoding, $stringToConvert) {
    return iconv($sourceEncoding, $targetEncoding, $stringToConvert);
  }
}

interface IInputStream {
  public function putchar($charToPut);
};

interface ICharWeightMapList {
  public function addWeightMap($charsetName, $charWeightMap);
};

class CharsetStreamAnalyzer implements
    IInputStream,
    ICharWeightMapList
{
  public function __construct() {
    $this->charsetWeightMap = new CharsetWeightMap;
  }
  public function putchar($charToPut) {
    $charOrdValue = ord($charToPut);
    foreach($this->weightMapList as $charset => $weightMap) {
      if (array_key_exists($charOrdValue, $weightMap))
        $this->charsetWeightMap->addWeight($charset, $weightMap[$charOrdValue]);
      // Useful for debug
      // printf("%x - %d<br/>\n", $charOrdValue, $weightMap[$charOrdValue]);
    }
  }
  public function addWeightMap($charsetName, $charWeightMap) {
    $this->weightMapList[$charsetName] = $charWeightMap;
    $this->charsetWeightMap->addWeight($charsetName, 0);
  }
  public function getCharsetWeightMap() {
    return clone $this->charsetWeightMap;
  }
  private $weightMapList = Array();
  private $charsetWeightMap;
};

class CharsetWeightMap {
  public function addWeight($charset, $weight) {
    $this->map[$charset] += $weight;
    $this->totalWeight += $weight;
  }
  public function createSortedIterator() {
    $iterator = new CharsetWeightMapIterator;
    if ($this->totalWeight > 0) {
      $sortedMap = $this->map;
      arsort($sortedMap);
      foreach($sortedMap as $charset => $weight) {
        $item = new CharsetWeightMapItem($charset, $weight, $weight/$this->totalWeight);
        // Useful for debug
        // printf("%s - %.2f %%<br/>\n", $charset, $weight/$this->totalWeight*100);
        $iterator->add($item);
      }
    }
    return $iterator;
  }
  private $map = Array();
  private $totalWeight;
};

interface ICharsetWeightMapIterator {
  public function next();
};

class CharsetWeightMapIterator implements ICharsetWeightMapIterator {
  public function next() {
    if ($this->index >= count($this->vector))
      return null;
    else
      return $this->vector[$this->index++];
  }
  public function add($charsetWeightMapItem) {
    $this->vector[]= $charsetWeightMapItem;
  }
  private $vector = Array();
  private $index = 0;
};

class CharsetWeightMapItem {
  public function __construct($charset, $absWeight, $relWeight) {
    $this->charset = $charset;
    $this->absWeight = $absWeight;
    $this->relWeight = $relWeight;
  }
  public function getCharset() {
    return $this->charset;
  }
  public function getAbsWeight() {
    return $this->absWeight;
  }
  public function getRelWeight() {
    return $this->relWeight;
  }
  private $charset;
  private $absWeight;
  private $relWeight;
};

class CharsetStreamAnalyzerFactory {
  public static function createEmpty() {
    return new CharsetStreamAnalyzer;
  }
  public static function createDefault() {
    $analyzer = self::createEmpty();
    
    // ISO-8859-2
    $charWeightMap = Array();
    for($i=0x80; $i<=0x9F; $i++)
      // Invalid character
      $charWeightMap[$i] = -5;
    for($i=0xA0; $i<=0xFF; $i++)
      // Default weight
      $charWeightMap[$i] = -1;
    // Positive weight
    $charWeightMap[0xA9] = $charWeightMap[0xAB] = $charWeightMap[0xAE] = 1;
    $charWeightMap[0xB9] = $charWeightMap[0xBB] = $charWeightMap[0xBE] = 1;
    $charWeightMap[0xC1] = $charWeightMap[0xC8] = $charWeightMap[0xC9] = 1;
    $charWeightMap[0xCC] = $charWeightMap[0xCD] = $charWeightMap[0xCF] = 1;
    $charWeightMap[0xD2] = $charWeightMap[0xD3] = $charWeightMap[0xD8] = 1;
    $charWeightMap[0xD9] = $charWeightMap[0xDA] = $charWeightMap[0xDD] = 1;
    $charWeightMap[0xE1] = $charWeightMap[0xE8] = $charWeightMap[0xE9] = 1;
    $charWeightMap[0xEC] = $charWeightMap[0xED] = $charWeightMap[0xEF] = 1;
    $charWeightMap[0xF2] = $charWeightMap[0xF3] = $charWeightMap[0xF8] = 1;
    $charWeightMap[0xF9] = $charWeightMap[0xFA] = $charWeightMap[0xFD] = 1;
    $analyzer->addWeightMap("ISO-8859-2", $charWeightMap);
    
    // Windows 1250
    $charWeightMap = Array();
    for($i=0x80; $i<=0xFF; $i++)
      // Default weight
      $charWeightMap[$i] = -1;
    // Invalid characters
    $charWeightMap[0x81] = $charWeightMap[0x83] = $charWeightMap[0x88] = -5;
    $charWeightMap[0x90] = $charWeightMap[0x98] = -5;
    // Positive weight
    $charWeightMap[0x8A] = $charWeightMap[0x8D] = $charWeightMap[0x8E] = 1;
    $charWeightMap[0x9A] = $charWeightMap[0x9D] = $charWeightMap[0x9E] = 1;
    $charWeightMap[0xC1] = $charWeightMap[0xC8] = $charWeightMap[0xC9] = 1;
    $charWeightMap[0xCC] = $charWeightMap[0xCD] = $charWeightMap[0xCF] = 1;
    $charWeightMap[0xD2] = $charWeightMap[0xD3] = $charWeightMap[0xD8] = 1;
    $charWeightMap[0xD9] = $charWeightMap[0xDA] = $charWeightMap[0xDD] = 1;
    $charWeightMap[0xE1] = $charWeightMap[0xE5] = $charWeightMap[0xE8] = 1;
    $charWeightMap[0xE9] = $charWeightMap[0xEC] = $charWeightMap[0xED] = 1;
    $charWeightMap[0xEF] = $charWeightMap[0xF2] = $charWeightMap[0xF3] = 1;
    $charWeightMap[0xF8] = $charWeightMap[0xFA] = $charWeightMap[0xFD] = 1;
    $analyzer->addWeightMap("CP1250", $charWeightMap);
    
    return $analyzer;
  }
};

?>

---