Posted By

kurokikaze on 08/17/09


Tagged

language parsing nlp


Versions (?)

Who likes this?

3 people have marked this snippet as a favorite

irishsk
vali29
umang_nine


Count instances of words in text


 / Published in: PHP
 

URL: http://wp.me/pfjwm-1U

$words is array of excluded words.

  1. function stringTokenize($sBuffer, $sSplit) {
  2. $iCount = 0;
  3.  
  4. if(strlen($sBuffer) == 0)
  5. return;
  6.  
  7. $sToken = strtok($sBuffer, $sSplit);
  8. $aTokens[$iCount] = $sToken;
  9.  
  10. while ($sToken !== false) {
  11. $sToken = strtok($sSplit);
  12. if(strlen($sToken) > 0) {
  13. $iCount++;
  14. $aTokens[$iCount] = $sToken;
  15. }
  16. }
  17.  
  18. return $aTokens;
  19. }
  20.  
  21. function lcount($string) {
  22. return array($string => 1);
  23. }
  24.  
  25. function rcount($w1, $w2) {
  26. if (!is_array($w1)) {
  27. $w1 = array();
  28. }
  29. foreach ($w2 as $word => $count) {
  30. $w1[$word] += $count;
  31. }
  32. return $w1;
  33. }
  34.  
  35. $text = 'Matthew Boulton (1728�1809) was an English manufacturer and the partner of engineer James Watt. In the final quarter of the 18th century, the partnership installed hundreds of Boulton & Watt steam engines, which were a great advance on the state of the art, making possible the mechanisation of factories and mills. He became associated with James Watt when Watt\'s business partner, John Roebuck, was unable to pay a debt to Boulton, who accepted Roebuck\'s share of Watt\'s patent as settlement. He then successfully lobbied Parliament to extend Watt\'s patent for an additional seventeen years, enabling the firm to market Watt\'s steam engine. Boulton applied modern techniques to the minting of coins, striking millions of pieces for Britain and other countries, and supplying the Royal Mint with up-to-date equipment. Boulton was a key member of the Lunar Society, a group of Birmingham-area men prominent in the arts, sciences, and theology. Members included Boulton, Watt, Erasmus Darwin, Josiah Wedgwood, and Joseph Priestley. Members of the Society have been given credit for developing concepts and techniques in science, agriculture, manufacturing, mining, and transportation that laid the groundwork for the Industrial Revolution and for later discoveries, including the theory of evolution.';
  36.  
  37. $result = array_reduce(array_map('lcount', array_map('strtolower', stringTokenize($text, ' ,-.:;?()'))), 'rcount');
  38.  
  39. $words = array_flip(array(
  40. 'the',
  41. 'of',
  42. 'and',
  43. 'to',
  44. 'for',
  45. 'a',
  46. 'was',
  47. 'in',
  48. 'an',
  49. 'He',
  50. 'but'
  51. ));
  52.  
  53. $result = array_diff_key($result, $words);
  54.  
  55. arsort($result);
  56. var_dump($result);

Report this snippet  

You need to login to post a comment.