Posted By

pantuts on 10/25/13


Tagged

email curl php extract address parallel


Versions (?)

AyeEmtract - PHP Class that implements Curl in Parallel to extract Email Addresses


 / Published in: PHP
 

  1. <?php
  2.  
  3. /**
  4.  * AyeEmtract a class that crawls pages and
  5.  * then searches for email addresses using curl in parallel.
  6.  *
  7.  * PHP version >= 5.x
  8.  *
  9.  * @category PHP
  10.  * @author PANTUTS
  11.  * @license http://www.gnu.org/licenses/gpl.txt
  12.  * @link http://pantuts.com/2013/10/24/ayeemtract-using-curl-parallel/
  13.  */
  14. class AyeEmtract
  15. {
  16. // variable to store data
  17. private $_results = array();
  18. // array of urls
  19. private $_urls = array();
  20. // addition curl options
  21. private $_options = array();
  22. // errors
  23. public $errors = array();
  24.  
  25. /**
  26.   * Creates object and store array of urls and additional options
  27.   *
  28.   * @param array $urls array of urls
  29.   * @param array $options addition curl options
  30.   */
  31. public function __construct($urls, $options = array())
  32. {
  33. $this->_urls = $urls;
  34. $this->_options = $options;
  35. }
  36.  
  37. /**
  38.   * Sets curl_multi, curl options, and executes curl
  39.   * per url.
  40.   */
  41. public function start()
  42. {
  43. // initialize curl multi handle
  44. $curlMaster = curl_multi_init();
  45. // curl singles array
  46. $curlh = array();
  47.  
  48. // loop all urls
  49. foreach ($this->_urls as $i => $url) {
  50. // curl_init each url
  51. $curlh[$i] = curl_init();
  52.  
  53. curl_setopt_array(
  54. $curlh[$i], array(
  55. CURLOPT_URL => htmlentities(trim($url)),
  56. CURLOPT_SSL_VERIFYPEER => false,
  57. CURLOPT_USERAGENT => self::_setUserAgent(),
  58. CURLOPT_HEADER => false,
  59. CURLOPT_RETURNTRANSFER => true,
  60. CURLOPT_FOLLOWLOCATION => true,
  61. CURLOPT_MAXREDIRS => 7,
  62. CURLOPT_CONNECTTIMEOUT => 20,
  63. CURLOPT_FRESH_CONNECT => true )
  64. );
  65.  
  66. // check additional options
  67. if (!empty($this->_options)) {
  68. curl_setopt_array($curlh[$i], $this->_options);
  69. }
  70. // now add multi handle
  71. curl_multi_add_handle($curlMaster, $curlh[$i]);
  72. }
  73.  
  74. // execute multi handles
  75. $running = null;
  76. do {
  77. // save errors if encountered
  78. if (curl_multi_exec($curlMaster, $running) === false) {
  79. $this->errors[$i] = 'ERROR: url = ' . curl_error($curlh[$i]) .
  80. ', code = ' . curl_errno($curlh[$i]);
  81. }
  82. } while ($running > 0);
  83.  
  84. // get content on each url from curlh
  85. foreach ($curlh as $j => $ch) {
  86. // save results and remove handle
  87. $this->_results[$j] = curl_multi_getcontent($ch);
  88.  
  89. curl_multi_remove_handle($curlMaster, $ch);
  90. }
  91.  
  92. // close handle
  93. curl_multi_close($curlMaster);
  94. }
  95.  
  96. /**
  97.   * Returns array of emails
  98.   *
  99.   * @return array_unique(array)
  100.   */
  101. public function getEmails()
  102. {
  103. $emails = array();
  104. // regex to find valid emails
  105. $re = "/([\s]*)([_a-zA-Z0-9-]+(\.[_a-zA-Z0-9-]+)*([ ]+|)@([ ]+|)([a-zA-Z0-9-]+\.)+([a-zA-Z]{2,}))([\s]*)/i";
  106.  
  107. foreach ($this->_results as $i => $res) {
  108. // decode htmlentities like %3C
  109. $res = html_entity_decode($res);
  110. preg_match_all($re, $res, $matches);
  111.  
  112. foreach ($matches[0] as $match) {
  113. // save to array found emails
  114. $emails[$i] = trim($match);
  115. }
  116. }
  117. // remove duplicates
  118. return array_unique($emails);
  119. }
  120.  
  121. /**
  122.   * Returns random user-agent
  123.   *
  124.   * @return $ua
  125.   */
  126. private static function _setUserAgent()
  127. {
  128. $userAgents = array(
  129. 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:24.0) Gecko/20100101 Firefox/24.0',
  130. 'Mozilla/5.0 (Windows NT 6.2; rv:22.0) Gecko/20130405 Firefox/22.0',
  131. 'Opera/9.80 (J2ME/MIDP; Opera Mini/5.0 (Windows; U; Windows NT 5.1; en) AppleWebKit/886; U; en) Presto/2.4.15',
  132. 'Opera/9.80 (Windows NT 6.0) Presto/2.12.388 Version/12.14',
  133. 'Mozilla/5.0 (Windows; U; MSIE 9.0; WIndows NT 9.0; en-US))',
  134. 'Mozilla/5.0 (Windows; U; MSIE 7.0; Windows NT 6.0; en-US)',
  135. 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1623.0 Safari/537.36',
  136. 'Mozilla/5.0 (X11; CrOS i686 4319.74.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.57 Safari/537.36',
  137. 'Mozilla/5.0 (iPad; CPU OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5355d Safari/8536.25',
  138. 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/537.13+ (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2'
  139. );
  140. $ua = array_rand($userAgents);
  141. return $ua;
  142. }
  143. }
  144. ?>

Report this snippet  

You need to login to post a comment.