New York Times Scrape to RSS

#!/usr/bin/perl -w
 
use strict;
use LWP::Simple;
use HTML::TreeBuilder;
use LWP::Parallel::UserAgent;
use WWW::Mechanize;
use XML::TreeBuilder;
use Getopt::Long;
use HTTP::Cookies;
use Encode;
 
my $username;
my $password;
my $feedurl;
 
GetOptions(	"user=s"	=> \$username,
		"pass=s"	=> \$password,
		"url=s"		=> \$feedurl
);
 
print STDERR "Getting login page...\n";
 
my $cookiejar = HTTP::Cookies->new();
 
my $mech = WWW::Mechanize->new();
 
$mech->agent_alias('Linux Mozilla');
$mech->cookie_jar($cookiejar);
 
$mech->get("http://www.nytimes.com/auth/login");
 
my $loginresponse = $mech->submit_form(
	form_name	=>	'login',
	fields		=>	{
		USERID		=>	$username,
		PASSWORD	=>	$password
	}
);
 
 
unless ($loginresponse->is_success()) {
	die("Error logging in!\n");
}
 
print STDERR "Logged in successfully!\n";
 
my $pua = LWP::Parallel::UserAgent->new();
$pua->cookie_jar($cookiejar);
$pua->redirect(1);
 
 
print STDERR "Getting XML...\n";
my $xml = get($feedurl);
my $feed = XML::TreeBuilder->new();
$feed->parse($xml);
my %entries;
 
print STDERR "Grabbing links...\n";
 
foreach my $item ($feed->look_down("_tag", "item")) {
	my $link = $item->look_down("_tag", "link")->as_text();
	$link =~ s/\?.*//;
 
	my $request = HTTP::Request->new();
 
	print STDERR "Registering $link...\n";
 
	$request->uri($link,);
	$request->method("GET");
 
	$pua->register($request);
 
	$entries{$link} = $item;
}
 
print STDERR "Downloading HTML...\n";
my $html = $pua->wait();
 
foreach my $entry (values(%$html)) {
	my $response = $entry->response();
	my $url = $response->base()->as_string();
	$url =~ s/\?.*//;
 
	print STDERR "Processing $url...\n";
 
	my $item = $entries{$url};
 
	my $articlehtml = HTML::TreeBuilder->new_from_content(
		decode_utf8($response->content())
	);
 
	if (my $redirelem = $articlehtml->look_down("_tag", "meta", "http-equiv", "refresh")) {
		print STDERR "Interstitial ad detected, skipping...\n";
		my $newurl = $redirelem->attr_get_i("content");
 
		($newurl) = ($newurl =~ m/url=(.*?)/);
		$newurl = "http://www.nytimes.com" . $newurl;
 
		print STDERR "Redirect URL is $newurl...\n";
	}
 
 
	# Let's clean this up for Liferea, shall we?
	my $messyelement;
 
	foreach ($articlehtml->look_down("_tag", "div", "class", "enlargeThis")) {
		$_->delete();
	}
	if ($messyelement = $articlehtml->look_down("_tag", "div", "class", "nextArticleLink")) {
		$messyelement->delete();
	}
 
	my $description = $entries{$url}->look_down("_tag", "description");
	$description->delete_content();
 
	foreach ($articlehtml->look_down("_tag", "div", "class", "image")) {
		$description->push_content($_->as_HTML());
	}
 
	if ($messyelement = $articlehtml->look_down("_tag", "div", "id", "articleInline")) {
		$messyelement->delete();
	}
 
	my %pages;
	my @pages;
	my $pageua = LWP::Parallel::UserAgent->new();
	$pageua->cookie_jar($cookiejar);
 
	if (my $pageelem = $articlehtml->look_down("_tag", "div", "id", "pageLinks")) {
		print STDERR "Multiple pages detected...\n";
 
		foreach ($pageelem->look_down("_tag", "a", "title", qr/^Page/)) {
			my $pageurl = "http://www.nytimes.com" . $_->attr_get_i("href");
			print STDERR "Registering $pageurl...\n";
 
			push(@pages, $pageurl);
			my $pagerequest = HTTP::Request->new();
			$pagerequest->uri($pageurl);
			$pagerequest->method('GET');
 
			$pageua->register($pagerequest);			
		}
 
		print STDERR "Downloading pages...\n";
 
		$pageelem->delete();
	}
 
	foreach ($articlehtml->look_down("_tag", "div", "id", "articlebody")) {
		$description->push_content($_->as_HTML());
	}
 
	my $htmlpages = $pageua->wait();
 
	if ($htmlpages) {
		print STDERR "Sorting pages...\n";
		foreach (values(%$htmlpages)) {
			my $pageresponse = $_->response();
			my $responseurl = $pageresponse->base()->as_string();
 
			$pages{$responseurl} = $pageresponse;
		}
 
		foreach (@pages) {
			print STDERR "Processing $_...\n";
 
			my $pagehtml = HTML::TreeBuilder->new_from_content(
				decode_utf8($pages{$_}->content())
			);
 
			# Let's clean this up for Liferea, shall we?
 
			if ($messyelement = $pagehtml->look_down("_tag", "div", "class", "enlargeThis")) {
				$messyelement->delete();
			}
			if ($messyelement = $pagehtml->look_down("_tag", "div", "class", "nextArticleLink")) {
				$messyelement->delete();
			}
 
			foreach ($pagehtml->look_down("_tag", "div", "class", "image")) {
				$description->push_content($_->as_HTML());
			}
 
			if ($messyelement = $pagehtml->look_down("_tag", "div", "id", "articleInline")) {
				$messyelement->delete();
			}
 
			if ($messyelement = $pagehtml->look_down("_tag", "div", "id", "pageLinks")) {
				$messyelement->delete();
			}
 
			foreach ($pagehtml->look_down("_tag", "div", "id", "articlebody")) {
				my $content = $_->as_HTML();
				$content =~ s/\(Page \d+ of \d+\)//g;
				$description->push_content($content);
			}
		}
	}
}
 
print $feed->as_XML();