Changeset 1373

Show
Ignore:
Timestamp:
08/18/06 01:43:35
Author:
miyagawa
Message:

Filter::Delicious: use Plagger::UserAgent? for caching. replace hotlist.t to something lighter.

Files:

Legend:

Unmodified
Added
Removed
Modified
Copied
Moved

  • trunk/plagger/lib/Plagger/Plugin/Filter/Delicious.pm

    r1349 r1373  
    44 
    55use Digest::MD5 qw(md5_hex); 
    6 use URI; 
     6use Plagger::UserAgent; 
    77use XML::Feed; 
    88 
    …… 
    1818    my($self, $context, $args) = @_; 
    1919 
    20     # xxx need cache & interval 
    2120    my $interval = $self->conf->{interval} || 1; 
    2221    sleep $interval; 
    …… 
    2423    my $md5  = md5_hex($args->{entry}->permalink); 
    2524    my $url  = "http://del.icio.us/rss/url/$md5"; 
    26     my $feed = XML::Feed->parse( URI->new($url) ); 
     25 
     26    $self->log(info => "Going to fetch $url"); 
     27 
     28    my $ua = Plagger::UserAgent->new; 
     29       $ua->timeout(30); 
     30 
     31    my $res  = $ua->fetch($url); 
     32 
     33    if ($res->is_error) { 
     34        $self->log(error => "Fetch URL $url failed."); 
     35        return; 
     36    } 
     37 
     38    my $feed = XML::Feed->parse(\$res->content); 
    2739 
    2840    unless ($feed) { 
    …… 
    4254    my $delicious_users = $feed->entries; 
    4355    if ($delicious_users >= 30 && $self->conf->{scrape_big_numbers}) { 
     56        my $url = "http://del.icio.us/url/$md5"; 
     57        $self->log(info => "users count is more than 30. Trying to scrape from $url."); 
    4458        sleep $interval; 
    4559 
    46         my $url = "http://del.icio.us/url/$md5"; 
    47         my $ua  = Plagger::UserAgent->new; 
    4860        my $res = $ua->fetch($url); 
    4961 
    …… 
    5870    $args->{entry}->meta->{delicious_rate} = rate_of_color($delicious_users); 
    5971    $args->{entry}->meta->{delicious_users} = $delicious_users; 
     72    $self->log(info => "set delicious_users to $delicious_users"); 
    6073} 
    6174 

  • trunk/plagger/t/plugins/Filter-Delicious/scrape.t

    r1362 r1373  
    1515    class: Plagger::Cache::Null 
    1616plugins: 
    17   - module: Subscription::Config 
     17  - module: CustomFeed::Debug 
    1818    config: 
    19       feed: 
    20        - http://del.icio.us/rss/ 
     19      title: foo 
     20      entry: 
     21        - title: bar 
     22          link: http://del.icio.us/ 
    2123  - module: Filter::Delicious 
     24    config: 
     25      scrape_big_numbers: 1 
    2226--- expected 
    23 ok $context->update->feeds->[0]->entries->[0]->tags->[0]; 
    24 ok $context->update->feeds->[0]->entries->[0]->meta->{delicious_users}; 
    25 ok $context->update->feeds->[0]->entries->[0]->meta->{delicious_rate}; 
     27ok $context->update->feeds->[0]->entries->[0]->meta->{delicious_users} > 30;