Changeset 856

Show
Ignore:
Timestamp:
05/30/06 15:14:41
Author:
miyagawa
Message:

merge from trunk to plagger-server for Enclosures support and such. Sorry for the big commit

Files:

Legend:

Unmodified
Added
Removed
Modified
Copied
Moved
  • branches/feature-server/plagger/AUTHORS

    r599 r856  
    2424Daisuke Murase (typester) 
    2525Manabu Ishii 
     26woremacx 
     27Tatsuya Noda 
     28Motokazu Sekine (cheebow) 
  • branches/feature-server/plagger/Changes

    r655 r856  
    11The latest, HTML version of this document is always available at http://plagger.org/trac.cgi/wiki/PlaggerChangeLog 
     2 
     3== 0.7.1 (2006/05/24) == 
     4 
     5=== Core === 
     6 
     7 * Added woremacx, topia and cheebow as AUTHORS 
     8 * Plagger::Cookies and Plagger::Mechanize allows Plagger to share cookies with your browser like Firefox, IE or Safari. 
     9 
     10=== New Plugins === 
     11 
     12 * Notify::UpdatePing: notify updates via XMLRPC pings (miyagawa) 
     13 * Publish::PalmDoc: publish updates to PalmDoc (cheebow) 
     14 * Publish::OutlineText: publish updates as outline text (cheebow) 
     15 
     16=== Plugins Updates === 
     17 
     18 * Filter::Regexp: now you can use utf-8 regular expressions (woremacx) 
     19 * Widget::Delicious: Support one_click_post to automatically post by clicking (s_nobu) 
     20 * Filter::FetchEnclosures: Now it's extensible using meta-plugins. Added youtube.pl as an example. Thanks to mizzy 
     21 * CustomFeed::Simple: deduplicate links by URLs. Don't add links associated with images without alt (miyagawa) 
     22 * Filter::TruePermalink: Added YouTube, MSN Mainichi 
     23 * Publish::MT: Fixed blog_id config bug. #252 
     24 * Subscription::LivedoorReader: Adds ApiKey as a sticky query to their API (suggested by mala) 
     25 * Publish::Gmail: Don't trim lines over 1000 by using quoted-printable. 
     26 * CustomFeed::GoogleNews: now accepts keyword search result page as well 
     27 * Filter::HatenaRSS: Update OPML URL. Uses Cookie sharing framework. 
     28 * CustomFeed::Frepa: Uses Cookie sharing. 
     29 * CustomFeed::Mixi: Uses Cookie sharing. 
     30 * CustomFeed::Yahoo360JP: Uses Cookie sharing. 
     31 * Subscription::LivedoorReader: Uses Cookie sharing. 
     32 * Notify::Campfire: Uses Cookie sharing. 
     33 
     34== 0.7.0 (2006/05/17) == 
     35 
     36=== Core === 
     37 
     38 * Shiny new Enclosure support! 
     39 * Dependency for MIME::Types 
     40 * Fix $cache->path_to auto creation bug 
     41 * Allow plugins/Foo/Bar.pm directory strcuture to be backward compatible 
     42 * Added regression and plugins tests suite for the first time 
     43 
     44 
     45=== New Plugins === 
     46 
     47 * Filter::FetchEnclosure: Download enclosures automatically to local 
     48 * CustomFeed::YouTube: Search YouTube and find enclosures (mizzy) 
     49 * Filter::RewriteEnclosureURL: Rewrite enclosure's URL when you republish downloaded enclosures 
     50 * Filter::POPFile: Use POPFile XMLRPC API to classify feeds (charsbar) 
     51 * Filter::TagsToTitle: Add tags to title as prefixes (charsbar) 
     52 * Filter::FindEnclosures: Automatically discover enclosures in entry content 
     53 * Filter::HEADEnclosureMetadata: Send HEAD requests to enclosures to get Length and correct filename 
     54 * Subscription::XPath: extract subscriptions from XHTML using XPath (youpy) 
     55 * Subscription::PlanetINI: extract subscriptions from Planet's config.ini file 
     56 
     57=== Plugins Updates === 
     58 
     59 * Publish::2chdat: Internal fix to use id_safe method 
     60 * Subscription::Bloglines: support enclosures taken from sync API 
     61 * Publish::Gmail: Attach enclosures if they're downloaded locally with FetchEnclosure 
     62 * Filter::TruePermalink: Now supports rewriting enclosure URL as well. 
     63 * Filter::TruePermalink: Support recursive mode and redirector resolution 
     64 * Publish::Feed: Support enclosure creation in RSS 2.0 and Atom feeds 
     65 * Filter::SpamAssassin: Nuked some options (charsbar) 
     66 * Filter::HatenaKeyword: Use title to extract keywords as well 
     67 * Filter::StripRSSAd: Added pheedo.jp pattern. Now can strip entry if it contains certain pattern. 
     68 *  
     69== 0.6.6 (2006/05/12) == 
     70 
     71=== Core === 
     72 
     73 * Added sites upgrade files (woremacx) 
     74 * Don't truncate newline after password rewrite #200 
     75 * Share $feed object between subscription and update 
     76 * encode detection now uses XML encoding declaration first 
     77 * Unhandled feed is removed from $context->subscription 
     78 * Plagger::Date->parse doesn't force preference timezone if parsed datetime is floating 
     79 * Load plugins from plugins/*/lib #212 
     80 
     81=== New Plugins === 
     82  
     83 * Filter::HatenaKeywordTag: Use Hatena Keyword API to auto-tag (secondlife) 
     84 * Search::Estraier: Search plugin to use Hyper Estraier Node API (miyagawa) 
     85 * Subscription::XOXO: load subscription from XOXO microformats (miyagawa) 
     86 * Publish::2chdat: Create 2ch subject.txt and *.dat files (miyagawa) 
     87 * Filter::2chRSSContent: Fix rss.s2ch.net feed content (miyagawa) 
     88 * Filter::Markdown: Filter entry body using Markdown syntax (s_nobu)  
     89 
     90=== Plugins Updates === 
     91 
     92 * Notify::IRC: added password config to plagger-ircbot #197 
     93 * Filter::TruePermalink: Updated Y! Blogsearch pattern. Added reedit.com 
     94 * rename Publish::Spotlight to Search::Spotlight #207 
     95 * CustomFeed::Mixi: support "ashiato". Update WWW::Mixi deps 
     96 * Rule::URLBL: now works with $args->{entry} to be used with Filter::Rule 
     97 * Publish::Planet: get members from subscription rather than update #198 
     98 * CustomFeed::BloglinesCitation: Handle local datetime PST #187 
     99 * Filter::EntryFullText: now work with rule 
    2100 
    3101== 0.6.5 (2006/04/28) == 
  • branches/feature-server/plagger/MANIFEST

    r669 r856  
    1111assets/plugins/Filter-EntryFullText/blog_goo_ne_jp.yaml 
    1212assets/plugins/Filter-EntryFullText/blog_tech.rikunabi_next.yaml 
     13assets/plugins/Filter-EntryFullText/business-i.yaml 
    1314assets/plugins/Filter-EntryFullText/chugoku-np.yaml 
    1415assets/plugins/Filter-EntryFullText/chuspo_dragons.yaml 
     
    3132assets/plugins/Filter-EntryFullText/japan_linux_com.yaml 
    3233assets/plugins/Filter-EntryFullText/japan_zdnet_com.yaml 
     34assets/plugins/Filter-EntryFullText/kyodo.yaml 
    3335assets/plugins/Filter-EntryFullText/kyoko_shimbun_news.yaml 
     36assets/plugins/Filter-EntryFullText/kyoto-np.yaml 
    3437assets/plugins/Filter-EntryFullText/linuxjournal.yaml 
    3538assets/plugins/Filter-EntryFullText/livedoorblog.pl 
     39assets/plugins/Filter-EntryFullText/mainichi-msn.yaml 
     40assets/plugins/Filter-EntryFullText/mycom_journal.yaml 
    3641assets/plugins/Filter-EntryFullText/netkeiba.yaml 
    3742assets/plugins/Filter-EntryFullText/news_com.yaml 
    3843assets/plugins/Filter-EntryFullText/newsforge.yaml 
    3944assets/plugins/Filter-EntryFullText/nikkansports.yaml 
     45assets/plugins/Filter-EntryFullText/nikkei.yaml 
    4046assets/plugins/Filter-EntryFullText/nytimes.yaml 
     47assets/plugins/Filter-EntryFullText/okinawatimes_day.yaml 
    4148assets/plugins/Filter-EntryFullText/osaka_nikkansports.yaml 
    42 assets/plugins/Filter-EntryFullText/pcweb_mycom.yaml 
    4349assets/plugins/Filter-EntryFullText/physorg.yaml 
    4450assets/plugins/Filter-EntryFullText/plaza_rakuten.yaml 
     
    4652assets/plugins/Filter-EntryFullText/rbbtoday_com.yaml 
    4753assets/plugins/Filter-EntryFullText/reuters.yaml 
     54assets/plugins/Filter-EntryFullText/ryukyushimpo.yaml 
    4855assets/plugins/Filter-EntryFullText/sanspo.yaml 
    4956assets/plugins/Filter-EntryFullText/sciam.yaml 
    5057assets/plugins/Filter-EntryFullText/searchenginejournal.yaml 
    5158assets/plugins/Filter-EntryFullText/sixapart.pl 
     59assets/plugins/Filter-EntryFullText/slashcode.pl 
    5260assets/plugins/Filter-EntryFullText/slashdot_jp.yaml 
    5361assets/plugins/Filter-EntryFullText/sponichi.yaml 
     
    5664assets/plugins/Filter-EntryFullText/theinquirer.yaml 
    5765assets/plugins/Filter-EntryFullText/theregister.yaml 
     66assets/plugins/Filter-EntryFullText/ti-da_net.yaml 
    5867assets/plugins/Filter-EntryFullText/usatoday.yaml 
    5968assets/plugins/Filter-EntryFullText/wired_com.yaml 
     69assets/plugins/Filter-EntryFullText/worldtimes.yaml 
    6070assets/plugins/Filter-EntryFullText/www_nikkeibp.yaml 
    6171assets/plugins/Filter-EntryFullText/yakult.yaml 
    6272assets/plugins/Filter-EntryFullText/yomiuri.yaml 
     73assets/plugins/Filter-EntryFullText/youtube.yaml 
    6374assets/plugins/Filter-EntryFullText/zakzak.yaml 
    6475assets/plugins/Filter-EntryFullText/zzz_google_adsense.pl 
     76assets/plugins/Filter-FindEnclosures/youtube.pl 
    6577assets/plugins/Filter-StripRSSAd/feedburner 
    6678assets/plugins/Filter-StripRSSAd/google_adsense 
    6779assets/plugins/Filter-StripRSSAd/google_adsense2 
    6880assets/plugins/Filter-StripRSSAd/pheedo 
     81assets/plugins/Filter-StripRSSAd/pheedo_jp 
     82assets/plugins/Filter-StripRSSAd/pheedo_jp_ad_entry.yaml 
    6983assets/plugins/Filter-StripRSSAd/plaza_rakuten 
    7084assets/plugins/Filter-StripRSSAd/rssad_jp 
     
    7286assets/plugins/Filter-StripRSSAd/valueclick 
    7387assets/plugins/Filter-TruePermalink/2chrss.yaml 
     88assets/plugins/Filter-TruePermalink/cnet_podcast.yaml 
     89assets/plugins/Filter-TruePermalink/imenu.yaml 
     90assets/plugins/Filter-TruePermalink/msn-mainichi.yaml 
    7491assets/plugins/Filter-TruePermalink/namaan.yaml 
    7592assets/plugins/Filter-TruePermalink/rd_yahoo.yaml 
    7693assets/plugins/Filter-TruePermalink/reddit.yaml 
     94assets/plugins/Filter-TruePermalink/redirectors.yaml 
    7795assets/plugins/Filter-TruePermalink/refrss.yaml 
    7896assets/plugins/Filter-TruePermalink/tech_souken.yaml 
     
    8098assets/plugins/Filter-TruePermalink/yahoo_blog_search2.yaml 
    8199assets/plugins/Filter-TruePermalink/yahoo_us_rd.yaml 
     100assets/plugins/Filter-TruePermalink/youtube.yaml 
    82101assets/plugins/Notify-IRC/irc_notify.tt 
     102assets/plugins/Notify-Tiarra/irc_notify.tt 
    83103assets/plugins/Publish-CHTML/chtml_entry.tt 
    84104assets/plugins/Publish-CHTML/chtml_feed.tt 
     
    89109assets/plugins/Publish-MTWidget/mt_widget.tt 
    90110assets/plugins/Publish-OPML/opml.tt 
     111assets/plugins/Publish-PalmDoc/palmdoc.tt 
    91112assets/plugins/Publish-Planet/default/static/css/handheld.css 
    92113assets/plugins/Publish-Planet/default/static/css/print.css 
     
    104125bin/spotlight_comment.scpt 
    105126Changes 
    106 examples/aggregator.yaml 
    107127examples/atode.yaml 
    108128examples/bloglines2gmail.yaml 
     
    110130examples/livedoorreader2gmail.yaml 
    111131examples/planet.yaml 
     132examples/podcast.yaml 
     133examples/search.yaml 
     134examples/xoxo-planet.yaml 
     135examples/xoxo2opml.yaml 
     136examples/yapcvideo.yaml 
    112137inc/Module/AutoInstall.pm 
    113138inc/Module/Install.pm 
     
    127152lib/Plagger/Cache/Null.pm 
    128153lib/Plagger/CacheProxy.pm 
     154lib/Plagger/Cookies.pm 
    129155lib/Plagger/Crypt.pm 
    130156lib/Plagger/Crypt/Base64.pm 
    131157lib/Plagger/Date.pm 
     158lib/Plagger/Enclosure.pm 
    132159lib/Plagger/Entry.pm 
    133160lib/Plagger/Feed.pm 
     161lib/Plagger/Mechanize.pm 
    134162lib/Plagger/Operator.pm 
    135163lib/Plagger/Plugin.pm 
     
    152180lib/Plagger/Plugin/CustomFeed/SVNLog.pm 
    153181lib/Plagger/Plugin/CustomFeed/Yahoo360JP.pm 
     182lib/Plagger/Plugin/CustomFeed/YouTube.pm 
    154183lib/Plagger/Plugin/Filter/2chNewsokuTitle.pm 
     184lib/Plagger/Plugin/Filter/2chRSSContent.pm 
    155185lib/Plagger/Plugin/Filter/2chRSSPermalink.pm 
    156186lib/Plagger/Plugin/Filter/AtomLinkRelated.pm 
     
    167197lib/Plagger/Plugin/Filter/FeedBurnerPermalink.pm 
    168198lib/Plagger/Plugin/Filter/FeedFlareStripper.pm 
     199lib/Plagger/Plugin/Filter/FetchEnclosure.pm 
     200lib/Plagger/Plugin/Filter/FindEnclosures.pm 
    169201lib/Plagger/Plugin/Filter/FloatingDateTime.pm 
    170202lib/Plagger/Plugin/Filter/HatenaBookmarkTag.pm 
     
    173205lib/Plagger/Plugin/Filter/HatenaDiaryKeywordUnlink.pm 
    174206lib/Plagger/Plugin/Filter/HatenaFormat.pm 
     207lib/Plagger/Plugin/Filter/HatenaKeywordTag.pm 
     208lib/Plagger/Plugin/Filter/HEADEnclosureMetadata.pm 
    175209lib/Plagger/Plugin/Filter/ImageInfo.pm 
     210lib/Plagger/Plugin/Filter/Markdown.pm 
    176211lib/Plagger/Plugin/Filter/NamaanPermalink.pm 
    177212lib/Plagger/Plugin/Filter/Pipe.pm 
     213lib/Plagger/Plugin/Filter/POPFile.pm 
    178214lib/Plagger/Plugin/Filter/Profanity.pm 
    179215lib/Plagger/Plugin/Filter/Regexp.pm 
    180216lib/Plagger/Plugin/Filter/ResolveRelativeLink.pm 
     217lib/Plagger/Plugin/Filter/RewriteEnclosureURL.pm 
    181218lib/Plagger/Plugin/Filter/Romanize.pm 
    182219lib/Plagger/Plugin/Filter/Romanize/Japanese.pm 
     
    186223lib/Plagger/Plugin/Filter/SpamAssassin.pm 
    187224lib/Plagger/Plugin/Filter/StripRSSAd.pm 
     225lib/Plagger/Plugin/Filter/TagsToTitle.pm 
    188226lib/Plagger/Plugin/Filter/tDiaryComment.pm 
    189227lib/Plagger/Plugin/Filter/Thumbnail.pm 
     
    202240lib/Plagger/Plugin/Notify/MSAgent.pm 
    203241lib/Plagger/Plugin/Notify/SSTP.pm 
     242lib/Plagger/Plugin/Notify/Tiarra.pm 
     243lib/Plagger/Plugin/Notify/UpdatePing.pm 
     244lib/Plagger/Plugin/Publish/2chdat.pm 
    204245lib/Plagger/Plugin/Publish/CHTML.pm 
    205246lib/Plagger/Plugin/Publish/CSV.pm 
     
    215256lib/Plagger/Plugin/Publish/MTWidget.pm 
    216257lib/Plagger/Plugin/Publish/OPML.pm 
     258lib/Plagger/Plugin/Publish/OutlineText.pm 
     259lib/Plagger/Plugin/Publish/PalmDoc.pm 
    217260lib/Plagger/Plugin/Publish/PDF.pm 
    218261lib/Plagger/Plugin/Publish/Pipe.pm 
     
    224267lib/Plagger/Plugin/Publish/Speech/Win32.pm 
    225268lib/Plagger/Plugin/Publish/Takahashi.pm 
     269lib/Plagger/Plugin/Search/Estraier.pm 
    226270lib/Plagger/Plugin/Search/Namazu.pm 
    227271lib/Plagger/Plugin/Search/Rast.pm 
     
    240284lib/Plagger/Plugin/Subscription/PingServer.pm 
    241285lib/Plagger/Plugin/Subscription/Planet.pm 
     286lib/Plagger/Plugin/Subscription/PlanetINI.pm 
     287lib/Plagger/Plugin/Subscription/XOXO.pm 
     288lib/Plagger/Plugin/Subscription/XPath.pm 
    242289lib/Plagger/Plugin/Widget/BloglinesSubscription.pm 
    243290lib/Plagger/Plugin/Widget/BulkfeedsSpamReport.pm 
  • branches/feature-server/plagger/MANIFEST.SKIP

    r669 r856  
    1616\.bak$ 
    1717\.orig$ 
     18plugins/.*\.pm$ 
     19tools/release\.pl 
     20t/plugins 
     21t/regression 
     22^# 
  • branches/feature-server/plagger/Makefile.PL

    r692 r856  
    2323requires('HTML::ResolveLink'); 
    2424requires('Date::Parse'); 
     25requires('MIME::Types', 1.16); 
    2526 
    2627build_requires(Test::More => 0.42); 
     
    3637        recommends('Test::Pod::Coverage'), 
    3738    ], 
     39    'Cookie sharing with Firefox' => [ 
     40        -default => 0, 
     41        recommends('HTTP::Cookies::Mozilla'), 
     42    ], 
     43); 
     44 
     45features( 
    3846    'Subscription::Bloglines' => [ 
    3947        -default => 1, 
    4048        recommends('WebService::Bloglines', 0.11), 
    41         recommends('XML::Liberal', 0.06), 
     49        recommends('XML::Liberal', 0.09), 
    4250    ], 
    4351    'Subscription::OPML' => [ 
     
    4957        recommends('XML::Feed', 0.08), 
    5058        recommends('XML::Atom'), 
    51         recommends('XML::RSS::LibXML', 0.19), 
     59        recommends('XML::RSS::LibXML', 0.20), 
    5260        recommends('XML::RSS::Liberal'), 
    5361    ], 
     
    214222        recommends('DateTime::Locale'), 
    215223    ], 
     224    'Subscription::PlanetINI' => [ 
     225        -default => 0, 
     226        recommends('Config::INI::Simple'), 
     227    ], 
     228    'Notify::UpdatePing' => [ 
     229        -default => 0, 
     230        recommends('XMLRPC::Lite'), 
     231    ], 
     232    'Publish::PalmDoc' => [ 
     233        -default => 0, 
     234        recommends('Palm::PalmDoc'), 
     235    ], 
    216236); 
    217237 
    218238if ($^O eq 'darwin') { 
    219239    features( 
     240        'Cookie sharing with Safari' => [ 
     241            -default => 0, 
     242            recommends('HTTP::Cookies::Safari'), 
     243        ], 
    220244        'Search::Spotlight' => [ 
    221245            -default => 1, 
    222246            recommends('Mac::Glue'), 
    223247        ], 
    224     ); 
    225     features( 
    226248        'Publish::Speech' => [ 
    227249            -default => 0, 
     
    234256} elsif ($^O eq 'MSWin32') { 
    235257    features( 
     258        'Cookie sharing with MSIE' => [ 
     259            -default => 0, 
     260            recommends('HTTP::Cookies::Microsoft'), 
     261        ], 
    236262        'Publish::Speech' => [ 
    237263            -default => 0, 
     
    247273    ); 
    248274} 
     275 
     276tests 't/*.t t/*/*.t t/*/*/*.t'; 
    249277 
    250278auto_include; 
  • branches/feature-server/plagger/assets/plugins/Filter-EntryFullText/impress.yaml

    r651 r856  
    11author: kazeburo 
    2 handle: http://\w+\.watch\.impress\.co\.jp/ 
     2handle: http://(\w+\.watch|k-tai)\.impress\.co\.jp/ 
    33extract: <!--\s?本文開始\s?-->(.*)<!--\s?本文終了\s?--> 
    44extract_capture: body 
  • branches/feature-server/plagger/assets/plugins/Filter-EntryFullText/itmedia.yaml

    r586 r856  
    11author: manabou 
    2 handle: http://www\.itmedia\.co\.jp/news/articles 
    3 extract: <div class="newart">.*?<h1>(.*?)</h1>.*?<!--BODY-->(.*?)<!--BODYEND--> 
    4 extract_capture: title body 
     2handle: http://(\w+)\.itmedia\.co\.jp/(\w+)/articles 
     3extract: <div class="newart">.*?(?:<div id="update">(\d{4}å¹´\d\d月\d\dæ—¥ \d\d時\d\d分) æ›´æ–°</div>)?.*?<h1>(.*?)</h1>\s*<h5>(.*?)</h5>\s*(?:<div id="update">(\d{4}å¹´\d\d月\d\dæ—¥ \d\d時\d\d分) æ›´æ–°</div>)?.*?<!--BODY-->(.*?)<!--BODYEND--> 
     4extract_capture: date1 title summary date2 body 
     5extract_after_hook: $data->{date} = $data->{date1} || $data->{date2} 
     6extract_date_format: %Yå¹´%m月%dæ—¥ %H時%M分 
  • branches/feature-server/plagger/assets/plugins/Filter-EntryFullText/mycom_journal.yaml

    r694 r856  
    1 # upgrade http://pcweb.mycom.co.jp/haishin/rss/index.rdf 
     1# upgrade http://journal.mycom.co.jp/haishin/rss/index.rdf 
    22author: Nobuhito Sato 
    33handle: http://journal\.mycom\.co\.jp/ 
  • branches/feature-server/plagger/assets/plugins/Filter-EntryFullText/nikkansports.yaml

    r559 r856  
    99  - %Yå¹´%m月%dæ—¥%H時%M分 
    1010  - %Y/%m/%d %H:%M 
    11  
     11extract_date_timezone: Asia/Tokyo 
  • branches/feature-server/plagger/assets/plugins/Filter-TruePermalink/yahoo_blog_search.yaml

    r612 r856  
    11author: Tatsuhiko Miyagawa 
    22match: http://rd\.yahoo\.co\.jp/rss/l/blogsearch 
    3 rewrite: s!^http://rd\.yahoo\.co\.jp/rss/l/blogsearch/search/\*!
     3rewrite: s!^http://rd\.yahoo\.co\.jp/rss/l/blogsearch/search/.*?\*\-http%3A!http:
    44 
  • branches/feature-server/plagger/examples/bloglines2gmail.yaml

    r402 r856  
    22  plugin_path: 
    33    - /home/miyagawa/plagger/plugins 
    4   #template_path: /home/miyagawa/plagger/template
     4  assets_path: /home/miyagawa/plagger/asset
    55  timezone: Asia/Tokyo 
    66  log: 
  • branches/feature-server/plagger/examples/livedoorreader2gmail.yaml

    r626 r856  
    22  plugin_path: 
    33    - /home/miyagawa/plagger/plugins 
    4   #assets_path: /home/miyagawa/plagger/assets 
     4  assets_path: /home/miyagawa/plagger/assets 
    55  timezone: Asia/Tokyo 
    66  log: 
  • branches/feature-server/plagger/lib/Plagger.pm

    r693 r856  
    11package Plagger; 
    22use strict; 
    3 our $VERSION = '0.6.5'; 
     3our $VERSION = '0.7.1'; 
    44 
    55use 5.8.1; 
     
    4141    if (-e $opt{config} && -r _) { 
    4242        $config = YAML::LoadFile($opt{config}); 
    43         $self->load_include($config); 
    44         $self->{conf} = $config->{global}; 
    45         $self->{conf}->{log} ||= { level => 'debug' }; 
    4643        $self->{config_path} = $opt{config}; 
     44    } elsif (ref($opt{config}) && ref($opt{config}) eq 'SCALAR') { 
     45        $config = YAML::Load(${$opt{config}}); 
     46    } elsif (ref($opt{config}) && ref($opt{config}) eq 'HASH') { 
     47        $config = $opt{config}; 
    4748    } else { 
    4849        croak "Plagger->bootstrap: $opt{config}: $!"; 
    4950    } 
    5051 
     52    $self->load_include($config); 
     53    $self->{conf} = $config->{global}; 
     54    $self->{conf}->{log} ||= { level => 'debug' }; 
     55 
     56    no warnings 'redefine'; 
    5157    local *Plagger::context = sub { $self }; 
    5258 
     
    6571sub rewrite_config { 
    6672    my $self = shift; 
     73 
     74    unless ($self->{config_path}) { 
     75        $self->log(warn => "config is not loaded from file. Ignoring rewrite tasks."); 
     76        return; 
     77    } 
    6778 
    6879    open my $fh, $self->{config_path} or $self->error("$self->{config_path}: $!"); 
     
    136147 
    137148    # use config filename as a base directory for cache 
    138     my $base = ( basename($config) =~ /^(.*?)\.yaml$/ )[0]
     149    my $base = ( basename($config) =~ /^(.*?)\.yaml$/ )[0] || 'config'
    139150    my $dir  = $base eq 'config' ? ".plagger" : ".plagger-$base"; 
    140151 
     
    159170                $ent = File::Spec->catfile($path, $ent); 
    160171                if (-f $ent && $ent =~ /\.pm$/) { 
    161                     my $pkg = $self->extract_package($ent) 
    162                         or die "Can't find package from $ent"; 
    163                     (my $base = $ent) =~ s!^$path/!!; 
    164                     $self->plugins_path->{$pkg} = $ent; 
     172                    $self->add_plugin_path($ent); 
    165173                } elsif (-d $ent) { 
    166174                    my $lib = File::Spec->catfile($ent, "lib"); 
     
    168176                        $self->log(debug => "Add $lib to INC path"); 
    169177                        unshift @INC, $lib; 
     178                    } else { 
     179                        my $rule = File::Find::Rule->new; 
     180                           $rule->file; 
     181                           $rule->name('*.pm'); 
     182                        my @modules = $rule->in($ent); 
     183                        for my $module (@modules) { 
     184                            $self->add_plugin_path($module); 
     185                        } 
    170186                    } 
    171187                } 
     
    177193        $self->load_plugin($plugin) unless $plugin->{disable}; 
    178194    } 
     195} 
     196 
     197sub add_plugin_path { 
     198    my($self, $file) = @_; 
     199 
     200    my $pkg = $self->extract_package($file) 
     201        or die "Can't find package from $file"; 
     202    $self->plugins_path->{$pkg} = $file; 
     203    $self->log(debug => "$file is added as a path to plugin $pkg"); 
    179204} 
    180205 
     
    220245    $module = "Plagger::Plugin::$module"; 
    221246 
    222     if (my $path = $self->plugins_path->{$module}) { 
     247    if ($module->isa('Plagger::Plugin')) { 
     248        $self->log(debug => "$module is loaded elsewhere ... maybe .t script?"); 
     249    } elsif (my $path = $self->plugins_path->{$module}) { 
    223250        eval { require $path } or die $@; 
    224251    } else { 
  • branches/feature-server/plagger/lib/Plagger/Cache.pm

    r590 r856  
    3636    if (@path > 1) { 
    3737        my @chunk = @path[0..$#path-1]; 
    38         mkpath(File::Spec->catfile(@chunk), 0, 0700); 
     38        mkpath(File::Spec->catfile($self->{base}, @chunk), 0, 0700); 
    3939    } 
    4040    File::Spec->catfile($self->{base}, @path); 
  • branches/feature-server/plagger/lib/Plagger/Entry.pm

    r384 r856  
    1717        tags    => [], 
    1818        meta    => {}, 
     19        enclosures => [], 
    1920    }, $class; 
    2021} 
     
    6970} 
    7071 
     72sub add_enclosure { 
     73    my($self, $enclosure) = @_; 
     74 
     75    # don't add enclosure with the same URL again and again 
     76    unless ($enclosure->url && grep { $_->url && $_->url eq $enclosure->url } $self->enclosures) { 
     77        push @{ $self->{enclosures} }, $enclosure; 
     78    } 
     79} 
     80 
     81sub enclosure { 
     82    my $self = shift; 
     83    wantarray ? @{$self->{enclosures}} : $self->{enclosures}->[0]; 
     84} 
     85 
     86sub enclosures { 
     87    my $self = shift; 
     88    wantarray ? @{$self->{enclosures}} : $self->{enclosures}; 
     89} 
     90 
     91sub has_enclosure { 
     92    my $self = shift; 
     93    scalar @{$self->{enclosures}} > 0; 
     94} 
     95 
    71961; 
    7297 
  • branches/feature-server/plagger/lib/Plagger/Feed.pm

    r455 r856  
    5555} 
    5656 
     57sub id_safe { 
     58    my $self = shift; 
     59    my $id = $self->id; 
     60    $id =~ s![^\w\s]+!_!g; 
     61    $id =~ s!\s+!_!g; 
     62    $id; 
     63} 
     64 
    5765sub title_text { 
    5866    my $self = shift; 
  • branches/feature-server/plagger/lib/Plagger/Plugin.pm

    r593 r856  
    55__PACKAGE__->mk_accessors( qw(conf rule rule_hook cache) ); 
    66 
     7use Plagger::Cookies; 
    78use Plagger::Crypt; 
    89use Plagger::Rule; 
     
    114115} 
    115116 
     117sub cookie_jar { 
     118    my $self = shift; 
     119 
     120    my $agent_conf = Plagger->context->conf->{user_agent} || {}; 
     121    if ($agent_conf->{cookies}) { 
     122        return Plagger::Cookies->create($agent_conf->{cookies}); 
     123    } 
     124 
     125    return $self->cache->cookie_jar; 
     126} 
     127 
    1161281; 
  • branches/feature-server/plagger/lib/Plagger/Plugin/Aggregator/Simple.pm

    r680 r856  
    44 
    55use Feed::Find; 
     6use Plagger::Enclosure; 
    67use Plagger::UserAgent; 
    78use List::Util qw(first); 
     
    1213 
    1314$XML::Feed::RSS::PREFERRED_PARSER = first { $_->require } qw( XML::RSS::Liberal XML::RSS::LibXML XML::RSS ); 
     15 
     16eval { require XML::Liberal }; 
     17if (!$@ && XML::Liberal->can('globally_override')) { 
     18    XML::Liberal->globally_override('LibXML'); 
     19} 
    1420 
    1521sub register { 
     
    8793    unless ($remote) { 
    8894        $context->log(error => "Parsing $url failed. " . ($@ || XML::Feed->errstr)); 
    89         next
     95        return
    9096    } 
    9197 
     
    137143        $entry->body(_u($e->content->body || $e->summary->body)); 
    138144 
     145        # enclosure support, to be added to XML::Feed 
     146        if ($remote->format =~ /^RSS / && $e->{entry}->{enclosure}) { 
     147            my $enclosure = Plagger::Enclosure->new; 
     148            $enclosure->url( URI->new($e->{entry}->{enclosure}->{url}) ); 
     149            $enclosure->length($e->{entry}->{enclosure}->{length}); 
     150            $enclosure->auto_set_type($e->{entry}->{enclosure}->{type}); 
     151            $entry->add_enclosure($enclosure); 
     152        } elsif ($remote->format eq 'Atom') { 
     153            for my $link ( grep { $_->rel eq 'enclosure' } $e->{entry}->link ) { 
     154                my $enclosure = Plagger::Enclosure->new; 
     155                $enclosure->url( URI->new($link->href) ); 
     156                $enclosure->length($link->length); 
     157                $enclosure->auto_set_type($link->type); 
     158                $entry->add_enclosure($enclosure); 
     159            } 
     160        } 
     161 
     162        # Media RSS 
     163        my $media_ns = "http://search.yahoo.com/mrss"; 
     164        my $media = $e->{entry}->{$media_ns}->{group} || $e->{entry}; 
     165        my $content = $media->{$media_ns}->{content} || []; 
     166           $content = [ $content ] unless ref $content; 
     167 
     168        for my $media_content (@{$content}) { 
     169            my $enclosure = Plagger::Enclosure->new; 
     170            $enclosure->url( URI->new($media_content->{url}) ); 
     171            $enclosure->auto_set_type($media_content->{type}); 
     172            $entry->add_enclosure($enclosure); 
     173        } 
     174 
     175        if (my $thumbnail = $media->{$media_ns}->{thumbnail}) { 
     176            $entry->icon({ 
     177                url   => $thumbnail->{url}, 
     178                width => $thumbnail->{width}, 
     179                height => $thumbnail->{height}, 
     180            }); 
     181        } 
     182 
    139183        my $args = { 
    140184            entry      => $entry, 
  • branches/feature-server/plagger/lib/Plagger/Plugin/CustomFeed/AmazonAssociateReportJP.pm

    r430 r856  
    44use base qw (Plagger::Plugin); 
    55 
    6 use WWW::Mechanize; 
     6use Plagger::Mechanize; 
    77 
    88sub register { 
     
    5151use strict; 
    5252use warnings; 
    53 use WWW::Mechanize; 
     53use Plagger::Mechanize; 
    5454use base qw(Class::Accessor::Fast); 
    5555 
     
    5959    my $class = shift; 
    6060    my $plugin = shift; 
    61     my $mech = WWW::Mechanize->new; 
     61    my $mech = Plagger::Mechanize->new; 
    6262    $mech->agent_alias( "Windows IE 6" ); 
    6363    return bless { 
     
    144144=head1 SEE ALSO 
    145145 
    146 L<Plagger>, L<WWW::Mechanize> 
     146L<Plagger>, L<Plagger::Mechanize> 
    147147 
    148148=cut 
  • branches/feature-server/plagger/lib/Plagger/Plugin/CustomFeed/Frepa.pm

    r650 r856  
    77use Time::HiRes; 
    88use UNIVERSAL::require; 
    9 use WWW::Mechanize; 
     9use Plagger::Mechanize; 
    1010 
    1111sub plugin_id { 
     
    2525    my ($self, $context) = @_; 
    2626 
    27     $self->{mech} = WWW::Mechanize->new(cookie_jar => $self->cache->cookie_jar); # enbug??? 
    28     $self->{mech}->agent_alias( "Windows IE 6" ); 
     27    $self->{mech} = Plagger::Mechanize->new(cookie_jar => $self->cookie_jar); 
    2928 
    3029    my $feed = Plagger::Feed->new; 
     
    130129    my $start_url = 'http://www.frepa.livedoor.com/'; 
    131130    my $res = $self->{mech}->get($start_url); 
    132     return 0 unless $self->{mech}->success; 
     131    return unless $self->{mech}->success; 
    133132 
    134133    if ($self->{mech}->content =~ /loginside/) { 
     134        unless ($args{livedoor_id} && $args{password}) { 
     135            Plagger->context->log(error => "Error logging in using existent Cookies. Your User-Agent (" . $self->{mech}->agent . ") should strictly match with the UA used with the Cookies."); 
     136            return; 
     137        } 
     138 
    135139        Plagger->context->log(debug => "cookie not found. logging in"); 
    136140        $self->{mech}->submit_form( 
     
    142146        ); 
    143147        $self->{mech}->submit; 
    144         return 0 unless $self->{mech}->success; 
    145         return 0 if $self->{mech}->content =~ /loginside/; 
     148        return unless $self->{mech}->success; 
     149        return if $self->{mech}->content =~ /loginside/; 
    146150    } 
    147151 
     
    208212C<fetch_body_interval> and C<show_icon>. 
    209213 
     214Note that you don't have to supply livedoor_id and password if you set 
     215global cookie_jar in your configuration file and the cookie_jar 
     216contains a valid login session there, such as: 
     217 
     218  global: 
     219    user_agent: 
     220      cookies: /path/to/cookies.txt 
     221 
     222See L<Plagger::Cookies> for details. 
     223 
     224 
    210225=head1 AUTHOR 
    211226 
     
    218233=head1 SEE ALSO 
    219234 
    220 L<Plagger>, L<Plagger::Plugin::CustomFeed::Mixi>, L<WWW::Mechanize>, 
     235L<Plagger>, L<Plagger::Plugin::CustomFeed::Mixi>, L<Plagger::Mechanize>, 
    221236L<http://frepa.livedoor.com/> 
    222237 
  • branches/feature-server/plagger/lib/Plagger/Plugin/CustomFeed/GoogleNews.pm

    r557 r856  
    5454    $feed->link($args->{feed}->url); 
    5555 
    56     while ($content =~ m!<a href="(http://[^"]*)" id=r-\d[^>]*><b>([^<]*)</b></a>!g) { 
     56    while ($content =~ m!<a href="(http://[^"]*)" id=r-\d[^>]*>(.*?)</a>!g) { 
     57        my($link, $title) = ($1, $2); 
     58        $title =~ s!<b>(.*?)</b>!$1!g; 
     59 
    5760        my $entry = Plagger::Entry->new; 
    58         $entry->title($2); 
    59         $entry->link($1); 
     61        $entry->title($title); 
     62        $entry->link($link); 
     63 
    6064        $feed->add_entry($entry); 
    6165    } 
     
    7882      feed: 
    7983        - http://news.google.com/news?ned=jp&rec=0&topic=s 
     84        - http://news.google.co.jp/news?hl=ja&ned=jp&q=%E5%9B%B2%E7%A2%81 
    8085 
    8186  - module: CustomFeed::GoogleNews 
  • branches/feature-server/plagger/lib/Plagger/Plugin/CustomFeed/Mixi.pm

    r677 r856  
    5454sub load { 
    5555    my($self, $context) = @_; 
     56 
     57    my $cookie_jar = $self->cookie_jar; 
     58    if (ref($cookie_jar) ne 'HTTP::Cookies') { 
     59        # using foreign cookies = don't have to set email/password. Fake them 
     60        $self->conf->{email}    ||= 'plagger@localhost'; 
     61        $self->conf->{password} ||= 'pl4gg5r'; 
     62    } 
     63 
    5664    $self->{mixi} = WWW::Mixi->new($self->conf->{email}, $self->conf->{password}); 
    57     $self->{mixi}->cookie_jar($self->cache->cookie_jar); 
     65    $self->{mixi}->cookie_jar($cookie_jar); 
    5866 
    5967    my $feed = Plagger::Feed->new; 
     
    7987    if ($response->content =~ /action=login\.pl/) { 
    8088        $context->log(debug => "Cookie not found. Logging in"); 
     89 
     90        if ($self->conf->{email} eq 'plagger@localhost') { 
     91            $context->log(error => 'email/password should be set to login'); 
     92        } 
     93 
    8194        $response = $self->{mixi}->post("http://mixi.jp/login.pl", { 
    8295            next_url => $next_url, 
     
    214227Credential you need to login to mixi.jp. 
    215228 
     229Note that you don't have to supply email and password if you set 
     230global cookie_jar in your configuration file and the cookie_jar 
     231contains a valid login session there, such as: 
     232 
     233  global: 
     234    user_agent: 
     235      cookies: /path/to/cookies.txt 
     236 
     237See L<Plagger::Cookies> for details. 
     238 
    216239=item fetch_body 
    217240 
  • branches/feature-server/plagger/lib/Plagger/Plugin/CustomFeed/Simple.pm

    r736 r856  
    5050    my $re = $args->{match}; 
    5151 
     52    my %seen; 
    5253    my $parser = HTML::TokeParser->new(\$content); 
    5354    while (my $token = $parser->get_tag('a')) { 
     
    5556 
    5657        my $text = $parser->get_trimmed_text('/a'); 
     58        next if !$text || $text eq '[IMG]'; 
     59 
     60        my $url = URI->new_abs($token->[1]->{href}, $url); 
     61        next if $seen{$url->as_string}++; 
     62 
    5763        my $entry = Plagger::Entry->new; 
    5864        $entry->title($text); 
    59         $entry->link( URI->new_abs($token->[1]->{href}, $url) ); 
     65        $entry->link($url); 
    6066        $feed->add_entry($entry); 
    6167 
    62         $context->log(debug => "Add $token->[1]->{href}"); 
     68        $context->log(debug => "Add $token->[1]->{href} ($text)"); 
    6369    } 
    6470 
  • branches/feature-server/plagger/lib/Plagger/Plugin/CustomFeed/Yahoo360JP.pm

    r533 r856  
    66use Encode; 
    77use Time::HiRes; 
    8 use WWW::Mechanize; 
     8use Plagger::Mechanize; 
    99 
    1010sub plugin_id { 
     
    3535    my $start = "http://360.yahoo.co.jp/"; 
    3636 
    37     my $mech = WWW::Mechanize->new(cookie_jar => $self->cache->cookie_jar); 
    38     $mech->agent_alias( 'Windows IE 6' ); 
     37    my $mech = Plagger::Mechanize->new(cookie_jar => $self->cookie_jar); 
    3938    $mech->get($start); 
    4039 
     
    274273Your Yahoo! ID and password to login. 
    275274 
     275Note that you don't have to supply these variables if you set global 
     276cookie_jar in your configuration file and the cookie_jar contains a 
     277valid login session there, such as: 
     278 
     279  global: 
     280    user_agent: 
     281      cookies: /path/to/cookies.txt 
     282 
     283See L<Plagger::Cookies> for details. 
     284 
    276285=item fetch_body 
    277286 
     
    292301=head1 SEE ALSO 
    293302 
    294 L<Plagger>, L<WWW::Mechanize>, L<Plagger::Plugin::CustomFeed::Mixi> 
     303L<Plagger>, L<Plagger::Mechanize>, L<Plagger::Plugin::CustomFeed::Mixi> 
    295304 
    296305=cut 
  • branches/feature-server/plagger/lib/Plagger/Plugin/Filter/2chRSSContent.pm

    r696 r856  
    1717 
    1818    my $body = $args->{entry}->body; 
    19     if ($body =~ s!^([^:]*):(\d{4}/\d\d/\d\d)\(.*?\) (\d\d:\d\d:\d\d)\.\d\d (ID:\S+)  ?!!) { 
     19    if ($body =~ s!^([^:]*):(\d{4}/\d\d/\d\d)\(.*?\) (\d\d:\d\d:\d\d)(?:\.\d\d)? (ID:\S+)  ?!!) { 
    2020        my($from, $day, $time, $id) = ($1, $2, $3, $4); 
    2121 
  • branches/feature-server/plagger/lib/Plagger/Plugin/Filter/BreakEntriesToFeeds.pm

    r580 r856  
    1818    $feed->clear_entries; 
    1919    $feed->add_entry($args->{entry}); 
    20     $feed->title($args->{entry}->title); 
    2120 
    2221    push @{$self->{feeds}}, $feed; 
  • branches/feature-server/plagger/lib/Plagger/Plugin/Filter/EntryFullText.pm

    r689 r856  
    109109    } 
    110110 
    111     my $res = $self->{ua}->fetch( $args->{entry}->permalink, $self ); 
    112     return if $res->http_response->is_error; 
     111    # NoNetwork: don't connect for 3 hours 
     112    my $res = $self->{ua}->fetch( $args->{entry}->permalink, $self, { NoNetwork => 60 * 60 * 3 } ); 
     113    return if $res->status != URI::Fetch::URI_OK && $res->is_error; 
    113114 
    114115    $args->{content} = decode_content($res); 
    115116 
    116117    # if the request was redirected, set it as permalink 
    117     my $base = $res->http_response->request->uri; 
    118     if ( $base ne $args->{entry}->permalink ) { 
    119         $context->log(info => "rewrite permalink to $base"); 
    120         $args->{entry}->permalink($base); 
     118    if ($res->http_response) { 
     119        my $base = $res->http_response->request->uri; 
     120        if ( $base ne $args->{entry}->permalink ) { 
     121            $context->log(info => "rewrite permalink to $base"); 
     122            $args->{entry}->permalink($base); 
     123        } 
    121124    } 
    122125 
     
    134137                $args->{entry}->body($data->{body}); 
    135138                $args->{entry}->title($data->{title}) if $data->{title}; 
     139                $args->{entry}->icon({ url => $data->{icon} }) if $data->{icon}; 
    136140 
    137141                # extract date using found one, falls back to Last-Modified 
     
    178182    # decode as UTF-8 
    179183    for my $key ( qw(extract extract_date_format) ) { 
     184        next unless defined $data->{$key}; 
    180185        if (ref $data->{$key} && ref $data->{$key} eq 'ARRAY') { 
    181186            $data->{$key} = [ map decode("UTF-8", $_), @{$data->{$key}} ]; 
     
    223228        @{$data}{@capture} = @match; 
    224229 
     230        if ($self->{extract_after_hook}) { 
     231            eval $self->{extract_after_hook}; 
     232            Plagger->context->error($@) if $@; 
     233        } 
     234 
    225235        if ($data->{date}) { 
    226236            if (my $format = $self->{extract_date_format}) { 
    227237                $format = [ $format ] unless ref $format; 
    228238                $data->{date} = (map { Plagger::Date->strptime($_, $data->{date}) } @$format)[0]; 
     239                if ($data->{date} && $self->{extract_date_timezone}) { 
     240                    $data->{date}->set_time_zone($self->{extract_date_timezone}); 
     241                } 
    229242            } else { 
    230243                $data->{date} = Plagger::Date->parse_dwim($data->{date}); 
    231244            } 
    232         } 
    233  
    234         if ($self->{extract_after_hook}) { 
    235             eval $self->{extract_after_hook}; 
    236             Plagger->context->error($@) if $@; 
    237245        } 
    238246 
  • branches/feature-server/plagger/lib/Plagger/Plugin/Filter/FeedBurnerPermalink.pm

    r400 r856  
    1515    my($self, $context, $args) = @_; 
    1616 
     17    my $fbns = 'http://rssnamespace.org/feedburner/ext/1.0'; 
     18 
    1719    # RSS 1.0 & 2.0 
    18     if (my $orig_link = $args->{orig_entry}->{entry}->{'http://rssnamespace.org/feedburner/ext/1.0'}->{origLink}) { 
    19         $args->{entry}->permalink($orig_link); 
    20         $context->log(info => "Permalink rewritten to $orig_link"); 
     20    if ($args->{orig_entry}->isa('XML::Feed::Entry::RSS')) { 
     21        if (my $orig_link = $args->{orig_entry}->{entry}->{$fbns}->{origLink}) { 
     22            $args->{entry}->permalink($orig_link); 
     23            $context->log(info => "Permalink rewritten to $orig_link"); 
     24        } 
     25    } 
     26    # Atom 1.0 
     27    elsif ($args->{orig_entry}->isa('XML::Feed::Entry::Atom')) { 
     28        my $ns = XML::Atom::Namespace->new(feedburner => $fbns); 
     29        if (my $orig_link = $args->{orig_entry}->{entry}->get($ns, 'origLink')) { 
     30            $args->{entry}->permalink($orig_link); 
     31            $context->log(info => "Permalink rewritten to $orig_link"); 
     32        } 
    2133    } 
    2234} 
  • branches/feature-server/plagger/lib/Plagger/Plugin/Filter/RSSTimeZoneString.pm

    r343 r856  
    55550.91) feeds to a correct one. 
    5656 
    57 Namely, when you create RSS feeds with POSIX C<strftime> function for 
     57Namely, when you create RSS feeds with POSIX C<ctime> function for 
    5858example, it'll create a following pubDate format if you're on the box 
    5959under Japanese standard time: 
     
    6161  Fri, 03 Mar 2006 03:52:42 JST 
    6262 
    63 which is invalid in RFC 822. (RFC 822 only allows timezone strings for 
    64 North America, like PST and CST). 
     63which is B<invalid> in RFC 822. (RFC 822 only allows timezone strings 
     64for North America, like PST and CST). 
    6565 
    6666This plugin fixes the string to: 
    6767 
    6868  Fri, 03 Mar 2006 03:52:42 +0900 
    69  
    70 and the correct one is re-parsed and set to C<< $entry->date >>. 
    7169 
    7270=head1 AUTHOR 
     
    7674=head1 SEE ALSO 
    7775 
    78 L<Plagger>, L<DateTime::Format::Mail>, L<Time::Zone> 
     76L<Plagger>, L<DateTime::Format::Mail>, L<Time::Zone>, L<Plagger::Plugin::Filter::RSSLiberalDateTime> 
    7977 
    8078=cut 
  • branches/feature-server/plagger/lib/Plagger/Plugin/Filter/Regexp.pm

    r415 r856  
    22use strict; 
    33use base qw( Plagger::Plugin::Filter::Base ); 
     4use Encode; 
    45 
    56sub init { 
     
    1718 
    1819    local $_ = $body; 
    19     my $count = eval $self->conf->{regexp}; 
     20    my $regexp = decode_utf8($self->conf->{regexp}, Encode::FB_CROAK); 
     21    my $count = eval $regexp; 
    2022 
    2123    if ($@) { 
  • branches/feature-server/plagger/lib/Plagger/Plugin/Filter/SpamAssassin.pm

    r374 r856  
    33use base qw( Plagger::Plugin ); 
    44 
    5 our $VERSION = '0.01'; 
     5our $VERSION = '0.02'; 
    66 
    77use Mail::SpamAssassin; 
     8use MIME::Lite; 
     9use Encode; 
     10use Encode::MIME::Header; 
    811 
    912sub register { 
     
    1114    $context->register_hook( 
    1215        $self, 
    13         'plugin.init' => \&init_spamassassin, 
     16        'plugin.init'        => \&init_spamassassin, 
    1417        'update.entry.fixup' => \&filter, 
    1518    ); 
     
    2831    my $sa    = $self->{spamassassin}; 
    2932    my $entry = $args->{entry}; 
    30     my $tag   = $self->conf->{spam_tag} || 'SPAM'; 
     33    my $tag   = $self->conf->{spam_tag} || 'spam'; 
    3134 
    3235    # create a pseudo mail header to skip some of the sa's default tests 
    3336    my $status = $sa->check_message_text( 
    34         join "\n", 'Subject: ' . $entry->title, "\n", $entry->body 
     37        MIME::Lite->new( 
     38            From    => 'plagger@localhost', 
     39            To      => 'plagger@localhost', 
     40            Subject => encode('MIME-Header', $entry->title_text), 
     41            Data    => $entry->body_text, 
     42        )->as_string 
    3543    ); 
    3644 
    3745    if ($status->is_spam) { 
    3846        $context->log(debug => "spam found"); 
    39  
    40         $entry->title("[$tag] " . $entry->title) if $self->conf->{add_tag_to_title}; 
    4147        $entry->body($entry->body . $status->get_report) if $self->conf->{add_report}; 
    4248        $entry->add_tag($tag); 
     
    5258=head1 NAME 
    5359 
    54 Plagger::Plugin::Filter::SpamAssassin - mark spam
     60Plagger::Plugin::Filter::SpamAssassin - Find spam entrie
    5561 
    5662=head1 SYNOPSIS 
    5763 
    58   - module: SmartFeed::SpamAssassin 
     64  - module: Filter::SpamAssassin 
    5965    config: 
    60       spam_tag: SPAM 
    61       add_tag_to_title: 1 
    62       add_report: 0 
     66      spam_tag: spam 
    6367      new: 
    6468        local_tests_only: 1 
    6569        config_text: 
    66           - score MISSING_SUBJECT 0.0 
    67           - score MISSING_HB_SEP  0.0 
    68           - score MISSING_HEADERS 0.0 
    69           - score EMPTY_MESSAGE   0.0 
    7070          - score NO_RELAYS       0.0 
    7171          - score NO_RECEIVED     0.0 
    72           - score TO_CC_NONE      0.0 
    7372 
    7473=head1 CONFIG 
     
    7877=item spam_tag 
    7978 
    80 Specifies a tag string that will be added to entry's title or 
    81 tag (category) 
     79A string that will be added to the entry's tag. Defaults to 'spam'. 
    8280 
    83 =item add_tag_to_title 
    84  
    85 If set to true, the tag (enclosed in brackets) will be added to spam 
    86 entry's title. 
    87  
    88 =item add_report 
     81=item add_report (for debugging) 
    8982 
    9083If set to true, the SpamAssassin's report will be added to spam  
  • branches/feature-server/plagger/lib/Plagger/Plugin/Filter/StripRSSAd.pm

    r604 r856  
    1717    my $dir = $self->assets_dir; 
    1818    my $dh = DirHandle->new($dir) or Plagger->context->error("$dir: $!"); 
    19     for my $file (grep -f $_->[0] && $_->[1] =~ /^[\w\-]+$/, 
     19    for my $file (grep -f $_->[0] && $_->[1] =~ /^[\w\-\.]+$/, 
    2020                  map [ File::Spec->catfile($dir, $_), $_ ], sort $dh->read) { 
    2121        $self->load_pattern(@$file); 
     
    2828    Plagger->context->log(debug => "loading $file"); 
    2929 
     30    if ($file =~ /\.yaml$/) { 
     31        $self->load_yaml($file, $base); 
     32    } else { 
     33        $self->load_regexp($file, $base); 
     34    } 
     35} 
     36 
     37sub load_regexp { 
     38    my($self, $file, $base) = @_; 
     39 
    3040    open my $fh, $file or Plagger->context->error("$file: $!"); 
    3141    my $re = join '', <$fh>; 
     
    3343 
    3444    push @{$self->{pattern}}, { site => $base, re => qr/$re/ }; 
     45} 
     46 
     47sub load_yaml { 
     48    my($self, $file, $base) = @_; 
     49 
     50    my $pattern = eval { YAML::LoadFile($file) } 
     51        or Plagger->context->error("$file: $@"); 
     52 
     53    push @{$self->{pattern}}, { site => $base, %$pattern }; 
    3554} 
    3655 
     
    4564sub update { 
    4665    my($self, $context, $args) = @_; 
    47     my $body = $self->filter($args->{entry}->body, $args->{entry}->link); 
    48     $args->{entry}->body($body); 
    49 
    50  
    51 sub filter { 
    52     my($self, $body, $link) = @_; 
     66    my $body = $args->{entry}->body; 
    5367 
    5468    for my $pattern (@{ $self->{pattern} }) { 
    55         my $re = $pattern->{re}; 
    56         if (my $count = $body =~ s!$re!defined($1) ? $1 : ''!egs) { 
    57             Plagger->context->log(debug => "Stripped $pattern->{site} Ad on $link"); 
     69        if (my $re = $pattern->{re}) { 
     70            if (my $count = $body =~ s!$re!defined($1) ? $1 : ''!egs) { 
     71                Plagger->context->log(info => "Stripped $pattern->{site} Ad on " . $args->{entry}->link); 
     72            } 
     73        } elsif (my $cond = $pattern->{condition}) { 
     74            local $args->{body} = $body; 
     75            if (eval $cond && $pattern->{strip}) { 
     76                $args->{feed}->delete_entry($args->{entry}); 
     77                Plagger->context->log(info => "Stripped Ad entry " . $args->{entry}->link); 
     78            } elsif ($@) { 
     79                Plagger->context->log(error => "Error evaluating $cond: $@"); 
     80            } 
    5881        } 
    5982    } 
    6083 
    61     $body
     84    $args->{entry}->body($body)
    6285} 
    6386 
  • branches/feature-server/plagger/lib/Plagger/Plugin/Filter/TruePermalink.pm

    r614 r856  
    55use DirHandle; 
    66use YAML; 
     7use Plagger::UserAgent; 
    78use URI; 
    89use URI::QueryParam; 
     
    2930 
    3031    Plagger->context->log(debug => "loading $file"); 
    31     push @{$self->{plugins}}, YAML::LoadFile($file); 
     32    my $data = YAML::LoadFile($file); 
     33    if (ref($data) eq 'ARRAY') { 
     34        push @{$self->{redirectors}}, { follow_link => "^(?:" . join("|", @$data) . ")" }; 
     35    } else { 
     36        push @{$self->{plugins}}, $data; 
     37    } 
    3238} 
    3339 
     
    4349    my($self, $context, $args) = @_; 
    4450 
    45     my $orig = $args->{entry}->permalink; 
     51    $self->rewrite(sub { $args->{entry}->link }, sub { $args->{entry}->link(@_) }); 
     52    for my $enclosure ($args->{entry}->enclosures) { 
     53        $self->rewrite(sub { $enclosure->url }, sub { $enclosure->url( URI->new(@_) ) }); 
     54    } 
     55
     56 
     57sub rewrite { 
     58    my($self, $getter, $callback) = @_; 
     59 
     60    my $loop; 
     61    while ($self->rewrite_link($getter, $callback)) { 
     62        if ($loop++ >= 100) { 
     63            Plagger->error("Possible infinite loop on " . $getter->()); 
     64        } 
     65    } 
     66
     67 
     68sub rewrite_link { 
     69    my($self, $getter, $callback) = @_; 
     70 
     71    my $context = Plagger->context; 
     72 
     73    my $link = $getter->(); 
     74    my $orig = $link; # copy 
    4675    my $count = 0; 
     76    my $rewritten; 
    4777 
    4878    for my $plugin (@{ $self->{plugins}}) { 
    4979        my $match = $plugin->{match} || '.'; # anything 
    50         next unless $args->{entry}->permalink =~ m/$match/i; 
     80        next unless $link =~ m/$match/i; 
    5181 
    5282        if ($plugin->{rewrite}) { 
    53             local $_ = $args->{entry}->permalink; 
    54             $count += eval $plugin->{rewrite}; 
     83            local $_ = $link; 
     84            my $done = eval $plugin->{rewrite}; 
    5585            if ($@) { 
    5686                $context->error("$@ in $plugin->{rewrite}"); 
     87            } elsif ($done) { 
     88                $count += $done; 
     89                $rewritten = $_; 
     90                last; 
    5791            } 
    58             $args->{entry}->link($_); 
    5992        } elsif ($plugin->{query_param}) { 
    60             my $link = URI->new($args->{entry}->permalink)->query_param($plugin->{query_param}) 
    61                 or $context->error("No query param $plugin->{query_param} in " . $args->{entry}->permalink); 
    62             $args->{entry}->link($link); 
     93            my $param = URI->new($link)->query_param($plugin->{query_param}) 
     94                or $context->error("No query param $plugin->{query_param} in " . $link); 
    6395            $count++; 
     96            $rewritten = $param; 
     97            last; 
    6498        } 
    6599    } 
    66100 
     101    unless ($count) { 
     102        for my $red (@{ $self->{redirectors} }) { 
     103            next unless $red->{follow_link}; 
     104            if ($link =~ /$red->{follow_link}/i) { 
     105                my $url = $self->follow_redirect($link); 
     106                if ($url && $url ne $link) { 
     107                    $count++; 
     108                    $rewritten = $url; 
     109                    last; 
     110                } 
     111            } 
     112        } 
     113    } 
     114 
    67115    if ($count) { 
    68         $context->log(info => "Permalink $orig rewritten to " . $args->{entry}->permalink); 
    69     } 
     116        $callback->($rewritten); 
     117        $context->log(info => "Link $orig rewritten to $rewritten"); 
     118    } 
     119 
     120    return $count; 
     121
     122 
     123sub follow_redirect { 
     124    my($self, $link) = @_; 
     125 
     126    my $url = $self->cache->get_callback( 
     127        "redirector:$link", 
     128        sub { 
     129            my $ua  = Plagger::UserAgent->new; 
     130            my $res = $ua->simple_request( HTTP::Request->new(GET => $link) ); 
     131            if ($res->is_redirect) { 
     132                return $res->header('Location'); 
     133            } 
     134            return; 
     135        }, 
     136        '1 day', 
     137    ); 
     138 
     139    Plagger->context->log(debug => "Resolving redirection of $link: $url") if $url; 
     140 
     141    return $url; 
    70142} 
    71143 
     
    89161this plugin. 
    90162 
    91 This plugin rewrites I<link> attribute of C<$entry>, rather than I<permalink>. 
     163This plugin rewrites I<link> attribute of C<$entry>, rather than 
     164I<permalink>. If C<$entry> has enclosures, this plugin also tries to 
     165rewrite url of them. 
    92166 
    93167=head1 PATTERN FILES 
  • branches/feature-server/plagger/lib/Plagger/Plugin/Notify/Campfire.pm

    r447 r856  
    4343 
    4444use strict; 
    45 use WWW::Mechanize; 
     45use Plagger::Mechanize; 
    4646use HTTP::Request::Common; 
    4747use Encode; 
     
    5151    my $plugin = shift; 
    5252 
    53     my $mech = WWW::Mechanize->new(cookie_jar => $plugin->cache->cookie_jar); 
     53    my $mech = Plagger::Mechanize->new(cookie_jar => $plugin->cookie_jar); 
    5454    $mech->agent_alias("Windows IE 6"); 
    5555 
     
    144144L<http://www.campfirenow.com/> chat room. 
    145145 
     146Note that you don't have to supply emali and password if you set 
     147global cookie_jar in your configuration file and the cookie_jar 
     148contains a valid login session there, such as: 
     149 
     150  global: 
     151    user_agent: 
     152      cookies: /path/to/cookies.txt 
     153 
     154See L<Plagger::Cookies> for details. 
     155 
    146156=head1 AUTHOR 
    147157 
  • branches/feature-server/plagger/lib/Plagger/Plugin/Publish/2chdat.pm

    r692 r856  
    3030 
    3131    my $feed = $args->{feed}; 
    32     my $out  = File::Spec->catfile($self->conf->{dir}, 'dat', $self->safe_id($feed->id) . ".dat"); 
     32    my $out  = File::Spec->catfile($self->conf->{dir}, 'dat', $feed->id_safe . ".dat"); 
    3333    $context->log(info => "Writing dat output to $out"); 
    3434 
     
    3939        ($feed->author || $feed->entries->[0]->author || $anonymous), 
    4040        $self->format_date( Plagger::Date->from_epoch(0) ), # Fix created date to handle bytes-range request 
    41         substr($self->safe_id($feed->id), 0, 8), 
     41        substr($feed->id_safe), 0, 8), 
    4242        $self->format_body($feed->description) . "<BR>" . $feed->link, 
    4343        $feed->title; 
     
    6565    open my $fh, ">:encoding(shift_jis)", $out or $context->erorr("$out: $!"); 
    6666    for my $feed ($context->update->feeds) { 
    67         printf $fh "%s.dat<>%s (%d)\n", $self->safe_id($feed->id), $feed->title, $feed->count; 
     67        printf $fh "%s.dat<>%s (%d)\n", $feed->id_safe, $feed->title, $feed->count; 
    6868    } 
    69 } 
    70  
    71 sub safe_id { 
    72     my($self, $id) = @_; 
    73     $id =~ s![^\w\s]+!_!g; 
    74     $id =~ s!\s+!_!g; 
    75     $id; 
    7669} 
    7770 
  • branches/feature-server/plagger/lib/Plagger/Plugin/Publish/Feed.pm

    r355 r856  
    5858        $entry->issued($e->date) if $e->date; 
    5959        $entry->author($e->author); 
     60 
     61        if ($e->has_enclosure) { 
     62            # RSS 2.0 by spec doesn't allow multiple enclosures 
     63            my @enclosures = $feed_format eq 'RSS' ? ($e->enclosures->[0]) : $e->enclosures; 
     64            for my $enclosure (grep { defined $_->url && !$_->is_inline } @enclosures) { 
     65                $entry->add_enclosure({ 
     66                    url    => $enclosure->url, 
     67                    length => $enclosure->length, 
     68                    type   => $enclosure->type, 
     69                }); 
     70            } 
     71        } 
     72 
    6073        $feed->add_entry($entry); 
    6174    } 
     
    6477    my $filepath = File::Spec->catfile($self->conf->{dir}, $self->gen_filename($f)); 
    6578 
    66     $context->log(info => "save feed for " . $f->url . " to $filepath"); 
     79    $context->log(info => "save feed for " . $f->link . " to $filepath"); 
    6780 
    6881    my $xml = $feed->as_xml; 
     
    100113} 
    101114 
     115# XXX okay, this is a hack until XML::Feed is updated 
     116*XML::Feed::Entry::Atom::add_enclosure = sub { 
     117    my($entry, $enclosure) = @_; 
     118    my $link = XML::Atom::Link->new; 
     119    $link->rel('enclosure'); 
     120    $link->type($enclosure->{type}); 
     121    $link->href($enclosure->{url}); 
     122    $link->length($enclosure->{length}); 
     123    $entry->{entry}->add_link($link); 
     124}; 
     125 
     126*XML::Feed::Entry::RSS::add_enclosure = sub { 
     127    my($entry, $enclosure) = @_; 
     128    $entry->{entry}->{enclosure} = { 
     129        url    => $enclosure->{url}, 
     130        type   => $enclosure->{type}, 
     131        length => $enclosure->{length}, 
     132    }; 
     133}; 
     134 
     135 
    1021361; 
    103137 
     
    157191Yoshiki KURIHARA 
    158192 
     193Tatsuhiko Miyagawa 
     194 
     195Gosuke Miyashita 
     196 
    159197=head1 SEE ALSO 
    160198 
  • branches/feature-server/plagger/lib/Plagger/Plugin/Publish/Gmail.pm

    r450 r856  
    99use Encode; 
    1010use Encode::MIME::Header; 
     11use HTML::Entities; 
     12use HTML::Parser; 
    1113use MIME::Lite; 
    1214 
     
    2224        'publish.feed' => \&notify, 
    2325    ); 
     26} 
     27 
     28sub init { 
     29    my $self = shift; 
     30    $self->SUPER::init(@_); 
     31 
     32    $self->conf->{mailto} or Plagger->context->error("mailto is required"); 
     33    $self->conf->{mailfrom} ||= 'plagger@localhost'; 
    2434} 
    2535 
     
    4555    my $feed = $args->{feed}; 
    4656    my $subject = $feed->title || '(no-title)'; 
     57 
     58    my @enclosure_cb; 
     59    if ($self->conf->{attach_enclosures}) { 
     60        for my $entry ($args->{feed}->entries) { 
     61            push @enclosure_cb, $self->prepare_enclosures($entry); 
     62        } 
     63    } 
     64 
    4765    my $body = $self->templatize($context, $feed); 
    4866 
     
    6785        Type => 'text/html; charset=utf-8', 
    6886        Data => encode("utf-8", $body), 
     87        Encoding => 'quoted-printable', 
    6988    ); 
    7089 
     90    for my $cb (@enclosure_cb) { 
     91        $cb->($msg); 
     92    } 
     93 
    7194    my $route = $cfg->{mailroute} || { via => 'smtp', host => 'localhost' }; 
     95    $route->{via} ||= 'smtp'; 
     96 
    7297    if ($route->{via} eq 'smtp_tls') { 
    7398        $self->{tls_args} = [ 
     
    88113} 
    89114 
     115sub prepare_enclosures { 
     116    my($self, $entry) = @_; 
     117 
     118    if (grep $_->is_inline, $entry->enclosures) { 
     119        # replace inline enclosures to cid: entities 
     120        my %url2enclosure = map { $_->url => $_ } $entry->enclosures; 
     121 
     122        my $output; 
     123        my $p = HTML::Parser->new(api_version => 3); 
     124        $p->handler( default => sub { $output .= $_[0] }, "text" ); 
     125        $p->handler( start => sub { 
     126                         my($tag, $attr, $attrseq, $text) = @_; 
     127                         # TODO: use HTML::Tagset? 
     128                         if (my $url = $attr->{src}) { 
     129                             if (my $enclosure = $url2enclosure{$url}) { 
     130                                 $attr->{src} = "cid:" . $self->enclosure_id($enclosure); 
     131                             } 
     132                             $output .= $self->generate_tag($tag, $attr, $attrseq); 
     133                         } else { 
     134                             $output .= $text; 
     135                         } 
     136                     }, "tag, attr, attrseq, text"); 
     137        $p->parse($entry->body); 
     138        $p->eof; 
     139 
     140        $entry->body($output); 
     141    } 
     142 
     143    return sub { 
     144        my $msg = shift; 
     145 
     146        for my $enclosure (grep $_->local_path, $entry->enclosures) { 
     147            my %param = ( 
     148                Type => $enclosure->type, 
     149                Path => $enclosure->local_path, 
     150                Filename => $enclosure->filename, 
     151            ); 
     152 
     153            if ($enclosure->is_inline) { 
     154                $param{Id} = '<' . $self->enclosure_id($enclosure) . '>'; 
     155                $param{Disposition} = 'inline'; 
     156            } else { 
     157                $param{Disposition} = 'attachment'; 
     158            } 
     159 
     160            $msg->attach(%param); 
     161        } 
     162    } 
     163} 
     164 
     165sub generate_tag { 
     166    my($self, $tag, $attr, $attrseq) = @_; 
     167 
     168    return "<$tag " . 
     169        join(' ', map { $_ eq '/' ? '/' : sprintf qq(%s="%s"), $_, encode_entities($attr->{$_}, q(<>"')) } @$attrseq) . 
     170        '>'; 
     171} 
     172 
     173sub enclosure_id { 
     174    my($self, $enclosure) = @_; 
     175    return Digest::MD5::md5_hex($enclosure->url->as_string) . '@Plagger'; 
     176} 
     177 
    90178sub templatize { 
    91179    my($self, $context, $feed) = @_; 
     
    158246 
    1592471; 
     248 
     249__END__ 
     250 
     251=head1 NAME 
     252 
     253Plagger::Plugin::Publish::Gmail - Notify updates to your email account 
     254 
     255=head1 SYNOPSIS 
     256 
     257  - module: Publish::Gmail 
     258    config: 
     259      mailto: example@gmail.com 
     260      mailfrom: you@example.net 
     261 
     262=head1 DESCRIPTION 
     263 
     264This plugin creates HTML emails and sends them to your Gmail mailbox. 
     265 
     266=head1 CONFIG 
     267 
     268=over 4 
     269 
     270=item mailto 
     271 
     272Your email address to send updatess to. Required. 
     273 
     274=item mailfrom 
     275 
     276Email address to send email from. Defaults to I<plagger@localhost>. 
     277 
     278=item mailroute 
     279 
     280Hash to specify how to send emails. Defaults to: 
     281 
     282  mailroute: 
     283    via: smtp 
     284    host: localhost 
     285 
     286the value of I<via> would be either I<smtp>, I<smtp_tls> or I<sendmail>. 
     287 
     288  mailroute: 
     289    via: sendmail 
     290    command: /usr/sbin/sendmail 
     291 
     292=item attach_enclosures 
     293 
     294Flag to attach enclosures as Email attachments. Defaults to 0. 
     295 
     296=back 
     297 
     298=head1 AUTHOR 
     299 
     300Tatsuhiko Miyagawa 
     301 
     302=head1 SEE ALSO 
     303 
     304L<Plagger>, L<MIME::Lite> 
     305 
     306=cut 
  • branches/feature-server/plagger/lib/Plagger/Plugin/Publish/HatenaBookmark.pm

    r250 r856  
    1616    ); 
    1717} 
     18 
     19sub rule_hook { 'publish.entry.fixup' } 
    1820 
    1921sub initialize { 
  • branches/feature-server/plagger/lib/Plagger/Plugin/Publish/IMAP.pm

    r561 r856  
    7474    Type => 'text/html; charset=utf-8', 
    7575    Data => $body, 
     76    Encoding => 'quoted-printable', 
    7677  ); 
    7778  $msg->add('X-Tags', encode('MIME-Header',join(' ',@{$entry->tags}))); 
  • branches/feature-server/plagger/lib/Plagger/Plugin/Publish/MT.pm

    r319 r856  
    2525    $self->{mt}->username($self->conf->{username}); 
    2626    $self->{mt}->password($self->conf->{password}); 
    27     $self->{mt}->blogId($self->{blog_id} || 1); 
     27    $self->{mt}->blogId($self->conf->{blog_id} || 1); 
    2828    return $self->{mt}; 
    2929} 
  • branches/feature-server/plagger/lib/Plagger/Plugin/Publish/Maildir.pm

    r561 r856  
    7979    Type => 'text/html; charset=utf-8', 
    8080    Data => $body, 
     81    Encoding => 'quoted-printable', 
    8182  ); 
    8283  $msg->add('X-Tags', encode('MIME-Header',join(' ',@{$entry->tags}))); 
  • branches/feature-server/plagger/lib/Plagger/Plugin/Subscription/Bloglines.pm

    r585 r856  
    117117    # catch bad XML feed by Bloglines 
    118118    eval { 
    119         @updates = $self->{bloglines}->getitems(0, $mark_read); 
     119        @updates = $self->{bloglines}->getitems(0, 0); 
    120120    }; 
    121121 
     
    129129            } 
    130130        } 
     131    } elsif ($mark_read) { 
     132        # no error found with XML ... call the API again to mark read 
     133        eval { 
     134            @updates = $self->{bloglines}->getitems(0, $mark_read); 
     135        }; 
    131136    } 
    132137 
     
    166171 
    167172            if ($item->{guid}) { 
    168                 my $is_permalink = $item->{guid}->{isPermaLink}; 
     173                my $is_permalink = eval { $item->{guid}->{isPermaLink}  } || 
     174                    'false'; 
    169175                my $guid_url     = "$item->{guid}"; # stringify MagicElement 
    170176                $entry->permalink($guid_url) 
     
    175181            $entry->id($item->{guid}); 
    176182            $entry->body($item->{description}); 
     183 
     184            if ($item->{enclosure}) { 
     185                my $enclosure = Plagger::Enclosure->new; 
     186                $enclosure->url( URI->new($item->{enclosure}->{url}) ); 
     187                $enclosure->length($item->{enclosure}->{length}); 
     188                $enclosure->auto_set_type($item->{enclosure}->{type}); 
     189                $entry->add_enclosure($enclosure); 
     190            } 
    177191 
    178192            $feed->add_entry($entry); 
  • branches/feature-server/plagger/lib/Plagger/Plugin/Subscription/HatenaRSS.pm

    r117 r856  
    33use base qw( Plagger::Plugin::Subscription::OPML ); 
    44 
    5 use WWW::Mechanize; 
     5use Plagger::Mechanize; 
    66 
    77sub register { 
     
    1818 
    1919    my $username = $self->conf->{username} 
    20         or $context->error('username is missing'); 
     20        or $context->error("username is missing"); 
    2121 
    22     my $start = "https://www.hatena.ne.jp/login?backurl=http%3A%2F%2Fr.hatena.ne.jp%2F"; 
     22    my $mech = Plagger::Mechanize->new(cookie_jar => $self->cookie_jar); 
     23    $mech->get("http://r.hatena.ne.jp/$username/opml"); 
    2324 
    24     # TODO: we should save the cookie and reuse 
    25     my $mech = WWW::Mechanize->new; 
    26     $mech->get($start); 
     25    if ($mech->content !~ /<opml version/) { 
     26        $mech->get("https://www.hatena.ne.jp/login?backurl=http%3A%2F%2Fr.hatena.ne.jp%2F"); 
     27        $mech->submit_form( 
     28            fields => { 
     29                key      => $username, 
     30                password => $self->conf->{password}, 
     31            }, 
     32        ); 
    2733 
    28     $mech->submit_form( 
    29         fields => { 
    30             key      => $username, 
    31             password => $self->conf->{password}, 
    32         }, 
    33     ); 
    34  
    35     if ( $mech->content =~ m!<div class="error">! ) { 
    36         $context->log(error => "Login to HatenaRSS failed."); 
    37         return; 
     34        if ( $mech->content =~ m!<div class="error">! ) { 
     35            $context->log(error => "Login to HatenaRSS failed."); 
     36            return; 
     37        } 
    3838    } 
    3939 
    4040    $context->log(info => "Login to HatenaRSS succeed."); 
    41  
    42     $mech->get("http://r.hatena.ne.jp/$username/config"); 
    43     $mech->submit_form(form_name => 'opmlexport'); 
    4441 
    4542    my $opml = $mech->content; 
     
    6259    config: 
    6360      username: example 
    64       password: xxxxxxxx 
    6561 
    6662=head1 DESCRIPTION 
    6763 
    6864This plugin creates Subscription by fetching Hatena RSS 
    69 L<http://r.hatena.ne.jp> OPML by HTTP. Since Hatena RSS OPML export 
    70 requires login state, it uses WWW::Mechanize module to emulate the 
    71 browser's login authentication procedure. 
     65L<http://r.hatena.ne.jp> OPML by HTTP. 
     66 
     67If your OPML is shared public (which is default), you don't have to 
     68pass password to the config. Also, even if you OPML is private, you 
     69can share Cookies with your favorite browser like Firefox, using 
     70 
     71  global: 
     72    user_agent: 
     73      cookies: /path/to/cookies.txt 
     74 
     75so that you don't have to pass password to the config, again. 
    7276 
    7377=head1 AUTHOR 
  • branches/feature-server/plagger/lib/Plagger/Plugin/Subscription/LivedoorReader.pm

    r639 r856  
    55use JSON::Syck; 
    66use URI; 
    7 use URI::QueryParam; 
    8 use WWW::Mechanize; 
     7use Plagger::Mechanize; 
    98use Plagger::Util; 
    109 
     
    2625sub init_reader { 
    2726    my $self = shift; 
    28     $self->{mech} = WWW::Mechanize->new(cookie_jar => $self->cache->cookie_jar); 
     27    $self->{mech} = Plagger::Mechanize->new(cookie_jar => $self->cookie_jar); 
    2928 
    3029    unless (defined($self->conf->{username}) && defined($self->conf->{password})) { 
     
    130129        } 
    131130    } 
     131 
     132    $self->{mech}->cookie_jar->scan( 
     133        sub { 
     134            my($key, $val) = @_[1,2]; 
     135            if ($key =~ /_sid/) { 
     136                $self->{apikey} = $val; 
     137                return; 
     138            } 
     139        }, 
     140    ); 
    132141} 
    133142 
     
    136145 
    137146    my $uri = URI->new_abs($method, "http://reader.livedoor.com/"); 
    138     $uri->query_param(%$param) if $param
     147    $uri->query_form(%$param, ApiKey => $self->{apikey})
    139148 
    140149    $self->{mech}->get($uri->as_string); 
     
    172181Your username & password to use with livedoor Reader. 
    173182 
     183Note that you don't have to supply username and password if you set 
     184global cookie_jar in your configuration file and the cookie_jar 
     185contains a valid login session there, such as: 
     186 
     187  global: 
     188    user_agent: 
     189      cookies: /path/to/cookies.txt 
     190 
     191See L<Plagger::Cookies> for details. 
     192 
    174193=item mark_read 
    175194 
  • branches/feature-server/plagger/lib/Plagger/Plugin/Subscription/Odeo.pm

    r117 r856  
    33use base qw( Plagger::Plugin::Subscription::OPML ); 
    44 
     5use Plagger::UserAgent; 
    56use URI::Escape; 
    67use HTML::Entities; 
  • branches/feature-server/plagger/lib/Plagger/Plugin/Subscription/XOXO.pm

    r687 r856  
    11package Plagger::Plugin::Subscription::XOXO; 
    22use strict; 
    3 use base qw( Plagger::Plugin ); 
    4  
    5 use HTML::TreeBuilder::XPath; 
    6 use Plagger::Util; 
    7 use URI; 
    8  
    9 sub register { 
    10     my($self, $context) = @_; 
    11  
    12     $context->register_hook( 
    13         $self, 
    14         'subscription.load' => \&load, 
    15     ); 
    16 
     3use base qw( Plagger::Plugin::Subscription::XPath ); 
    174 
    185sub load { 
    196    my($self, $context) = @_; 
    20     my $uri = URI->new($self->conf->{url}) 
    21         or $context->error("config 'url' is missing"); 
    227 
    23     $self->load_xoxo($context, $uri); 
    24 
    25  
    26 sub load_xoxo { 
    27     my($self, $context, $uri) = @_; 
    28  
    29     my $xhtml = Plagger::Util::load_uri($uri, $self); 
    30     my $tree = HTML::TreeBuilder::XPath->new; 
    31     $tree->parse($xhtml); 
    32     $tree->eof; 
    33  
    34     $self->find_xoxo($tree); 
    35 
    36  
    37 sub find_xoxo { 
    38     my($self, $tree) = @_; 
    39  
    40     for my $child ($tree->findnodes('//ul[@class="xoxo" or @class="subscriptionlist"]//a')) { 
    41         my $href  = $child->attr('href') or next; 
    42         my $title = $child->attr('title') || $child->as_text; 
    43  
    44         my $feed = Plagger::Feed->new; 
    45         $feed->url($href); 
    46         $feed->title($title); 
    47  
    48         Plagger->context->subscription->add($feed); 
    49     } 
     8    $self->conf->{xpath} = '//ul[@class="xoxo" or @class="subscriptionlist"]//a'; 
     9    $self->SUPER::load($context); 
    5010} 
    5111 
  • branches/feature-server/plagger/lib/Plagger/Plugin/Widget/Delicious.pm

    r73 r856  
    2323    my($self, $entry) = @_; 
    2424    my $uri = URI->new('http://del.icio.us/' . $self->conf->{username}); 
    25     $uri->query_form( 
    26         v => 3, 
    27         url => $entry->permalink, 
    28         title => encode('utf-8', $entry->title), 
    29     ); 
     25    my %query; 
     26    $query{url}         = $entry->permalink; 
     27    $query{description} = encode('utf-8', $entry->title); 
     28    $query{tags}        = $self->conf->{tags} if $self->conf->{tags}; 
     29    $query{jump}        = 'doclose' if $self->conf->{one_click_post}; 
     30 
     31    $uri->query_form(%query); 
    3032 
    3133    my $url = HTML::Entities::encode($uri->as_string); 
     
    3436 
    35371; 
     38 
     39__END__ 
     40 
     41=head1 NAME 
     42 
     43Plagger::Plugin::Widget::Delicious - Widget to post to del.icio.us 
     44 
     45=head1 SYNOPSIS 
     46 
     47   module: Widget::Delicious 
     48   config: 
     49     username: miyagawa 
     50 
     51=head1 DESCRIPTION 
     52 
     53This plugin creates a widget to post to del.icio.us in the Publish 
     54modules output. 
     55 
     56=head1 CONFIG 
     57 
     58=over 4 
     59 
     60=item username 
     61 
     62Your del.icio.us username. Required. 
     63 
     64=item tags 
     65 
     66Preset tags to tag the post. 
     67 
     68  tags: foo bar 
     69 
     70will set I<foo bar> as a predefined tag to use. Optional. 
     71 
     72=item one_click_post 
     73 
     74Flag to indicate that clicking the widget will automatically post the 
     75item, without showing the form. Defaults to 0. (Optional) 
     76 
     77=back 
     78 
     79=head1 AUTHOR 
     80 
     81Tatsuhiko Miyagawa 
     82 
     83=head1 SEE ALSO 
     84 
     85L<Plagger>, L<http://del.icio.us/> 
     86 
     87=cut 
  • branches/feature-server/plagger/lib/Plagger/Rule/URLBL.pm

    r679 r856  
    2626 
    2727    return unless $url; 
    28      
     28 
    2929    my $res = Net::DNS::Resolver->new; 
    3030    my $dnsbl = $self->{dnsbl}; 
  • branches/feature-server/plagger/lib/Plagger/TT/Plagger/Util.pm

    r346 r856  
    1010    bless {}, shift; 
    1111} 
     12 
     13sub DESTROY { } 
    1214 
    1315sub AUTOLOAD { 
  • branches/feature-server/plagger/lib/Plagger/UserAgent.pm

    r572 r856  
    33use base qw( LWP::UserAgent ); 
    44 
     5use Plagger::Cookies; 
    56use URI::Fetch 0.06; 
    67 
     
    89    my $class = shift; 
    910    my $self  = $class->SUPER::new(); 
    10     $self->agent("Plagger/$Plagger::VERSION (http://plagger.bulknews.net/)"); 
    11     $self->timeout(15); # xxx to be config 
     11 
     12    my $conf = Plagger->context->conf->{user_agent}; 
     13    if ($conf->{cookies}) { 
     14        $self->cookie_jar( Plagger::Cookies->create($conf->{cookies}) ); 
     15    } 
     16 
     17    $self->agent( $conf->{agent} || "Plagger/$Plagger::VERSION (http://plagger.org/)" ); 
     18    $self->timeout( $conf->{timeout} || 15 ); 
     19    $self->env_proxy(); 
     20 
    1221    $self; 
    1322} 
    1423 
    1524sub fetch { 
    16     my($self, $url, $plugin) = @_; 
     25    my($self, $url, $plugin, $opt) = @_; 
    1726 
    1827    URI::Fetch->fetch($url, 
     
    2029        $plugin ? (Cache => $plugin->cache) : (), 
    2130        ForceResponse => 1, 
     31        ($opt ? %$opt : ()), 
    2232    ); 
     33} 
     34 
     35sub mirror { 
     36    my($self, $request, $file) = @_; 
     37 
     38    unless (ref($request)) { 
     39        return $self->SUPER::mirror($request, $file); 
     40    } 
     41 
     42    # below is copied from LWP::UserAgent 
     43    if (-e $file) { 
     44        my($mtime) = (stat($file))[9]; 
     45        if($mtime) { 
     46            $request->header('If-Modified-Since' => 
     47                             HTTP::Date::time2str($mtime)); 
     48        } 
     49    } 
     50    my $tmpfile = "$file-$$"; 
     51 
     52    my $response = $self->request($request, $tmpfile); 
     53    if ($response->is_success) { 
     54 
     55        my $file_length = (stat($tmpfile))[7]; 
     56        my($content_length) = $response->header('Content-length'); 
     57 
     58        if (defined $content_length and $file_length < $content_length) { 
     59            unlink($tmpfile); 
     60            die "Transfer truncated: " . 
     61                "only $file_length out of $content_length bytes received\n"; 
     62        } 
     63        elsif (defined $content_length and $file_length > $content_length) { 
     64            unlink($tmpfile); 
     65            die "Content-length mismatch: " . 
     66                "expected $content_length bytes, got $file_length\n"; 
     67        } 
     68        else { 
     69            # OK 
     70            if (-e $file) { 
     71                # Some dosish systems fail to rename if the target exists 
     72                chmod 0777, $file; 
     73                unlink $file; 
     74            } 
     75            rename($tmpfile, $file) or 
     76                die "Cannot rename '$tmpfile' to '$file': $!\n"; 
     77 
     78            if (my $lm = $response->last_modified) { 
     79                # make sure the file has the same last modification time 
     80                utime $lm, $lm, $file; 
     81            } 
     82        } 
     83    } 
     84    else { 
     85        unlink($tmpfile); 
     86    } 
     87    return $response; 
    2388} 
    2489 
  • branches/feature-server/plagger/lib/Plagger/Util.pm

    r683 r856  
    22use strict; 
    33our @ISA = qw(Exporter); 
    4 our @EXPORT_OK = qw( strip_html dumbnail decode_content extract_title load_uri ); 
     4our @EXPORT_OK = qw( strip_html dumbnail decode_content extract_title load_uri mime_type_of ); 
    55 
    66use Encode (); 
    77use List::Util qw(min); 
    88use HTML::Entities; 
     9use MIME::Types; 
     10use MIME::Type; 
    911 
    1012our $Detector; 
     
    2123    } 
    2224} 
     25 
     26 
    2327 
    2428sub strip_html { 
     
    6468    # 1) if it is HTTP response, get charset from HTTP Content-Type header 
    6569    if ($res) { 
    66         $charset = ($res->http_response->content_type =~ /charset=([\w\-]+)/)[0]; 
     70        $charset = ($res->content_type =~ /charset=([\w\-]+)/)[0]; 
    6771    } 
    6872 
     
    129133} 
    130134 
     135our $mimetypes = MIME::Types->new; 
     136$mimetypes->addType( MIME::Type->new(type => 'video/x-flv', extensions => [ 'flv' ]) ); 
     137$mimetypes->addType( MIME::Type->new(type => 'audio/aac', extensions => [ 'm4a', '.aac' ]) ); 
     138 
     139sub mime_type_of { 
     140    my $ext = shift; 
     141 
     142    if (UNIVERSAL::isa($ext, 'URI')) { 
     143        $ext = ( $ext->path =~ /\.(\w+)/ )[0]; 
     144    } 
     145 
     146    return unless $ext; 
     147    return $mimetypes->mimeTypeOf($ext); 
     148} 
     149 
    1311501;