WWW::Mixi::Scraper - PC日記

本家の方で、WWW::Mixi::Scraper 0.08がリリースされた

ので、早速patchも0.08に対応してみた。
今回から、descriptionはhtmlがデフォルトになったので、Plugin.pmの修正は必要なし。（と言うか前回の修正は意味なしだったけど）

WWW::Mixi::Scraper::Plugin::ViewBBS
コメントのリンクが発言者なのが気に入らない。本当は、コメント自身にlinkを張りたいんだけど、コメント自身を表すpermalinkがそもそも存在しないし、view_bbs.plのページからトピックのURLも直では取り出せなかった(必要な情報はあるので、scrapeしてから文字列処理をすれば良いんだけど)ので、とりあえず諦めた。

*** ViewBBS.pm.ORIG     Sun Oct  7 17:09:55 2007
--- ViewBBS.pm  Sun Oct  7 18:14:14 2007
***************
*** 25,30 ****
--- 25,32 ----
    };

    $scraper{topic} = scraper {
+     process 'div[class="pageTitle communityTitle002"]>h2',
+       comtitle => 'TEXT';
      process 'dl[class="bbsList01 bbsDetail"]>dt>span.date',
        time => 'TEXT';
      process 'dl[class="bbsList01 bbsDetail"]>dt>span.titleSpan',
***************
*** 38,44 ****
        'images[]' => $scraper{images};
      process 'div#localNavigation>ul.localNaviCommunity>li.top>a',
        link => '@href';
!     result qw( time subject description name name_link images link );
    };

    # bbs topic is not an array
--- 40,46 ----
        'images[]' => $scraper{images};
      process 'div#localNavigation>ul.localNaviCommunity>li.top>a',
        link => '@href';
!     result qw( comtitle time subject description name name_link images link );
    };

    # bbs topic is not an array
***************
*** 50,72 ****
        name => 'TEXT';
      process 'dd',
        description => $self->html_or_text;
!     result qw( link name description );
    };

    $scraper{list} = scraper {
      process 'dl.commentList01>dt[class="commentDate clearfix"]>span.date',
        'times[]' => 'TEXT';
      process 'dl.commentList01>dd>dl.commentContent01',
        'comments[]' => $scraper{comments};
!     result qw( times comments );
    };

    my $stash_c = $self->post_process($scraper{list}->scrape(\$html))->[0];

    my @comments = @{ $stash_c->{comments} || [] };
    my @times    = @{ $stash_c->{times} || [] };
    foreach my $comment ( @comments ) {
      $comment->{time} = _datetime( shift @times );
      $comment->{link} = _uri( $comment->{link} );
    }
    $stash->{comments} = \@comments;
--- 52,80 ----
        name => 'TEXT';
      process 'dd',
        description => $self->html_or_text;
!     process 'dd>table>tr>td',
!       'images[]' => $scraper{images};
!     result qw( link name description images );
    };

    $scraper{list} = scraper {
      process 'dl.commentList01>dt[class="commentDate clearfix"]>span.date',
        'times[]' => 'TEXT';
+     process 'dl.commentList01>dt[class="commentDate clearfix"]>span.senderId',
+       'nums[]' => 'TEXT';
      process 'dl.commentList01>dd>dl.commentContent01',
        'comments[]' => $scraper{comments};
!     result qw( times nums comments );
    };

    my $stash_c = $self->post_process($scraper{list}->scrape(\$html))->[0];

    my @comments = @{ $stash_c->{comments} || [] };
    my @times    = @{ $stash_c->{times} || [] };
+   my @nums    = @{ $stash_c->{nums} || [] };
    foreach my $comment ( @comments ) {
      $comment->{time} = _datetime( shift @times );
+     $comment->{subject} = '[' . $stash->{comtitle} . '] ' . $stash->{subject} . ": " . shift @nums;
      $comment->{link} = _uri( $comment->{link} );
    }
    $stash->{comments} = \@comments;

Plagger::Plugin::CustomFeed::MixiScraper

*** /home/sugimoto/MixiScraper.pm.ORIG  Sat Oct  6 11:27:45 2007
--- MixiScraper.pm      Sun Oct  7 18:15:22 2007
***************
*** 14,19 ****
--- 14,25 ----
          icon       => 'owner_id',
      },
      # can't get icon
+     Bbs => {
+         title      => 'コミュニティ最新書き込み',
+         get_list   => 'new_bbs',
+         get_detail => 'view_bbs',
+     },
+     # can't get icon
      Message => {
          title      => 'ミクシィメッセージ受信箱',
          get_list   => 'list_message',
***************
*** 110,115 ****
--- 116,124 ----
          $entry->link($msg->{link});
          $entry->author($msg->{name});
          $entry->date( Plagger::Date->parse($format, $msg->{time}) );
+       if ($entry->date) {
+           $entry->date->set_time_zone('Asia/Tokyo');
+       }

          if ($self->conf->{show_icon} && !$blocked && defined $MAP->{$type}->{icon}) {
              my $owner_id = $msg->{link}->query_param($MAP->{$type}->{icon});
***************
*** 136,144 ****
              }
          }

          if ($self->conf->{fetch_body} && !$blocked && $msg->{link} =~ /view_/ && defined $MAP->{$type}->{get_detail}) {
              $context->log(info => "Fetch body from $msg->{link}");
!             my $item = $self->cache->get_callback(
                  "item-$msg->{link}",
                  sub {
                      Time::HiRes::sleep( $self->conf->{fetch_body_interval} || 1.5 );
--- 145,155 ----
              }
          }

+       my $item = 0;
+
          if ($self->conf->{fetch_body} && !$blocked && $msg->{link} =~ /view_/ && defined $MAP->{$type}->{get_detail}) {
              $context->log(info => "Fetch body from $msg->{link}");
!             $item = $self->cache->get_callback(
                  "item-$msg->{link}",
                  sub {
                      Time::HiRes::sleep( $self->conf->{fetch_body_interval} || 1.5 );
***************
*** 161,166 ****
--- 172,180 ----
                  $entry->body($body);

                  $entry->date( Plagger::Date->parse($format, $item->{time}) );
+               if ($entry->date) {
+                   $entry->date->set_time_zone('Asia/Tokyo');
+               }
              } else {
                  $context->log(warn => "Fetch body failed. You might be blocked?");
                  $blocked++;
***************
*** 168,173 ****
--- 182,215 ----
          }

          $feed->add_entry($entry);
+
+       if ($item) {
+           for my $comment (@{ $item->{comments} || [] }) {
+               $entry = Plagger::Entry->new;
+               if ($comment->{subject}) {
+                   $entry->title($comment->{subject});
+               } else {
+                   $entry->title('comments for [' . $msg->{subject} . ']');
+               }
+               $entry->link($comment->{link});
+               $entry->author($comment->{name});
+               $entry->date( Plagger::Date->parse($format, $comment->{time}) );
+               if ($entry->date) {
+                   $entry->date->set_time_zone('Asia/Tokyo');
+               }
+                 my $body = $comment->{description};
+                 for my $image (@{ $comment->{images} || [] }) {
+                     $body .= qq(<div><a href="$image->{link}"><img src="$image->{thumb_link}" style="border:0" /></a></div>);
+                     my $enclosure = Plagger::Enclosure->new;
+                     $enclosure->url($image->{thumb_link});
+                     $enclosure->auto_set_type;
+                     $enclosure->is_inline(1);
+                     $entry->add_enclosure($enclosure);
+                 }
+                 $entry->body($body);
+               $feed->add_entry($entry);
+           }
+       }
      }

      $context->update->add($feed);

これでだいぶ快適になった。作者の方ありがとう。