#!/usr/bin/perl

# an HTML screen-scraper for http://www.propublica.org/site/author/julia_angwin/
# that pulls the list of her latest articles off
# for help in constructing  http://paperlined.org/external/Dragnet_Nation__Julia_Angwin.html

    use strict;
    use warnings;

    use LWP::Simple;
    use Data::Dumper;
    #use Devel::Comments;           # uncomment this during development to enable the ### debugging statements



my $html = get "http://www.propublica.org/site/author/julia_angwin";

$html =~ s/^.*?(<div class="author-articles">)/$1/si;
$html =~ s/<!--\/author-articles-->.*//si;

my @articles = split /class="article-excerpt"/, $html;
        #print join("\n"x12, @articles), "\n";  exit;

foreach my $article (@articles) {
    my ($link) = ($article =~ /(<a href=.*?<\/a>)/si);
    my ($date) = ($article =~ /<p class="byline">([^<]*)/si);
    $date =~ s/^[\s\n\r]*|[\s\n\r]+$//sg;
            #print "$link\n\t\t\t$date\n";  next;
    $date =~ s/,.*//;       # remove the time, only keep the day
    $date =~ s/^(\w{3})\. /$1 /;        # remove the period from the end of the abbreviated month

    print "    <li>$date <del\/> $link\n";
}