http://paperlined.org/src/pl/jollies/backend/fetch.pl

#!/usr/bin/perl

# just the TGP-fetching module for now...  we'll integrate it into a monolithic program later

    use strict;
    use warnings;

    use FindBin;
    use lib '/home/interiot/cpan/lib/';
    use lib $FindBin::Bin;


    use NeverTMPbackend;
    use NeverTMPbackend_dbm;
    use URI;
    use Data::Dumper;


sub hrs {shift() * 60 * 60}


my @sites = (

    {url => "http://restrainedmaidens.com/",
        referer => "http://www.google.com/search?hl=en&ie=ISO-8859-1&q=restrained+maidens&btnG=Google+Search",
        every => hrs(24),
    },


    {url => "http://www.fetishcabin.com/", every => hrs(8) },
    {url => "http://bdsmnewlife.com/", every => hrs(8) },
    {url => "http://bondagehotporn.com/", every => hrs(24) },
    {url => "http://www.bondagepaper.com/", every => hrs(24) },
    {url => "http://www.brutaltgp.com/", every => hrs(24) },
    {url => "http://spank-tgp.com/", every => hrs(24) },
    {url => "http://www.extremeapril.com/", every => hrs(24) },
    {url => "http://www.bondagester.com/", every => hrs(24) },
    {url => "http://www.bdsmmpegs.net/", every => hrs(24) },
    {url => "http://www.pain-vids.com/", every => hrs(24) },
    {url => "http://www.harassedslavegirl.com/", every => hrs(24) },
    {url => "http://www.brutalfetish.com/", every => hrs(24) },
    {url => "http://www.xxxbdsmvideo.com/", every => hrs(24) },
    {url => "http://www.rudepig.com/", every => hrs(24) },
    {url => "http://www.xxxbdsmmovie.com/", every => hrs(24) },
    {url => "http://cutyournipples.com/", every => hrs(24) },
    {url => "http://www.brutalobsession.ws/", every => hrs(24) },
    {url => "http://www.yobt.com/bondage.html", every => hrs(24) },
    {url => "http://www.bondageco.com/", every => hrs(24) },

);



NeverTMPbackend_dbm::open_dbs("$FindBin::Bin/db/");

my $ua = NeverTMPbackend::configure_UserAgent();


foreach my $site (@sites) {
    my $url = $site->{url};
    my $domain = URI->new($url)->host;
    my $cron = NeverTMPbackend_dbm::get($transdb, "cron\\$url");


    if (!$cron || (time() - $cron->{lastfetched} > $site->{every})) {
        print "Fetching $url...\n";
        my $req = HTTP::Request->new(GET => $url);
        if ($site->{referer}) {
            $req->header( referer => $site->{referer} );
        }
        my $res = $ua->request($req);
        if ($res->is_success) {
            my $queue_entry = {
                page_contents => $res->content,
                fetched => time(),
            };

            NeverTMPbackend_dbm::enqueue($transdb, "TGP_parse\\$domain", $queue_entry);
        }


        if (!$cron) {
            $cron = {url => $url};
        }
        $cron->{lastfetched} = time();
        NeverTMPbackend_dbm::put($transdb, "cron\\$url", $cron)

    }
}



#my $data = [1, 2, 3];
#my $key = NeverTMPbackend_dbm::enqueue($permdb, "foo", $data);
#print "$key\n";

#print Dumper( NeverTMPbackend_dbm::dequeue($permdb, "foo") );