#!/usr/bin/perl

    use strict;
    use warnings;

    use URI::QueryParam;
    use URI;

my %seen;

open FIN, "<list.txt" or die;
while (<FIN>) {
    s#^https://#http://#si;

    my $u = URI->new($_)->canonical;

    # see http://search.cpan.org/dist/URI/URI/QueryParam.pm
    foreach my $key (qw[  SearchID fromrss Template template SectionCat updates ]) {
        $u->query_param_delete($key);
    }

    my @date = $u->query_param_delete("Date");
    if (@date) {
        my @category = $u->query_param_delete("Category");
        my @artno = $u->query_param_delete("ArtNo");

        $u->query_param("AID", "/$date[0]/$category[0]/$artno[0]");
    }

    my $url = $u->as_string;
    $url =~ s#%2F#/#sgi;

    $seen{$url}++;

    #print "$url\n";
}
close FIN;

open FOUT, ">winnowed.txt" or die;
print FOUT join("\n", sort keys %seen), "\n";
close FOUT;