I wrote a fast file cleaner the grep use is interesting ...
Clean your lists up, etc. ...

Code:

#!/usr/bin/perl
####################################
# nodupes.cgi
#you may use free as-is with  no warranty
#make the outfile's chmod 666 if using the webserver
#chmod this script to 755
####################################
use CGI::Carp qw/fatalsToBrowser/;
use CGI qw/:standard/;

print "Content-type: text/html\n\n";

my $query = "$ENV{'QUERY_STRING'}";
	if ($query =~ s/[^a-zA-Z0-9\_]//g) {print qq~HUH???~;       exit;}

my $infile="somesiteurl.txt";
my $outfile="somesiteurlduped.txt";

   open(INPUT, "<", $infile) || die "infile not found\n";
      my @array=(<INPUT>);
   open(OUTPUT, ">>",$outfile )|| die "outfile not found\n";


         my %seen = ();
         my @unique = grep { ! $seen{ $_ }++ } @array;

            foreach my $unique(@unique){
                  chomp $unique;
                  print OUTPUT "$unique\n"
                    }

close OUTPUT;
close INPUT;