use LWP::UserAgent;

my $ua = LWP::UserAgent->new();
$ua->agent('ExBot/alpha/' . $ua->_agent);
$ua->cookie_jar({ file => "$ENV{HOME}/.tmp/cookies.txt" });

use HTTP::Request;

my $url = "http://www.example.com/?fu=bar";
my $request = HTTP::Request->new("GET", $url);

use HTTP::Request;

my $url = "http://www.example.com/";
my $param = "fu=bar";
my $request = HTTP::Request->new("HEAD", $url);
$request->content_type('application/x-www-form-urlencoded');
$request->content($param);

my $response = $ua->request($request);

if ($response->is_success()) {
print $response->content;
}
else {
print "An error occured: " . $response->status_line() . "\n";
}

use URI;

my $url = "http://www.example.com/?fu=bar";
# url of the page I want to index
my $p_uri = URI->new($url);
# url of the robots.txt
my $r_uri = URI->new();

$r_uri->scheme("http");
$r_uri->host($p_uri->host());
$r_uri->path("robots.txt");

use WWW::RobotRules;

my $url = "http://www.example.com/?fu=bar";
my $robots_txt = "user-agent: *\ndisallow: /tmp\n";

if (defined($robots_txt)) {
my $rules = WWW::RobotRules->new("ExBot");
$rules->parse($r_uri->as_string, $robots_txt);

print $rules->allowed($url);
}
else {
print "Error retrieving robots.txt\n";
}

use HTML::TokeParser;

my $stream = HTML::TokeParser->new(\$content);

while (my $tag = $stream->get_tag("meta")) {
if (exists($tag->[1]{name}) and lc($tag->[1]{name}) eq "robots") {
if (exists($tag->[1]{content})) {
my $content = $tag->[1]{content};
if ($content =~ m/noindex/i) {
print "Not allowed to index this page\n";
}
if ($content =~ m/nofollow/i) {
print "Not allowed to follow links on this page\n";
}
last;
}
}
}

#!/usr/bin/perl

use strict;
use warnings;
use Desktop::Notify;
use LWP::Simple;
use HTML::TokeParser;

my $sn = shift;

my $logfile = "/home/rork/Scripts/dev/logs/dhl.log";

my $url = "http://nolp.dhl.de/nextt-online-public/set_identcodes.do?lang=en&zip=00823&idc=" . $sn;

my $page = get($url) or die $!;

my %log;

open(LOG, "<", $logfile) or die "Can't open $logfile for reading: $!";
while() {
chomp($_);
my ($serial, $date, $action) = split(/\//, $_, 3);
$log{$serial}->{$date} = $action;
}
close(LOG);

my $stream = HTML::TokeParser->new(\$page);

while (my $tag = $stream->get_tag("table")) {
next unless ($tag->[1]{class} and $tag->[1]{class} eq 'full eventList');
# skip the header
$tag = $stream->get_tag("tr");
while ($tag = $stream->get_tag("tr")) {
my ($date, $action) = ("", "");
$tag = $stream->get_tag("td");
if ($tag->[1]{class} and $tag->[1]{class} eq "event_date") {
$date = $stream->get_trimmed_text();
}
$tag = $stream->get_tag("td");
$tag = $stream->get_tag("td");
if ($tag->[1]{class} and $tag->[1]{class} eq "status lasttd") {
$tag = $stream->get_tag("div");
$action = $stream->get_trimmed_text();
}

if ($date ne "" and $action ne "") {
# print "$sn/$date/$action\n";
if (!exists($log{$sn}->{$date})) {
my $notify = Desktop::Notify->new();
my $notification = $notify->create(body => $action);
$notification->show();


$log{$sn}->{$date} = $action;
}
}
}
}

open(LOG, ">", $logfile) or die "Can't open $logfile for writing: $!";
foreach my $serial(keys %log) {
foreach my $date(keys %{$log{$serial}}) {
print LOG join("/", $serial, $date, $log{$serial}->{$date}) . "\n";
}
}
close(LOG);

0 * * * * DISPLAY=:0 perl -w /home/rork/Scripts/dev/dhl-notify.pl 467509123888 >/dev/null 2>>/dev/null

<%if(skintype,item)%>
<META NAME = "robots" CONTENT = "follow,index">
<%else%>
<META NAME = "robots" CONTENT = "follow,noindex">
<%endif%>