develooper Front page | perl.beginners | Postings from April 2002

WEB Monitoring Script going crazy

From:
Zachary Buckholz
Date:
April 27, 2002 02:07
Subject:
WEB Monitoring Script going crazy
Message ID:
OE34PM1EhXTwpb1iTDS0000078b@hotmail.com
I wrote the script below to pull data from a mysql database and update the
datebase based on the results of a check. I have used Parallel::ForkManager
which I used in the past in another script without problems. But for some
reason this script killed my server today when it tried to check 4,000 plus
web sites to see if they were up or down. I am not sure if it's my code or
if I am just expecting to much.

I would appreciate any comments or feedback, I just recently started using
array references and I think that maybe I might be going about it all wrong
and there is a better more efficient way to accomplish what I am trying to
do. I had to perform a hardboot on the server to kill the process.

The script runs from the crontab every 60 seconds. Which could also be the
problem, I might need to add a field to the database to show that the URL
has been 'checked out' before the next cron job attempts to check it out
again.

The code below is the complete script. Hopefully someone can point out a
logical error and help improve the flow. I also included a SQL table
structure dump at the end, maybe I have a lousy db structure causing
problems.

Thanks

Zack

#!/usr/bin/perl -w

# TODO:
# Remove die statements because in production I do not want the program
# to exit on an error. I want to be notified and have it continue if
possible.
# learn how to use eval statements

use strict;
use DBI;
use LWP::UserAgent;
use Digest::MD5 qw(md5_hex);
use Parallel::ForkManager;
use Mail::Mailer qw(qmail);

  my $current_time = time();
  my $url_status;
  my $message;
  my $up;

  my $dbh =
DBI->connect("DBI:mysql:database=saint;host=localhost","username","password"
, {'RaiseError' => 1});

# Build http response code reference table

  my $http_resp_table = build_http_resp_table();

# Get URL list to monitor

  my $url_list = select_url();

# Do monitoring here

  check_url(@$url_list);

# End program execution

  $dbh->disconnect;
  exit();

# Core program

sub select_url() {

  my $select_query = "SELECT check.url_id, check.url_timeout,
url.url_protocol, url.url, url.contact_ids, MD5.MD5, check.mc_id,
check.lc_status, check.lc_MD5_status
                      FROM monitor_check check
                      LEFT OUTER JOIN monitor_url url
                      ON check.url_id = url.url_id
                      LEFT OUTER JOIN monitor_MD5 MD5
                      ON check.url_id = MD5.url_id
                      WHERE ((active = 1)
                      AND ((chk_freq + UNIX_TIMESTAMP(lc_epoch)) <= ?))";

  my $url_list = $dbh->selectall_arrayref($select_query, undef,
$current_time)
        or die "Can't execute statement: $DBI::errstr";

  return($url_list);

}

sub check_url(@$url_list) {

# I should be able to increase this to 60 without any problems
# but 4,219 URL test on 4/26/2002 set at 10 Parallel killed
# server. Running at 3 causes <8% cpu but 35-50% mem usage.
# When it ran at 10 today it caused the unix top comment to
# report 18.0 load average. ????

  my $pm = new Parallel::ForkManager(3); # <---- Safe number for now.

  foreach my $url (@$url_list) {


        my ($url_id) = (@$url[0]);
        my ($url_timeout) = (@$url[1]);
        my ($url_protocol) = (@$url[2]);
        my ($url_url) = (@$url[3]);
        my ($url_contacts) = (@$url[4]);
        my ($url_MD5) = (@$url[5]);
        my ($mc_id) = (@$url[6]);
        my ($lc_status) = (@$url[7]);
        my ($lc_MD5_status) = (@$url[8]);

        my ($chk_status_alive, $chk_status_resp_time, $chk_status_code);
        my $chk_status_MD5 = 1;


        my $ua = LWP::UserAgent->new(agent => "url_checker",
                                     from  => "zbuckholz@hotmail.com",
                                     timeout => $url_timeout,
                                    );

        my $pid = $pm->start and next;  # <---- Begin actual parallel fork
here

        my $url_string = $url_protocol . "://" . $url_url;

        my $request = HTTP::Request->new('GET', $url_string);

        my $start = time();
        my $response = $ua->request($request);
        my $stop = time();


        if ( $response->is_success && $lc_status == 0 ) {
print "DEBUG FIRST TIME\n";
                $chk_status_alive = 1;
                $chk_status_code = $response->code;
                $chk_status_resp_time = $stop - $start;
                if( $url_MD5 ) {
                        my $current_digest = md5_hex($response->content);
                        my $stored_digest = $url_MD5;
                        if($current_digest ne $stored_digest) {
                                $chk_status_MD5 = 0;
                        } else {
                                $chk_status_MD5 = 1;
                        }
                }
                my $url_status = [$url_id, $chk_status_alive,
$chk_status_resp_time, $chk_status_MD5, $chk_status_code, $mc_id];
                if($chk_status_MD5 == 0) {
                        notify_down($url_id, $chk_status_MD5,
$chk_status_code, $url_contacts, $url_url);
                }
                update_url_status(@$url_status);
        } elsif ( $response->is_success && $lc_status != 200 ) {
print "$url_url DEBUG IS SUCC and BAD LC STATUS\n";
                $chk_status_alive = 1;
                $chk_status_code = $response->code;
                $chk_status_resp_time = $stop - $start;
                if( $url_MD5 ) {
                        my $current_digest = md5_hex($response->content);
                        my $stored_digest = $url_MD5;
                                if($current_digest ne $stored_digest) {
                                        $chk_status_MD5 = 0;
                                        if($lc_MD5_status == 0) {
                                        # Do nothing already sent notice
print "$url_url BAD MD5 Already Sent Notice\n";
                                        } else {
print "$url_url BAD MD5 Sending Notice\n";
                                                notify_down($url_id, $chk_st
atus_MD5, $chk_status_code, $url_contacts, $url_url);
                                        }
                                } else {
                                $chk_status_MD5 = 1;
                        }
                }
                notify_up($url_id, $chk_status_MD5, $chk_status_code,
$url_contacts, $url_url);
                my $url_status = [$url_id, $chk_status_alive,
$chk_status_resp_time, $chk_status_MD5, $chk_status_code, $mc_id];
                update_url_status(@$url_status);
        } elsif ($response->is_success) {
print "$url_url DEBUG IS SUCC\n";
                $chk_status_alive = 1;
                $chk_status_code = $response->code;
                $chk_status_resp_time = $stop - $start;
                if( $url_MD5 ) {
                        my $current_digest = md5_hex($response->content);
                        my $stored_digest = $url_MD5;
                        if($current_digest ne $stored_digest) {
                                $chk_status_MD5 = 0;
                                if($lc_MD5_status == 0) {
                                # Do nothing already sent notice
print "$url_url BAD MD5 Already Sent Notice\n";
                                } else {
print "$url_url BAD MD5 Sending Notice\n";
                                        notify_down($url_id,
$chk_status_MD5, $chk_status_code, $url_contacts, $url_url);
                                }
                        } else {
                                $chk_status_MD5 = 1;
                        }
                }
                my $url_status = [$url_id, $chk_status_alive,
$chk_status_resp_time, $chk_status_MD5, $chk_status_code, $mc_id];
                update_url_status(@$url_status);
        } else {
print "$url_url DEBUG BAD \n";
                my $rc = $response->code;
                $up = 0;
                $chk_status_alive = 0;
                        if (exists($http_resp_table->{$rc})) {
#                               print $rc . " : " .
$http_resp_table->{$rc}->{mnemonic} . "\n";
                                $chk_status_code = $rc;
                        } else {
                                $chk_status_code = 303;
                        }
                if($lc_status != 200) {
                #already sent notice so do nothing but log it
print "$url_url Already Sent Notice\n";
                } else {
                        notify_down($url_id, $chk_status_MD5,
$chk_status_code, $url_contacts, $url_url);
print "$url_url Sending Notice\n";
                }
        my $url_status = [$url_id, $chk_status_alive, $chk_status_resp_time,
$chk_status_MD5, $chk_status_code, $mc_id];

        update_url_status(@$url_status);
        }

        $pm->finish;    # <---- End of parallel fork


  }

}

sub build_http_resp_table() {

  my $select_query = "SELECT mnemonic, code
                      FROM monitor_response_codes";
  my $http_response_list = $dbh->selectall_hashref($select_query, 'code')
        or die "Can't execute statement: $DBI::errstr";

  return($http_response_list);
}

sub update_url_status {

        my ($url_id) = ($_[0]);

        my ($chk_status_alive) = ($_[1]);

        my ($chk_status_resp_time) = ($_[2]);

        my ($chk_status_MD5) = ($_[3]);

        my ($chk_status_code) = ($_[4]);

        my ($mc_id) = ($_[5]);

        my @update_values = ($current_time,$chk_status_code,
$chk_status_MD5, $url_id);
        my @insert_values = ($mc_id, $url_id, $current_time,
$chk_status_code, $chk_status_resp_time);


        my $update_query_statement = "UPDATE monitor_check
                                      SET lc_epoch = FROM_UNIXTIME(?) ,
lc_status = ? , lc_MD5_status = ?
                                      WHERE url_id = ?";

        my $update_query = $dbh->prepare($update_query_statement)
            or die "Unable to prepare update_query: $DBI::errstr";

        my $update = $update_query->execute(@update_values)
            or die "Unable to execute update_query: $DBI::errstr";

        my $insert_query_statement = "INSERT INTO monitor_report
                                      VALUES ( ?, ?, FROM_UNIXTIME(?), ?,
? )";

        my $insert_query = $dbh->prepare($insert_query_statement)
            or die "Unable to prepare insert_query: $DBI::errstr";

        my $insert = $insert_query->execute(@insert_values)
            or die "Unable to execute update_query: $DBI::errstr";

}

sub notify_down() {
        my ($url_id, $chk_status_MD5, $chk_status_code, $url_contacts,
$url_url) = @_;



        my @contact_ids = split(/-/, $url_contacts);

        foreach my $contact_id(@contact_ids) {
          my $select_query = "SELECT contact_email
                              FROM monitor_contact
                              WHERE contact_id = ?";

          my $select_handler = $dbh->prepare($select_query)
            or die "Unable to prepare select_handler: $DBI::errstr";

          my $select_execute = $select_handler->execute($contact_id)
            or die "Unable to execute email select query: $DBI::errstr";

          my $email = $select_handler->fetchrow_array;


          my $time = scalar localtime;

          if ($chk_status_MD5 == 0) {
          $message = "Please check $url_url\n";
          $message = $message . "MD5 Finger Print Alert\n";
          $message = $message . "If you have changed your web content\n";
          $message = $message . "you must generate a new MD5 finger\n";
          $message = $message . "or disable MD5 checking to avoid\n";
          $message = $message . "future alerts like this.\n";
          $message = $message . "Time: $time\n";
          } else {
          $message = "Please check $url_url\n";
          $message = $message . "Time: $time\n";
          }

        my $mailer = new Mail::Mailer;
        my %headers = ('To' => "$email",
                       'From' => "zbuckholz@hotmail.com",
                       'Subject' => "Down Alert $url_url");

        $mailer->open(\%headers);

        print $mailer $message;

        $mailer->close;
        }

}

sub notify_up() {

        my ($url_id, $chk_status_MD5, $chk_status_code, $url_contacts,
$url_url) = @_;



        my @contact_ids = split(/-/, $url_contacts);

        foreach my $contact_id(@contact_ids) {
          my $select_query = "SELECT contact_email
                              FROM monitor_contact
                              WHERE contact_id = ?";

          my $select_handler = $dbh->prepare($select_query)
            or die "Unable to prepare select_handler: $DBI::errstr";

          my $select_execute = $select_handler->execute($contact_id)
            or die "Unable to execute email select query: $DBI::errstr";

          my $email = $select_handler->fetchrow_array;


          my $time = scalar localtime;

          if ($chk_status_MD5 == 0) {
          $message = "$url_url is back UP!\n";
          $message = $message . "MD5 Finger Print Alert\n";
          $message = $message . "Time: $time\n";
          } else {
          $message = "$url_url is back UP!\n";
          $message = $message . "Time: $time\n";
          }

        my $mailer = new Mail::Mailer;
        my %headers = ('To' => "$email",
                       'From' => "zbuckholz@hotmail.com",
                       'Subject' => "Up Alert $url_url");

        $mailer->open(\%headers);

        print $mailer $message;

        $mailer->close;
        }

}


#
# Table structure for table `monitor_MD5`
#

CREATE TABLE monitor_MD5 (
  mc_id int(11) NOT NULL default '0',
  url_id int(11) NOT NULL default '0',
  MD5 varchar(32) NOT NULL default '',
  timestamp timestamp(14) NOT NULL
) TYPE=MyISAM;
#

#
# Table structure for table `monitor_check`
#

CREATE TABLE monitor_check (
  mc_id int(11) NOT NULL default '0',
  url_id int(11) NOT NULL default '0',
  chk_freq int(4) NOT NULL default '0',
  url_timeout int(11) NOT NULL default '0',
  lc_epoch timestamp(14) NOT NULL,
  lc_status int(3) NOT NULL default '0',
  active int(1) NOT NULL default '0',
  lc_MD5_status int(11) default NULL,
  PRIMARY KEY  (url_id)
) TYPE=MyISAM;
#

#
# Table structure for table `monitor_contact`
#

CREATE TABLE monitor_contact (
  mc_id int(11) NOT NULL default '0',
  contact_id int(11) NOT NULL auto_increment,
  contact_email varchar(255) NOT NULL default '',
  UNIQUE KEY mon_contact (mc_id,contact_email),
  UNIQUE KEY contact_id (contact_id)
) TYPE=MyISAM;
#

#
# Table structure for table `monitor_customers`
#

CREATE TABLE monitor_customers (
  mc_primary_key mediumint(8) unsigned NOT NULL auto_increment,
  mc_email varchar(255) NOT NULL default '',
  mc_password varchar(32) NOT NULL default '',
  mc_userlevel tinyint(3) default NULL,
  mc_last_login datetime NOT NULL default '0000-00-00 00:00:00',
  mc_name varchar(255) NOT NULL default '',
  mc_company varchar(255) default NULL,
  mc_address1 varchar(255) default NULL,
  mc_address2 varchar(255) default NULL,
  mc_city varchar(255) default NULL,
  mc_state varchar(255) default NULL,
  mc_zip varchar(255) default NULL,
  mc_phone varchar(255) default NULL,
  PRIMARY KEY  (mc_primary_key),
  KEY user (mc_email)
) TYPE=MyISAM;
#

#
# Table structure for table `monitor_report`
#

CREATE TABLE monitor_report (
  mc_id int(11) NOT NULL default '0',
  url_id int(11) NOT NULL default '0',
  toc_epoch timestamp(14) NOT NULL,
  chk_status int(3) NOT NULL default '0',
  resp_sec int(2) default '0',
  UNIQUE KEY mc_id (mc_id,url_id,toc_epoch),
  KEY url_id (url_id),
  KEY mc_id_2 (mc_id)
) TYPE=MyISAM;
#

#
# Table structure for table `monitor_response_codes`
#

CREATE TABLE monitor_response_codes (
  mnemonic text NOT NULL,
  code int(3) NOT NULL default '0'
) TYPE=MyISAM;
#

#
# Table structure for table `monitor_url`
#

CREATE TABLE monitor_url (
  mc_id int(11) NOT NULL default '0',
  url_id int(11) NOT NULL auto_increment,
  url_protocol varchar(5) NOT NULL default '',
  url varchar(255) NOT NULL default '',
  contact_ids varchar(255) default NULL,
  PRIMARY KEY  (url_id,url_id)
) TYPE=MyISAM;



nntp.perl.org: Perl Programming lists via nntp and http.
Comments to Ask Bjørn Hansen at ask@perl.org | Group listing | About