Coding my own automated web site monitoring with Google Apps Script and a Spreadsheet

There are a great deal of good Web site monitoring and testing services out there. I’ve tested a few of them, but I needed two things I did not find (yet). 1) Good programmability and 2) Familiar presentation of the result in a Spreadsheet manner.

First I thought I could to create some scripts on my server and present the result in a Google Sheet. But then I found out that the Google Apps Scripts is capable of pretty much everything I need. I think they call the language “Javascript with objects”, which describes it very good. Use Javascript, but with a set of useful objects (and obviously functions).

So I created a script which can take a list of any urls and test them for avaliability, http response code, response time and look for some content. Here’s how my resulting sheet looks like after a test run. The full url’s are in the first column, then there’s a command column and an options column, currently accepting three different options (should-contain, should-not-contain and response-code):

The orange markers on the cells in column D are comments. I added the full page contents as comment texts, so that if there’s an error – I can go back and check the requested content.

I use this to test sites created with the Umbraco CMS, but it could of course run on any site. One thing that might be a bit particular for Umbraco sites is that the script is searching each content for the word “error”, which could appear on some pages if there’s an macro error or something else.

If the tester find one or more errors it will send me an email. Otherwise it’s just updating the sheet.

The script runs automatically every hour with the help of the built in scheduler in Google Apps Scripts.

Usage : 1) create a new Google sheet, add your URL’s from cell A2 and down. 2) Add the script in the script editor. 3) Run the function checkUrls(). 4) For scheduling – use the scheduler in the script editor.

function isCellEmpty(cellData) {
  return typeof(cellData) == "string" && cellData == "";
}

function checkUrls() {

  // The code below iterates over rows in a sheet and uses the value in
  // column A as an url, requests it and puts the response code in column B
  // and the request string as the comment

  var doc = SpreadsheetApp.getActiveSpreadsheet();
  var cell = doc.getRange('a1');
  var mailMessage = "";

  
  // leftmost resultcolumn
  
  var resultColumn = 3;
  
  cell.offset(0,resultColumn).setValue(new Date());
  cell.offset(0,resultColumn+1).setValue("Content type");
  cell.offset(0,resultColumn+2).setValue("Success?");
  cell.offset(0,resultColumn+3).setValue("Seconds");
  cell.offset(0,resultColumn+4).setValue("Comment");
  
  for (var i = 1; i < 20; ++i)
  {
    var cellData = cell.offset(i,0).getValue();
    if (!isCellEmpty(cellData))
    {

      var command = cell.offset(i,1).getValue();
      var optionData = cell.offset(i,2).getValue();
      if (optionData=="") optionData="{}";
      var options = Utilities.jsonParse(optionData);
      
      var hasError = false;
      var startTime = new Date();      
      
      if (command=="" | command=="GET")
      {

        var responseCode = 404;
        var requestContentText = "";
        var results = "";        
        var headers;
        var requestType ="";
        
        var expectedResponseCode = 200;
        if (options["response-code"]!=undefined)
          expectedResponseCode = options["response-code"];

        try
        {
          var response = UrlFetchApp.fetch(cellData);
          responseCode = response.getResponseCode();
          requestContentText = response.getContentText();
          headers = response.getHeaders();         
          if (headers!=undefined)
            contentType=headers["Content-Type"].toLowerCase();
        }
        catch (e)
        {
          requestContentText = e.message;
        } 
                
        cell.offset(i,resultColumn).setValue(responseCode);
        if (responseCode!=expectedResponseCode) 
        {
          hasError = true;      
          results += "Expected response code: " + expectedResponseCode + ". ";
        }
        
        if (contentType.indexOf("html")!=-1)
          cell.offset(i,resultColumn).setComment(requestContentText);
        else
          cell.offset(i,resultColumn).setComment("");
        
        cell.offset(i,resultColumn+1).setValue(contentType);
        
        // print results in column
        var colOffset = resultColumn+4;
        
        // not contain the word ERROR
        var containsError = (requestContentText.toLowerCase().indexOf("error") != -1);
        //cell.offset(i,colOffset).setValue("Error: " + containsError);
        if (containsError) 
        {
          results += "Error found. ";
          hasError = true;                      
        }
        
        if (options["should-contain"]!=undefined)
        {
          // not contain the word ERROR
          var shouldContain = options["should-contain"].toLowerCase();
          var doesContain = (requestContentText.toLowerCase().indexOf(shouldContain) != -1);
          if (!doesContain) 
          {
            results += "Not found: " + options["should-contain"] + ". ";
            hasError = true;                      
          }
          
        }

        if (options["should-not-contain"]!=undefined)
        {
          var shouldNotContain = options["should-not-contain"].toLowerCase();
          var doesContain = (requestContentText.toLowerCase().indexOf(shouldNotContain) != -1);
          if (doesContain) 
          {
            results += "Found: " + options["should-not-contain"] + ". ";
            hasError = true;                      
          }
          
        }
        cell.offset(i,colOffset).setValue(results);
        
      }
      
      // timer        
      var endTime = new Date();      
      var timeDiff = endTime-startTime;
      
      // success? (no errors)
      cell.offset(i,resultColumn+2).setValue(!hasError);
      if (hasError)
      {
        cell.offset(i,resultColumn+2).setBackgroundColor("red");  
        mailMessage += "ERROR on " + cellData + ":" + results;
      }
      else
        cell.offset(i,resultColumn+2).setBackgroundColor("green");  
      
      // time spent (in seconds)      
      cell.offset(i,resultColumn+3).setValue(timeDiff/1000);
      

    }
    else
    {
      break;
    }
  }
  if (mailMessage!="")
  {
    MailApp.sendEmail("me@myaddress.com", "ERROR on your web ", mailMessage);
  }
  
}

function getTime()
{
  var startTime = new Date();
  Browser.msgBox(startTime);
  var endTime = new Date();
  var timeDiff = endTime-startTime;
  Browser.msgBox(timeDiff);
}

Obviously there’s a lot more the ready made services can do, but the beauty of this little thing is that I can add features to it easily whenever I need. And I can share the sheets with my clients, add nice reporting stuff like charts, pivots or whatever.

One current pretty big limitation is I could not figure out how to test pages that is behind login. I hope I can solve that soon.