# Copyright (c) 2012 Flowerfire, Inc.  All Rights Reserved.
webmetrics = {

  plugin_version = "1.0"

  # 2012-04-04 - 1.0 - GMF - Initial creation

  info.1.manufacturer = "Neustar"
  info.1.device = "Webmetrics"
  info.1.version.1 = ""

  # 2007/09/11 - 1.0 - KBB - added version number and changed file name from beta_apache_combined.cfg
  # 2007/12/18 - 1.0.1 - GMF - Fixed a bug which would result in all 0's if hits was not selected.
  # 2010/10/01 - 1.0.2 - MSG - Edited info lines

  # The name of the log format
  log.format.format_label = "Neustar Webmetrics Log Format"
  log.miscellaneous.log_data_type = "http_access"  
  log.miscellaneous.log_format_type = "web_server"

  # The log is in this format if any of the first ten lines match this regular expression
  log.format.autodetect_regular_expression = "^[A-Z][a-z][a-z] [0-9]+/[0-9]+ [0-9][0-9]:[0-9][0-9]:[0-9][0-9][.][0-9][0-9] NOTE: [0-9.]+ seconds to load page "

  # Log fields
  log.fields = {

    date = ""
    time = ""

    server = ""

    sampleguid = ""
    baseline = ""
    servicegui = ""
    host = ""
    agenttype = ""
    agenturl = ""

    url = ""
    ip_address = ""

    page_loads = ""
    item_loads = ""
    page_load_time = ""
    item_load_time = ""
    dns_time = ""
    connect_time = ""
    first_packet_time = ""
    bytes = ""

  } # log.fields


  # Database fields
  database.fields = {

    date_time = ""
    day_of_week = ""
    hour_of_day = ""

    server = ""

    sampleguid = ""
    baseline = ""
    servicegui = ""
    host = ""
    agenttype = ""
    agenturl = ""

    url = ""
    ip_address = ""

  } # database.fields

  log.format.parse_only_with_filters = "true"
  log.filter_initialization = `
string year = substr(epoc_to_date_time(now()), 7, 4);
`

  # Parse with filters
  log.parsing_filters.parse = `

# If there is a line containins year=, use that as the year of the log data
#https://api.webmetrics.com/v2/?method=logdownload.getdata&username=abc&sig=scyrLwz177wqANVWHQKFh/jZAwI&serviceid=245731&day=1&month=3&year=2012<?xml version="1.0" encoding="utf-8" ?><rsp stat="ok">
if (matches_regular_expression(current_log_line(), '^https:.*year=([0-9][0-9][0-9][0-9])')) then
  year = $1;

# Server: Cleveland, OH
else if (matches_regular_expression(current_log_line(), '^Server: (.*)$')) then
  server = $1;

# META: sampleguid=(5D38B80063A711E1965AA27C539EE119) baseline=() serviceguid=(23022BCC623E11E1BC11ACF57B5DB1C5) host=(stwscprfs3.va.neustar.com) agenttype=(S) agenturl=(https://66.6.161.120:6212/cgi-bin/rev2.8)
else if (matches_regular_expression(current_log_line(), '^META: sampleguid=[(]([^)]*)[)] baseline=[(]([^)]*)[)] serviceguid=[(]([^)]*)[)] host=[(]([^)]*)[)] agenttype=[(]([^)]*)[)] agenturl=[(]([^)]*)[)]')) then (
  set_collected_field('', 'sampleguid', $1);
  set_collected_field('', 'baseline', $2);
  set_collected_field('', 'serviceguid', $3);
  set_collected_field('', 'host', $4);
  set_collected_field('', 'agenttype', $5);
  set_collected_field('', 'agenturl', $6);
);

# Thu 3/1 08:03:41.48 NOTE: 4.72 seconds to load page 1: http://www.somewhere.com/abc/ (1673594 bytes)
else if (matches_regular_expression(current_log_line(), '^[A-Z][a-z][a-z] ([0-9]+/[0-9]+) ([0-9][0-9]:[0-9][0-9]:[0-9][0-9])[.][0-9]+ NOTE: (.*)$')) then (
  set_collected_field('', 'date', $1 . '/' . year);
  set_collected_field('', 'time', $2);
  v.remainder = $3;

  # Thu 3/1 08:03:41.48 NOTE: 4.72 seconds to load page 1: http://www.somewhere.com/abc/ (1673594 bytes)
  if (matches_regular_expression(v.remainder, '^([0-9.]+) seconds to load page [0-9]+: ([^ ]+) [(]([0-9]+) bytes')) then (
    set_collected_field('', 'page_load_time', $1);
    set_collected_field('', 'url', $2);
    set_collected_field('', 'bytes', $3);
    set_collected_field('', 'page_loads', 1);
    set_collected_field('', 'item_loads', 0);
    accept_collected_entry('', true);
    set_collected_field('', 'page_loads', 0);
  ); # if page load line

  # Thu 3/1 08:03:41.48 NOTE: 0.12 secs to load item 1: http://www.metropcs.com/ (0.03 DNS, 0.05 connect, 0.12 first packet, 187 bytes, 69.93.142.82)
  else if (matches_regular_expression(v.remainder, '^([0-9.]+) secs to load item [0-9]+: ([^ ]+) [(]([0-9.]+) DNS, ([0-9.]+) connect, ([0-9.]+) first packet, ([0-9.]+) bytes, ([^)]+)[)]')) then (
    set_collected_field('', 'item_load_time', $1);
    set_collected_field('', 'url', $2);
    set_collected_field('', 'dns_time', $3);
    set_collected_field('', 'connect_time', $4);
    set_collected_field('', 'first_packet_time', $5);
    set_collected_field('', 'bytes', $6);
    set_collected_field('', 'ip_address', $7);
    set_collected_field('', 'page_loads', 0);
    set_collected_field('', 'item_loads', 1);
    accept_collected_entry('', true);
    set_collected_field('', 'item_loads', 1);
  ); # if page load line

); # if NOTE line    
`


  # Log Filters
  log.filters = {
  } # log.filters

  database.numerical_fields = {

    page_loads = {
      default = true
    } # page_loads

    item_loads = {
      default = true
    } # item_loads

    page_load_time = {
      default = true
      type = "float"
      display_format_type = "duration_compact"
    } # page_load_time

    item_load_time = {
      default = true
      type = "float"
      display_format_type = "duration_compact"
    } # item_load_time

    dns_time = {
      type = "float"
      display_format_type = "duration_compact"
    } # dns_time

    connect_time = {
      type = "float"
      display_format_type = "duration_compact"
    } # connect_time

    first_packet_time = {
      type = "float"
      display_format_type = "duration_compact"
    } # first_packet_time

    bytes = {
      integer_bits = 64
      display_format_type = "bandwidth"
    } # bytes

  } # database.numerical_fields


  create_profile_wizard_options = {
  
    # How the reports should be grouped in the report menu
    report_groups = {
      date_time_group = ""
    } # report_groups

  } # create_profile_wizard_options

} # webmetrics