# Copyright (c) 2010 Flowerfire, Inc.  All Rights Reserved.
safesquid_combined = {

  plugin_version = "2.1"

  info.1.manfacturer = "SafeSquid"
  info.1.device = "SafeSquid (Extended Logging)"
  info.1.version.1 = "4.2.1"

  # 2006/Jun/08: 1.0beta: initial creation - KBB
  # 2007-09-14 - 1.0 - KBB - renumbered per new beta policy
  # 2007-12-21 - 2.0 - GMF - Added support for new format; renamed fields to match vendor docs
  # 2008-01-01 - 2.1 - GMF - Minor change to support IPs in USER_NAME field

  # The name of the log format
  log.format.format_label = "SafeSquid Combined/Extended Log Format"
  log.miscellaneous.log_data_type = "firewall"  
  log.miscellaneous.log_format_type = "proxy_server"

  #30478 69.684.66.679 "gverde" "gverde" [03/Jun/2006:06:43:21] "GET http://mail.elsewhere.com:80/mail/channel/bind?&at=cbee713af5c68362-10b989ace15&RID=rpc&SID=FE560EF93340DC2&CI=1&AID=59&TYPE=xmlhttp&zx=pcjumd-b68ktd" 200 41 "" "Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.8.0.1) Gecko/20060111 Firefox/1.5.0.1" text/plain "-"
  #1665 prefetch "-" "-" [07/Jun/2006:07:55:59] "GET http://www.there.com:80/images/spacer.gif" 200 43 "" "" image/gif "-"
  #281 58.787.8.897 "jgreen" "jgreen" [07/Jun/2006:07:20:34] "GET http://download.microsoft.com:80/favicon.ico" 404 1635 "http://rad.microsoft.com/ADSAdClient31.dll?GetAd=&PG=CMSDLG&SC=F2&AP=1027" "Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.8.0.4) Gecko/20060508 Firefox/1.5.0.4" text/html "mime-filter,application/octet-stream"
  #5261 59.184.6.195 "oe" "oe" [07/Jun/2006:07:06:42] "GET http://download.microsoft.com:80/download/1/7/0/170c2a0b-ca56-4085-a76e-03cc413280c3/q331906.exe" 200 419968 "http://rad.microsoft.com/ADSAdClient31.dll?GetAd=&PG=CMSDLG&SC=F2&AP=1027" "Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.8.0.4) Gecko/20060508 Firefox/1.5.0.4" application/octet-stream "mime-filter,application/octet-stream"

  # 2007-12-21 - GMF - New format example:
  #"1192087876.252-11-192.168.0.210-8080" 603 192.168.0.21 "someone" "11" [11/Oct/2007:13:01:16] "GET http://image.somedomain.com:80/somefile.html" 200 50 "http://www.referringdomain.com/html/anotherfile.html" "Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.7.12) Gecko/20050915 Firefox/1.0.7" image/gif "- -" "http,virus_scan,uncachable,linked-urls,anti-popup,BlockJavaScript,BlockActiveX,Mozilla_Firefox" "192.168.0.210:8080"
  # SafeSquid describes this format as:
  #"UNIQUE_RECORDID" ELAPSED_TIME_IN_MSEC CLIENT_IP "USER_NAME" "CLIENT_CONNECTION_ID" [DATE_TIME_OF_REQUEST] "METHOD URL" "HTTP_STATUS_CODE" BYTES_TRANSFERRED "REFERRER_URL" "USER_AGENT" MIME_TYPE "FILTER_NAME FILTERING_REASON" "COMMA_SEPARATED_LIST_OF_PROFILES_APPLIED" "INTERFACE_IP:INTERFACE_PORT"

  # The log is in this format if any of the first ten lines match this regular expression
  # 2007-12-21 - GMF - Added detection of new format
  log.format.autodetect_expression = `
matches_regular_expression(volatile.log_data_line, '^[0-9]+ ([0-9.]+|prefetch) "([a-zA-Z][a-zA-Z0-9._]+|-)" "([a-zA-Z][a-zA-Z0-9._]+|-)" \\[../.../....:..:..:..\\] "[A-Z]+ [A-Za-z0-9]+://[^"]+" [-0-9]+ [-0-9]+ "[^"]*" "[^"]*" [^/ ]+/[^ ]+ "([^",]*,[^"]*|-)"$') or
matches_regular_expression(volatile.log_data_line, '^"[^"]+" [0-9]+ ([0-9.]+|prefetch) "([^"]+)" "([^"]+)" \\[../.../....:..:..:..\\] "[A-Z]+ [A-Za-z0-9]+://[^"]+" [-0-9]+ [-0-9]+ "[^"]*" "[^"]*" [^/ ]+/[^ ]+ "[^"]*" "[^"]+" "[0-9.]+:[0-9]+"')
`


  # Treat fields surrounded by square brackets (e.g. the date/time field) as a single quoted field.
  log.format.treat_brackets_as_quotes = "true"
  log.format.common_log_format = "true"
  log.format.parse_only_with_filters = "true"

  # The format of dates and times in this log
  log.format.date_format = "dd/mmm/yyyy:hh:mm:ss"
  log.format.time_format = "dd/mmm/yyyy:hh:mm:ss"

  # Log fields
  log.fields = {

    events = ""
    events_profile = ""

    unique_record_id = ""
    elapsed_time = ""
    elapsed_time_profile = ""
    client_ip.type = "host"
    client_connection_id = ""
    user_name = ""
    date_time = ""
    method = ""
    url = {
      type = "page"
      hierarchy_dividers = "/?"
      left_to_right = true
      leading_divider = "true"
    } # url

    http_status_code = ""

    bytes_transferred = ""
    bytes_transferred_profile = ""

    referrer = {
      type = "URL"
      hierarchy_dividers = "/?"
      left_to_right = true
      leading_divider = "false"
    } # referrer

    user_agent = {
      type = "agent"
    } # user_agent

    mime_type = ""
    filter_name = ""
    filter_reason = ""
    profile = ""
    interface_ip = ""
    interface_port = ""

  } # log.fields


  # Database fields
  database.fields = {

    date_time = ""
    day_of_week = ""
    hour_of_day = ""

    client_ip = ""

    # Complex fields disabled by default for best performance.
#    unique_record_id = ""
#    client_connection_id = ""

    url = {
      suppress_top = 1
      suppress_bottom = 3
    } # url

    file_type = ""

    method = ""
    worm = ""

    user_name = ""

    web_browser = ""

    operating_system = ""
    spider = ""

    search_phrase = ""

    search_engine = ""

    http_status_code = ""

    referrer  = {
      suppress_top = 1
      suppress_bottom = 3
    } # referrer 

	mime_type = ""

	filter_name = ""

	filter_reason = ""
    profile = ""
    interface_ip = ""
    interface_port = ""

  } # database.fields


  # Log Filters
  log.filters = {

    remove_query = {
      label = "$lang_admin.log_filters.remove_query_label"
      comment = "$lang_admin.log_filters.remove_query_comment"
      value = "if (contains(url, '?')) then url = substr(url, 0, index(url, '?') + 1) . '(parameters)';"
    } # remove_query

    detect_page_views = {
      label = '$lang_admin.log_filters.detect_page_views_label'
      comment = '$lang_admin.log_filters.detect_page_views_comment'
      value = "if ((file_type eq 'JPEG') or (file_type eq 'JPG') or (file_type eq 'GIF') or (file_type eq 'ICO') or (file_type eq 'PNG') or (file_type eq 'CSS') or (file_type eq 'SWF') or (file_type eq 'JS')) then page_views = 0; else page_views = 1;"
    } # detect_page_views

    simplify_url = {
      label = "$lang_admin.log_filters.simplify_url_label"
      comment = "$lang_admin.log_filters.simplify_url_comment"
      value = "(if (file_type) then '' else ''); if (matches_regular_expression(url, '^([^:]+://[^/]+/)')) then url = $1 . '(omitted)'"
    } # simplify_url

    strip_non_page_views = {
      label = '$lang_admin.log_filters.strip_non_page_views_label'
      comment = '$lang_admin.log_filters.strip_non_page_views_comment'
      value = "if (page_views == 0) then url = substr(url, 0, last_index(url, '/') + 1) . '(nonpage)';"
    } # strip_non_page_views

    not_authenticated = {
      label = "$lang_admin.log_filters.not_authenticated_label"
      comment = "$lang_admin.log_filters.not_authenticated_comment"
      value = "if (user_name eq '-') then user_name = '(not authenticated)';"
    } # not_authenticated

  } # log.filters


  log.parsing_filters.parse = `

# 2007-12-21 - GMF - Handle old format
if (matches_regular_expression(current_log_line(), '^([0-9]+) ([0-9.]+|prefetch) "([a-zA-Z][a-zA-Z0-9._]+|-)" "([a-zA-Z][a-zA-Z0-9._]+|-)" \\[(../.../....:..:..:..)\\] "([A-Z]+) ([A-Za-z0-9]+://[^"]+)" ([-0-9]+) ([-0-9]+) "([^"]*)" "([^"]*)" ([^/ ]+/[^ ]+) "([^",]*,[^"]*|-)"$')) then (

  set_collected_field('', 'client_ip', $2);
  set_collected_field('', 'user_name', $3);
  set_collected_field('', 'date_time', $5);
  set_collected_field('', 'method', $6);
  set_collected_field('', 'url', $7);
  set_collected_field('', 'http_status_code', $8);
  set_collected_field('', 'bytes_transferred', $9);
  set_collected_field('', 'referrer', $10);
  set_collected_field('', 'user_agent', $11);
  set_collected_field('', 'mime_type', $12);
  set_collected_field('', 'v.filter', $13);

  # Extract filter
  if (matches_regular_expression(v.filter, '([^,]+),(.*)')) then (
    set_collected_field('', 'filter_name', $1);
    set_collected_field('', 'filter_reason', $2);
  );
  
  accept_collected_entry('', false);
  
); # if old format

# 2007-12-21 - GMF - Handle new format
else if (matches_regular_expression(current_log_line(), '^"([^"]+)" ([0-9]+) ([0-9.]+|prefetch) "([^"]+)" "([^"]+)" \\[(../.../....:..:..:..)\\] "([A-Z]+) ([A-Za-z0-9]+://[^"]+)" ([-0-9]+) ([-0-9]+) "([^"]*)" "([^"]*)" ([^/ ]+/[^ ]+) "([^ ]+) ([^"]+)" "([^"]+)" "([0-9.]+):([0-9]+)"')) then (

  set_collected_field('', 'unique_record_id', $1);
  v.elapsed_time = $2;
  set_collected_field('', 'client_ip', $3);
  set_collected_field('', 'user_name', $4);
  set_collected_field('', 'client_connection_id', $5);
  set_collected_field('', 'date_time', $6);
  set_collected_field('', 'method', $7);
  set_collected_field('', 'url', $8);
  set_collected_field('', 'http_status_code', $9);
  v.bytes_transferred = $10;
  set_collected_field('', 'referrer', $11);
  set_collected_field('', 'user_agent', $12);
  set_collected_field('', 'mime_type', $13);
  set_collected_field('', 'filter_name', $14);
  set_collected_field('', 'filter_reason', $15);
  v.profiles = $16;
#  set_collected_field('', 'profiles', $16);
  set_collected_field('', 'interface_ip', $17);
  set_collected_field('', 'interface_port', $18);

  while (matches_regular_expression(v.profiles, '^([^,]+),(.*)$') or
         matches_regular_expression(v.profiles, '^([^,]+)()$')) (
    set_collected_field('', 'profile', $1);
    v.profiles = $2;
    set_collected_field('', 'bytes_transferred', 0);
    set_collected_field('', 'bytes_transferred_profile', v.bytes_transferred);
    set_collected_field('', 'events', 0);
    set_collected_field('', 'events_profile', 1);
    set_collected_field('', 'elapsed_time', 0);
    set_collected_field('', 'elapsed_time_profile', v.elapsed_time);
    accept_collected_entry('', true);
  );

  set_collected_field('', 'profile', '');

  set_collected_field('', 'bytes_transferred', v.bytes_transferred);
  set_collected_field('', 'bytes_transferred_profile', 0);
  set_collected_field('', 'events', 1);
  set_collected_field('', 'events_profile', 0);
  set_collected_field('', 'elapsed_time', v.elapsed_time);
  set_collected_field('', 'elapsed_time_profile', 0);

#  set_collected_field('', 'bytes_transferred', 0);
  accept_collected_entry('', false);

); # if new format  

`

  # Get web browser, operating system, web browser, and spider information from the user-agent field.
  log.parsing_filters.derive_from_user_agent = `
get_user_agent_info(user_agent);
web_browser = volatile.web_browser;
operating_system = volatile.operating_system;
spider = volatile.spider;
`


  database.numerical_fields = {

    events = ""
    events_profile = ""

    page_views = {
      default = true
      requires_log_field = false
      type = "int"
      display_format_type = "integer"
    } # page_views

    unique_client_ips = {
      requires_log_field = true
      log_field = "client_ip"
      type = "unique"
      display_format_type = "integer"
    } # unique_client_ips

    bytes_transferred = {
      requires_log_field = true
      type = "float"
      display_format_type = "bandwidth"
    } # bytes_transferred

    bytes_transferred_profile = {
      requires_log_field = true
      type = "float"
      display_format_type = "bandwidth"
    } # bytes_transferred_profile

    elapsed_time = {
      type = "float"
      display_format_type = "duration_milliseconds"
    } # elapsed_time

    elapsed_time_profile = {
      type = "float"
      display_format_type = "duration_milliseconds"
    } # elapsed_time_profile

  } # database.numerical_fields

  create_profile_wizard_options = {

    database_field_associations = {

      profile = {
        elapsed_time_profile = true
        bytes_transferred_profile = true
        events_profile = true
      }

    }

    # How the reports should be grouped in the report menu
    report_groups = {
      date_time_group = ""
      content_group = {
        url = true
        file_type = true
        mime_type = true
      }
      source_group = {
        client_ip = true
        user_name = true
        web_browser = true
        operating_system = true
        spider = true
      }
      referrer_group = {
        referrer = true
        search_phrase = true
        search_engine = true
      }
      other_group = {
        worm = true
        method = true
        http_status_code = true
        filter_name = true
        filter_reason = true
        interface_ip = true
        interface_port = true
        profile = true
      }
    } # report_groups

  } # create_profile_wizard_options

} # safesquid_combined