# Copyright (c) 2010 Flowerfire, Inc.  All Rights Reserved.
microsoft_media_server = {

  plugin_version = "1.20"

  info.1.manfacturer = "Microsoft"
  info.1.device = "Media Server"
  info.1.version.1 = "4.1"
  info.1.version.2 = "9"

  # 2006-03-10 - 1.1 - GMF - added play duration per visitor, and play duration per clip
  # 2006-07-20 - 1.2 - GMF - modified the login/logout calculation to use the timestamp as login time,
  # rather than logout time
  # 2006-08-25 - 1.3 - GMF - Removed copying of cs_uri_stem to logout event, as an easy way to handle
  # cs_uri_stem being absent.
  # 2007-03-20 -     - GMF - Reversed 1.3. If cs_uri_stem is not copied, it causes incorrect results
  # (duration mismatch) when there's a cs_uri_stem log filter (sspec profile).
  # 2006-11-10 - 1.4 - GMF - Added "successful clips" field to track clips with c_status 2xx.
  # 2006-11-16 - 1.5 - GMF - Changed "successful clips" to "clips", and changed "clips" to "events"
  # 2007-03-26 - 1.6 - GMF - Fixed bug which would give incorrect session durations if
  # date_offset was not 0.
  # 2007-05-21 - 1.7 - KBB - Added support for newer formats in which the field cs_uri_stem has been
  # renamed cs_url. The name cs_uri_stem is still used here, even though cs_url is the more correct
  # name, because it is treated as type page and cs_url is not. Both formats can be processed together.
  # 2007-06-05 - 1.8 - KBB - Added a log filter to keep only the last element of the URL. It is disabled
  # by default.
  # 2007-07-02 - 1.9 - KBB - Removed sessions reports that don't work with special sessions reporting.
  # 2007-07-19 - 1.10 - GMF - Added x-duration check for ridiculously large values/
  # 2007-07-24 - 1.11 - GMF - Added experimental support for playlists
  # 2007-08-06 - 1.12 - MSG - Added log filter which parses the web browser out of user_agent when
  # there are underbars.
  # 2007-08-06 - 1.13 - MSG - Fixed a bug where the first login event did not have a c_ip.
  # 2008-04-02 - 1.14 - GMF - Added simplify playerid filter.
  # 2008-11-24 - 1.15 - MSG - Changed value of average_denominator_field = "clips" to "events"
  # (except for the x_duration_per_clip field.)
  # 2009-03-17 - 1.16 - KBB - Corrected typo that was breaking sessioning (sessions_id -> session_id)
  # and restored session_id to database.fields. Changed clips to successful_accesses.
  # Removed c_rate from numeric fields (see comment below).
  # 2009-05-08 - 1.16.1 - KBB - Also added session_id to log.fields.
  # 2009-12-04 - 1.17 - GMF - Changed x_duration_per_successful_access to x_duration_per_success_access
  # 2010-06-04 - 1.18 - GMF - Added requires_fields clause for session_events
  # 2010-12-28 - 1.19 - GMF - Changed session_events to stream_start_stop_events to differentiate it from ssession_events
  # 2011-03-28 - 1.20 - GMF - Removed filesize field because summing it makes no sense.  Non-aggregated might make sense, but removing it for now for simplicity, unless someone asks.

  # The name of the log format
  log.format.format_label = "Microsoft Media Server Log Format"
  log.miscellaneous.log_data_type = "generic_w3c"  
  log.miscellaneous.log_format_type = "media_server"

  # The log is in this format if any of the first ten lines match this regular expression
  #log.format.autodetect_regular_expression = "^#Software: (Microsoft|Windows) Media Services"
  # Exclude vbrick_ethernetv_portal_server
  log.format.autodetect_expression = `
if (matches_regular_expression(volatile.log_data_line, "^#Software: (Microsoft|Windows) Media Services")) then (
  v.seen_microsoft_media_server = true;
  false;
);
else if (node_exists('v.seen_microsoft_media_server')) then ( # Can't initialize, so existence = truth
  if (matches_regular_expression(volatile.log_data_line, "^#Fields:( |	)") and (volatile.log_data_line ne "#Fields: cs-username c-ip cs-uri-stem x-ip x-port c-max-bandwidth x-type s-ip time date x-duration c-os c-osversion")) then (
    true;
  );
  else
    false;
);
else
  false;
`
  log.format.autodetect_lines = "20"

  # The format of dates and times in this log
  log.format.date_format = "yyyy-mm-dd"
  log.format.time_format = "hh:mm:ss"

  # Don't track these fields as discrete database fields
  # 2011-03-28 - GMF - 1.20 - Removing filesize; see note in database.fields
  auto_setup.omit_database_fields = "c_starttime,s_session_id,cs_url.filesize"

  # Ignore lines like "#EndDate: YYYY-MM-DD HH:MM:SS" when doing checksumming for skipping;
  # the server modifies these lines after the fact.
  log.processing.ignore_regexp_for_skip_checksum = "#EndDate: ....-..-.. ..:..:.."

  # Don't truncate long fields
  statistics.sizes.table_cell.maximum_text_length = 0

  # Don't time out session, discard them for being long, or remove reloads
  statistics.miscellaneous = {
    maximum_session_duration = "0"
    session_timeout = "0"
    remove_reloads_from_sessions = "false"
  } # statistics.miscellaneous

  log.parsing_filters.copy_cs_url = {
    value = `
if (cs_uri_stem eq '(empty)' and cs_url ne '(empty)') then (
  cs_uri_stem = cs_url;
);
`
  }

  log.parsing_filters.convert_user_agent = `cs_user_agent = replace_all(cs_user_agent, "_", " ")`

  # Get search engine and search phrase information from the referrer field (before it gets simplified).
  log.parsing_filters.compute_se_sp = {
    value = `
if (get_search_engine_info(referrer)) then (
  search_engine = volatile.search_engine;
  search_phrase = volatile.search_phrase;
);
`
    requires_fields = {
      referrer = true
    }
  }

  # Due to a bug in MSS, c-max-bandwidth sometimes appears as 4294967295; set it to 0 when that happens
  log.parsing_filters.fix_maximum_bandwidth = {
    value = `if (c_max_bandwidth > 2000000000) then c_max_bandwidth = 0`
    requires_fields = {
      c_max_bandwidth = true
    }
  }


  # Due to a bug in MSS, c-pkts-lost-net sometimes appears as 2013265920; set it to 0 when that happens
  log.parsing_filters.fix_c_pkts_lost_net = {
    value = `if (c_pkts_lost_net > 2000000000) then c_pkts_lost_net = 0`
    requires_fields = {
      c_pkts_lost_net = true
    }
  }

  # Due to a bug in MSS, c-pkts-lost-client sometimes appears as 2013265920; set it to 0 when that happens
  log.parsing_filters.fix_c_pkts_lost = {
    value = `if (c_pkts_lost_client > 2000000000) then c_pkts_lost_client = 0`
    requires_fields = {
      c_pkts_lost_client = true
    }
  }

  # Due to a bug in MSS, x-duration sometimes appears as a huge number; set it to 0 when that happens
  log.parsing_filters.fix_x_duration = {
    value = `if (x_duration > 2000000000) then x_duration = 0`
    requires_fields = {
      x_duration = true
    }
  }

#  # Set clips to 1 for 200-series status
#  log.parsing_filters.set_clips = {
#    value = `if ((c_status >= 200) and (c_status <= 299)) then clips = 1;`
  # Set successful_accesses to 1 for 200-series status
  log.parsing_filters.set_successful_accesses = {
    value = `if ((c_status >= 200) and (c_status <= 299)) then successful_accesses = 1;`
    requires_fields = {
      c_status = true
    }
  }

  log.filter_initialization = `
int login_date_time_epoc;
int logout_date_time_epoc;
string logout_date_time;
string session_id;
int session_id_counter = 0;
int date_offset_seconds = log.processing.date_offset * (60*60);

## 2007-07-24 - GMF - Experimental code for handling playlists
#int date_time_as_epoc;
#node pe;
#int duration;
`

  # This filter adds a "logout" event at the logged time, so sessions can be calculated based
  # on connection time.  The "normal" event will be added normally, but this also subtracts the
  # duration from the date and time, so it logs the event at the time it *connected*, and the
  # logout at the time it *disconnected*.
  log.parsing_filters.add_logout_event = {

    value = `

# Compute the session id
session_id = c_ip . '_' . session_id_counter;
session_id_counter++;

# Add the login event.  This has the same c_ip, cs_uri_stem as the logout event, so it can be
# correllated that way.
# 2008-04-01 - GMF - Added c_ip, because without it the first hit in the database has (empty) as the c_ip.
set_collected_field('', 'date', date);
set_collected_field('', 'time', time);
set_collected_field('', 'c_ip', c_ip);
set_collected_field('', 'session_event_type', 'login');
set_collected_field('', 'session_id', session_id);
set_collected_field('', 'cs_uri_stem', cs_uri_stem);
set_collected_field('', 'stream_start_stop_events', 1);
set_collected_field('', 'events', 0);
accept_collected_entry('', false);

# Compute the date_time of the login event.
# Note that we need to add date_offset_seconds to apply the date_offset value.
login_date_time_epoc = date_time_to_epoc(normalize_date(date, 'auto') . ' ' . time);
login_date_time_epoc += date_offset_seconds;
logout_date_time_epoc = login_date_time_epoc + x_duration;
logout_date_time = epoc_to_date_time(logout_date_time_epoc);

# Set up to make the next event a logout
session_event_type = '(logout)';
stream_start_stop_events = 1;
events = 1;

`
      requires_fields = {
        date = true
        time = true
        c_ip = true
        x_duration = true
        stream_start_stop_events = true
      }

  } # add_logout_event

  # Log Filters
  log.filters = {

    login = {
      label = "Handle session information"
      comment = "This performs internal bookkeeping required to treat each access as a separate session"
      value = `
if (session_event_type eq '(logout)') then (
    date_time = logout_date_time;
);
`
    } # login

    last_url_elements = {
      label = "$lang_admin.log_filters.last_url_elements_label"
      comment = "$lang_admin.log_filters.last_url_elements_comment"
      value = `
#v.match = 'www.mydomain.com|mydirectory1|mydirectory2'; # Example
#v.N = "5"; # Example - up to six elements, including last
#v.N = ""; # Example - use with a pattern to include everything after the pattern
v.match = "";
v.N = "0";
if (matches_regular_expression(cs_uri_stem, '(' . v.match . ')[^/]*/(([^/]*/){0,' . v.N . '}[^/]*)$')) then (
  cs_uri_stem = $2;
);
`
      disabled = "true"
    } # last_url_elements

#    # 2007-07-24 - GMF - Experimental code for handling playlists
#    handle_playlist = `
##echo("playlist: " . node_as_string(playlist));
#date_time_as_epoc = date_time_to_epoc(date_time);
##echo("date_time_as_epoc: " . date_time_as_epoc);
#foreach pe 'playlist' (
##echo("pe: " . node_as_string(pe));
#  if (date_time_as_epoc > node_name(pe)) then (
##echo("PAST BEGINNING");
#    duration = node_value(subnode_by_name(pe, "duration"));
##echo("duration: " . duration);
#    if (date_time_as_epoc <= (node_name(pe) + duration)) then (
##echo("BEFORE END");
#      song = node_value(subnode_by_name(pe, "song"));
#    ); # if event before end of song
#  ); # if event after start of song
#); # foreach pe
#echo("Song playing: " . song);
#`

    simplify_referrer = {
      label = "$lang_admin.log_filters.simplify_referrer_label"
      comment = "$lang_admin.log_filters.simplify_referrer_comment"
      value = "if (referrer eq '-') then referrer = '(no referrer)' else if (matches_regular_expression(referrer, '^([^:]+://[^/]+/)')) then referrer = $1 . '(omitted)'"
      requires_fields = {
        referrer = true
      }
    } # simplify_referrer

    simplify_playerid = {
      label = "$lang_admin.log_filters.simplify_playerid_label"
      comment = "$lang_admin.log_filters.simplify_playerid_comment"
      value = "c_playerid = '(simplified by log filter)'"
      requires_fields = {
        c_playerid = true
      }
    } # simplify_playerid

    internal_referrer = {
      label = "$lang_admin.log_filters.internal_referrer_label"
      comment = "$lang_admin.log_filters.internal_referrer_comment"
      value = "if (contains(referrer, 'mydomain.com/')) then referrer = '(internal referrer)';"
      disabled = true
      requires_fields = {
        referrer = true
      }
    } # internal_referrer

    remove_query = {
      label = "$lang_admin.log_filters.remove_query_label"
      comment = "$lang_admin.log_filters.remove_query_comment"
      value = "if (contains(cs_uri_stem, '?')) then cs_uri_stem = substr(cs_uri_stem, 0, index(cs_uri_stem, '?') + 1) . '(parameters)';"
      requires_fields = {
        cs_uri_stem = true
      }
    } # remove_query

#    mark_entry = {
#      label = '$lang_admin.log_filters.mark_entry_label'
#      comment = '$lang_admin.log_filters.mark_entry_comment'
#      value = 'stream_start_stop_events = 1; clips = 1;'
#    } # mark_entry

  } # log.filters

  # Force there to be host and date_time fields in advance, so we get prompted for day-by-day and bottom-level
#  log.fields.c_ip = {
#      type = "host"
 #     label = "$lang_stats.field_labels.c_ip"
#      index = "1"
#      subindex = "0"
#      hierarchy_dividers = "."
#      left_to_right_hierarchy = "false"
#      leading_divider = "false"
#      case_sensitive = "false"
#      is_sessions_visitor_id = "true"
#  } # c_ip
#
#  log.fields.date_time = {
#      label = "$lang_stats.field_labels.date_time"
#      type = "date_time"
#      derived_from_1 = "date"
#      derived_from_2 = "time"
#      index = "0"
#      subindex = "0"
#  } # date_time

  log.fields = {

	session_id = ""
    session_event_type = ""
#    clips = ""
    successful_accesses = ""
    events = ""

#    # 2007-07-24 - GMF - Experimental code for handling playlists
#    song = ""

    # One or the other of these might exist, so create fields explicitly
    cs_uri_stem.type = "page"
    cs_url.type = "flat"

  } # log.fields

  log.field_options = {

    sessions_page_field = "session_event_type"
    sessions_visitor_id_field = "c_ip"
    sessions_event_field = "stream_start_stop_events"
    sessions_id_field = "session_id"

  } # log.field_options


  database.fields = {
  
    session_event_type = ""

    # 2008-12-08 - GMF - Removing session_id as database field because it is now a session field
    # 2009-03-18 - KBB - Restored this since results are wrong without it.
    session_id = ""

    cs_uri_stem.suppress_bottom = "20"
    cs_uri_stem.itemnums_hash_function = "rand_sum"

#    # 2007-07-24 - GMF - Experimental code for handling playlists
#    song = ""

  } # database.fields

  database.numerical_fields = {

    events = {
      label = "$lang_stats.field_labels.events"
      default = true
      requires_log_field = false
      type = "int"
      display_format_type = "integer"
      entries_field = true
    } # events

#    clips = {
#      default = true
#    } # clips
    successful_accesses = {
      default = true
    } # successful_accesses

#    page_views = {
#      label = "$lang_stats.field_labels.page_views"
#      default = true
#      requires_log_field = false
#      type = "int"
#      display_format_type = "integer"
#    } # page_views

    visitors = {
      label = "$lang_stats.field_labels.visitors"
      default = false
      requires_log_field = true
      log_field = "c_ip"
      type = "unique"
      display_format_type = "integer"
    } # visitors

    x_duration = {
      default = false
      requires_log_field = false
      type = "float"
      display_format_type = "duration_compact"
    }

    x_duration_per_visitor = {
      default = false
      type = "float"
      display_format_type = "duration_compact"
      log_field = "x_duration"
      aggregation_method = "average"
      average_denominator_field = "visitors"
    }

#    x_duration_per_clip = {
    # 2009-12-04 - GMF - shunk name from x_duration_per_successful_access to x_duration_per_success_access so it fits under the 30-character Oracle identifier limit
    x_duration_per_success_access = {
      default = false
      requires_log_field = false
      type = "float"
      log_field = "x_duration"
      display_format_type = "duration_compact"
      aggregation_method = "average"
#      average_denominator_field = "clips"
      average_denominator_field = "successful_accesses"
    }

# 2009-03-17 - KBB
#
# http://www.microsoft.com/windows/windowsmedia/howto/articles/loggingmodel.aspx
#
# c-rate: The rate at which data is sent from the server to the client.
# Possible values:
#
# 0.5: Half of the real-time rate.
#
# 1: Real-time rate
#
# 2: Twice as fast as real-time
#
# 5: fast forward
#
# -5: fast rewind
#
# If you are using Fast Streaming, these values could be considerably higher or lower
# depending on the content and the available bandwidth.

#    c_rate = {
#     label = "$lang_stats.field_labels.c_rate"
#     default = false
#     requires_log_field = false
#     type = "int"
#     display_format_type = "integer"
#     aggregation_method = "average"
#     average_denominator_field = "events"
#   }

    filelength = {
      label = "$lang_stats.field_labels.filelength"
      default = false
      requires_log_field = false
      type = "float"
      display_format_type = "bandwidth"
    }

    # 2011-03-28 - GMF - 1.20 - Removing filesize field because summing it makes no sense.  Non-aggregated might make sense, but removing it for now for simplicity, unless someone asks.
#    filesize = {
#      label = "$lang_stats.field_labels.filesize"
#      default = false
#      requires_log_field = false
#      type = "float"
#      display_format_type = "bandwidth"
#    }

    avgbandwidth = {
      label = "$lang_stats.field_labels.avgbandwidth"
      default = false
      requires_log_field = false
      type = "float"
      display_format_type = "bandwidth"
      aggregation_method = "average"
      average_denominator_field = "events"
    }

    sc_bytes = { 
      label = "$lang_stats.field_labels.sc_bytes"
      default = false
      requires_log_field = false
      type = "float"
      display_format_type = "bandwidth"
    }

    cs_bytes = {
      label = "$lang_stats.field_labels.cs_bytes"
      default = false
      requires_log_field = false
      type = "float"
      display_format_type = "bandwidth"
    }

    c_bytes = {
      label = "$lang_stats.field_labels.c_bytes"
      default = false
      requires_log_field = false
      type = "float"
      display_format_type = "bandwidth"
    }

    s_pkts_sent = {
      label = "$lang_stats.field_labels.s_pkts_sent"
      default = false
      requires_log_field = false
      type = int
      display_format_type = integer
    }

    c_pkts_received = {
      label = "$lang_stats.field_labels.c_pkts_received"
      default = false
      requires_log_field = false
      type = "int"
      display_format_type = "integer"
    }

    c_pkts_lost_client = {
      label = "$lang_stats.field_labels.c_pkts_lost_client"
      default = false
      requires_log_field = false
      type = "int"
      display_format_type = "integer"
    }

    c_pkts_lost_net = {
      label = "$lang_stats.field_labels.c_pkts_lost_net"
      default = false
      requires_log_field = false
      type = "int"
      display_format_type = "integer"
    }

    c_pkts_lost_cont_net = {
      label = "$lang_stats.field_labels.c_pkts_lost_cont_net"
      default = false
      requires_log_field = false
      type = "int"
      display_format_type = "integer"
    }

    c_resendreqs = {
      label = "$lang_stats.field_labels.c_resendreqs"
      default = false
      requires_log_field = false
      type = "int"
      display_format_type = "integer"
    }

    c_pkts_recovered_ecc = {
      label = "$lang_stats.field_labels.c_pkts_recovered_ecc"
      default = false
      requires_log_field = false
      type = "int"
      display_format_type = "integer"
    }

    c_pkts_recovered_resent = {
      label = "$lang_stats.field_labels.c_pkts_recovered_resent"
      default = false
      requires_log_field = false
      type = "int"
      display_format_type = "integer"
    }

    c_buffercount = {
      label = "$lang_stats.field_labels.c_buffercount"
      default = false
      requires_log_field = false
      type = "int"
      display_format_type = "integer"
    }

    c_totalbuffertime = {
      label = "$lang_stats.field_labels.c_totalbuffertime"
      default = false
      requires_log_field = false
      type = "int"
      display_format_type = "duration_compact"
    }

    c_quality = {
      label = "$lang_stats.field_labels.c_quality"
      default = false
      requires_log_field = false
      type = "int"
      display_format_type = "integer"
      aggregation_method = "average"
      average_denominator_field = "events"
    }

    c_totalclients = {
      label = "$lang_stats.field_labels.c_totalclients"
      default = false
      requires_log_field = false
      type = "int"
      display_format_type = "integer"
      aggregation_method = "average"
      average_denominator_field = "events"
    }

    s_totalclients = {
      label = "$lang_stats.field_labels.s_totalclients"
      default = false
      requires_log_field = false
      type = "int"
      log_field = "s_totalclients"
      display_format_type = "integer"
      aggregation_operator = "maximum"
    } # s_totalclients

   s_cpu_util = {
      label = "$lang_stats.field_labels.s_cpu_util"
      default = false
      requires_log_field = false
      type = "int"
      display_format_type = "integer"
      aggregation_method = "average"
      average_denominator_field = "events"
    }

    c_max_bandwidth = {
      label = "$lang_stats.field_labels.c_max_bandwidth"
      default = false
      requires_log_field = false
      type = "float"
      display_format_type = "bandwidth"
      aggregation_method = "max"
    }

    stream_start_stop_events = {
      label = "$lang_stats.field_labels.stream_start_stop_events"
      default = true
      requires_log_field = false
      type = "int"
      display_format_type = "integer"
      entries_field = true
    } # stream_start_stop_events

  } # database.numerical_fields


  create_profile_wizard_options = {
    date_time_tracking = true
    host_tracking = true

    # How the reports should be grouped in the report menu
    report_groups = {
      date_time_group = ""
      content_group = {
        cs_uri_stem = true
        #cs_url = true
        cs_uri_query = true
        file_type = true
        audiocodec = true
        videocodec = true
        sc_realm = true
        cs_media_name = true
        cs_media_role = true
      } # content_group
      server_group = {
        s_ip = true
        s_dns = true
        s_content_path = true
      } # server_group
      visitor_demographics_group = {
        c_ip = true
        domain_description = true
        location = true
        organization = true
        domain = true
        isp = true
        c_dns = true
        c_username = true
        cs_user_name = true
      }
      referrer_group = {
        referrer = true
        search_engine = true
        search_phrase = true
        referrer_description = true
        search_phrase_by_search_engine = true
      }
      visitor_systems_group = {
        screen_dimensions = true
        screen_depth = true
        c_playerid = true
        c_playerversion = true
        c_playerlanguage = true
        web_browser = true
        operating_system = true
        spider = true
        c_os = true
        c_osversion = true
        c_cpu = true
        c_hostexe = true
        c_hostexever = true
        channelurl = true
        c_rate = true
      }
      other_group = {
        worm = true
        c_status = true
        protocol = true
        transport = true
        s_proxied = true
        #session_event_type = true
        #session_id = true
      }
    } # report_groups

    final_step = `

include "templates.admin.profiles.setup_reports_util";

string profile = "profiles." . volatile.new_profile_name;
string reports = profile . ".statistics.reports";
string menu = profile . ".statistics.reports_menu";

# Create the standard reports
add_standard_reports(profile);

# Remove irrelevant sessions reports
delete_node(reports . ".entry_pages");
delete_node(menu . ".sessions_group.items.entry_pages");
delete_node(reports . ".exit_pages");
delete_node(menu . ".sessions_group.items.exit_pages");
delete_node(reports . ".session_pages");
delete_node(menu . ".sessions_group.items.session_pages");
delete_node(reports . ".session_page_paths");
delete_node(menu . ".sessions_group.items.session_page_paths");
delete_node(reports . ".session_paths");
delete_node(menu . ".sessions_group.items.session_paths");
delete_node(menu . ".session_event_type"); # always login or (logout)
delete_node(menu . ".session_id"); # duplicates session users report
`

  } # create_profile_wizard_options

} # microsoft_media_server