beta_helix_universal = { info.1.manfacturer = "Real" info.1.device = "Helix Universal Proxy" info.1.version.1 = "11" plugin_version = "1.0.1beta" # 2006-09-20 - gas - Created initial plug-in, based on log format detailed here: # http://service.real.com/help/library/guides/helixuniversalproxy/proxy9.htm?page=htmfiles/tracking.htm%23acc_log_format # 2006-10-08 - GMF - 1.0.1beta - Changes sent_time to seconds, according to docs (see comment on numerical field, below). # The name of the log format log.format.format_label = "Helix Universal Proxy Server (Types: 0,1,2,3,4,5) Log Format" log.miscellaneous.log_data_type = "http_access" log.miscellaneous.log_format_type = "proxy_server" # The log is in this format if any of the first ten lines match this regular expression log.format.autodetect_regular_expression = '^[^ ]+ - - \\[[0-9]+/[A-Za-z]+/[0-9]+:[0-9]+:[0-9]+:[0-9]+ [+-][0-9]+\\] *\\"[^"]+\\" [0-9]+ [0-9]+ \\[[^]]+\\]' # Treat fields surrounded by square brackets (e.g. the date/time field) as a single quoted field. log.format.treat_brackets_as_quotes = "true" # The format of dates and times in this log log.format.date_format = "dd/mmm/yyyy:hh:mm:ss" log.format.time_format = "dd/mmm/yyyy:hh:mm:ss" # Format: parse_only_with_filters ( true | false ) log.format.parse_only_with_filters = "true" # Log fields - IP_address - - [timestamp] "GET filename protocol/version" HTTP_status_code bytes_sent [client_info] [client_ID] [client_stats_results] file_size file_time sent_time resends failed_resends presentation_id log.fields = { # 0,1,2,3,4,5 hostname.type = "host" date_time.type = "date_time" operation = "" file = "" protocol = "" server_response = "" bytes_sent.type = "size" client_info.type = "agent" # 2,3,4,5 client_id = "" # 1,2,3,4,5 client_stats_results = "" file_size = "" file_time = "" sent_time = "" resends = "" failed_resends = "" # 3,4 stream_components = "" start_time = "" server_address = "" # 4 average_bitrate = "" packets_sent = "" # 5 presentation_id = "" # 1,2,3,4,5 proxy_info = "" } # log.fields # Database fields database.fields = { date_time = "" day_of_week = "" hour_of_day = "" hostname = "" operation = "" file = "" protocol = "" server_response = "" client_info = "" client_id = "" client_stats_results = "" file_size = "" file_time = "" sent_time = "" resends = "" failed_resends = "" stream_components = "" start_time = "" server_address = "" average_bitrate = "" packets_sent = "" presentation_id = "" proxy_info = "" } # database.fields # Log Filters log.filters = { cleanup_file = { label = "Clean up file" comment = "Handle empty file field" value = "if (matches_regular_expression(file, '^[A-Z]+/[0-9.]+$')) then file = ''" } # cleanup_file # stat1 = { # label = "Stat1" # comment = "Extract stat1 subfields" # value = "if (matches_regular_expression(client_stats_results, 'Stat1: +([0-9]+) +([0-9]+) +([0-9]+) +([0-9]+) +([0-9]+) +([^]]+)\\]')) then ( received = $1; out_of_order = $2; missing = $3; early = $4; late = $5; codec = $6 )" # } # stat1 # # stat2 = { # label = "Stat2" # comment = "Extract Stat2 subfields" # value = "if (matches_regular_expression(client_stats_results, '\\[Stat2: +([0-9]+) +([0-9]+) +([0-9]+) +([0-9]+) +([0-9]+) +([0-9]+) +([0-9]+) +([0-9]+) +([0-9]+) +([0-9]+) +([0-9]+) +([^]]+)')) then ( bandwidth = $1; available = $2; highest = $3; lowest = $4; average = $5; requested = $6; received = $7; late = $8; rebuffering = $9; transport = $10; startup = $11; codec = $12 )" # } # stat2 # # stat4 = { # label = "Stat4" # comment = "Extract Stat4 subfields" # value = "if (matches_regular_expression(client_stats_results, '\\[Stat4:([^|]+)\\|([^|]+)\\|([^|]+)\\|([^|]+)\\|([^|]+)\\|([^|]+)\\|([^|]+)\\|([^|]+)\\|([^|]+)\\|([^]]+)')) then ( stream_number = $1; mime_type = $2; codec = $3; received = $4; lost = $5; resent = $6; average_bandwidth = $7; current_bandwidth = $8; transport = $9; turboplay = $10; duration = $11; clip_end = $12 )" # } # stat4 # # turboplay_1 = { # label = "turboplay 1" # comment = "" # value = "if (starts_with(turboplay, '1')) then turboplay = '1'" # } # turboplay_1 # # turboplay_2 = { # label = "compute turboplay" # comment = "" # value = "turboplay = node_value(subnode_by_name('lang_stats.log_formats.helix_universal.turboplay', turboplay))" # } # turboplay_2 # # transport = { # label = "compute transport" # comment = "" # value = "transport = node_value(subnode_by_name('lang_stats.log_formats.helix_universal.transport', transport))" # } # transport # # clip_end = { # label = "extract clip_end" # comment = "" # value = "clip_end = node_value(subnode_by_name('lang_stats.log_formats.helix_universal.clip_end', clip_end))" ## value = "clip_end = '$lang_stats.log_formats.helix_universal.clip_end.' . clip_end" # } # clip_end # # cpu = { # label = "cpu" # comment = "Extract cpu subfields" # value = "if (matches_regular_expression(cpu, '^([^_]*)_([^_]*)_([^_]*)_([^_]*)_([^_]*)_([^_]*)')) then ( client_info = $1; version = $2; client = $3; type = $4; distribution = $5; language = $6 )" # } # cpu # # not_authenticated = { # label = "$lang_admin.log_filters.not_authenticated_label" # comment = "$lang_admin.log_filters.not_authenticated_comment" # value = "if (authenticated_user eq '-') then authenticated_user = '(not authenticated)';" # } # not_authenticated # mark_entry = { label = '$lang_admin.log_filters.mark_entry_label' comment = '$lang_admin.log_filters.mark_entry_comment' value = 'clips = 1;' } # mark_entry } # log.filters # Parsing Filters log.parsing_filters.parse = ` # there are 6 known types for this server, 0,1,2,3,4,5, the customer log this was created from differred from any of these, so I have listed this format first.gas # type custom1 if (matches_regular_expression(current_log_line(), '^([^ ]+) - - \\[([0-9]+/[A-Za-z]+/[0-9]+:[0-9]+:[0-9]+:[0-9]+) [+-][0-9]+\\] *\\"([^"]+)\\" ([0-9]+) ([0-9]+) \\[([^]]+)\\] \\[([^]]+)\\] \\[([^]]+)\\] ([0-9]+) ([0-9]+) ([0-9]+) ([0-9]+) ([0-9]+) \\[([^]]+)\\] \\[([^]]+)\\] ([^ ]+) ([0-9]+) ([0-9]+) [0-9]+ [0-9]+ [0-9]+ \\[([^]]+)\\]$')) then ( set_collected_field('', 'hostname', $1); set_collected_field('', 'date_time', $2); if (matches_regular_expression($3, '^(GET|POST) ([^ ]+) ([^$]+)')) then ( set_collected_field('', 'operation', $1); set_collected_field('', 'file', $2); set_collected_field('', 'protocol', $3); ); set_collected_field('', 'server_response', $4); set_collected_field('', 'bytes_sent', $5); set_collected_field('', 'client_info', $6); set_collected_field('', 'client_id', $7); set_collected_field('', 'client_stats_results', $8); set_collected_field('', 'file_size', $9); set_collected_field('', 'file_time', $10); set_collected_field('', 'sent_time', $11); set_collected_field('', 'resends', $12); set_collected_field('', 'failed_resends', $13); set_collected_field('', 'stream_components', $14); set_collected_field('', 'start_time', $15); set_collected_field('', 'server_address', $16); set_collected_field('', 'average_bitrate', $17); set_collected_field('', 'packets_sent', $18); set_collected_field('', 'proxy_info', $19); accept_collected_entry('', false); ); # as far as we can tell from the documentation of the Helix Proxy Admin guide, these are the standard logging formats of this device. # check for type 5 first, as it is the default type # type 5 else if (matches_regular_expression(current_log_line(), '^([^ ]+) - - \\[([0-9]+/[A-Za-z]+/[0-9]+:[0-9]+:[0-9]+:[0-9]+) [+-][0-9]+\\] *\\"([^"]+)\\" ([0-9]+) ([0-9]+) \\[([^]]+)\\] \\[([^]]+)\\] \\[([^]]+)\\] ([0-9]+) ([0-9]+) ([0-9]+) ([0-9]+) ([0-9]+) ([0-9]+) \\[([^]]+)\\]$')) then ( set_collected_field('', 'hostname', $1); set_collected_field('', 'date_time', $2); if (matches_regular_expression($3, '^(GET|POST) ([^ ]+) ([^$]+)')) then ( set_collected_field('', 'operation', $1); set_collected_field('', 'file', $2); set_collected_field('', 'protocol', $3); ); set_collected_field('', 'server_response', $4); set_collected_field('', 'bytes_sent', $5); set_collected_field('', 'client_info', $6); set_collected_field('', 'client_id', $7); set_collected_field('', 'client_stats_results', $8); set_collected_field('', 'file_size', $9); set_collected_field('', 'file_time', $10); set_collected_field('', 'sent_time', $11); set_collected_field('', 'resends', $12); set_collected_field('', 'failed_resends', $13); set_collected_field('', 'presentation_id', $14); set_collected_field('', 'proxy_info', $15); accept_collected_entry('', false); ); # type 0 else if (matches_regular_expression(current_log_line(), '^([^ ]+) - - \\[([0-9]+/[A-Za-z]+/[0-9]+:[0-9]+:[0-9]+:[0-9]+) [+-][0-9]+\\] *\\"([^"]+)\\" ([0-9]+) ([0-9]+) \\[([^]]+)\\] \\[([^]]+)\\] \\[([^]]+)\\]$')) then ( set_collected_field('', 'hostname', $1); set_collected_field('', 'date_time', $2); if (matches_regular_expression($3, '^(GET|POST) ([^ ]+) ([^$]+)')) then ( set_collected_field('', 'operation', $1); set_collected_field('', 'file', $2); set_collected_field('', 'protocol', $3); ); set_collected_field('', 'server_response', $4); set_collected_field('', 'bytes_sent', $5); set_collected_field('', 'client_info', $6); set_collected_field('', 'client_stats_results', $7); set_collected_field('', 'proxy_info', $8); accept_collected_entry('', false); ); # type 1 else if (matches_regular_expression(current_log_line(), '^([^ ]+) - - \\[([0-9]+/[A-Za-z]+/[0-9]+:[0-9]+:[0-9]+:[0-9]+) [+-][0-9]+\\] *\\"([^"]+)\\" ([0-9]+) ([0-9]+) \\[([^]]+)\\] \\[([^]]+)\\] ([0-9]+) ([0-9]+) ([0-9]+) ([0-9]+) ([0-9]+) \\[([^]]+)\\]$')) then ( set_collected_field('', 'hostname', $1); set_collected_field('', 'date_time', $2); if (matches_regular_expression($3, '^(GET|POST) ([^ ]+) ([^$]+)')) then ( set_collected_field('', 'operation', $1); set_collected_field('', 'file', $2); set_collected_field('', 'protocol', $3); ); set_collected_field('', 'server_response', $4); set_collected_field('', 'bytes_sent', $5); set_collected_field('', 'client_info', $6); set_collected_field('', 'client_stats_results', $7); set_collected_field('', 'file_size', $8); set_collected_field('', 'file_time', $9); set_collected_field('', 'sent_time', $10); set_collected_field('', 'resends', $11); set_collected_field('', 'failed_resends', $12); set_collected_field('', 'proxy_info', $13); accept_collected_entry('', false); ); # type 2 else if (matches_regular_expression(current_log_line(), '^([^ ]+) - - \\[([0-9]+/[A-Za-z]+/[0-9]+:[0-9]+:[0-9]+:[0-9]+) [+-][0-9]+\\] *\\"([^"]+)\\" ([0-9]+) ([0-9]+) \\[([^]]+)\\] \\[([^]]+)\\] \\[([^]]+)\\] ([0-9]+) ([0-9]+) ([0-9]+) ([0-9]+) ([0-9]+) \\[([^]]+)\\]$')) then ( set_collected_field('', 'hostname', $1); set_collected_field('', 'date_time', $2); if (matches_regular_expression($3, '^(GET|POST) ([^ ]+) ([^$]+)')) then ( set_collected_field('', 'operation', $1); set_collected_field('', 'file', $2); set_collected_field('', 'protocol', $3); ); set_collected_field('', 'server_response', $4); set_collected_field('', 'bytes_sent', $5); set_collected_field('', 'client_info', $6); set_collected_field('', 'client_id', $7); set_collected_field('', 'client_stats_results', $8); set_collected_field('', 'file_size', $9); set_collected_field('', 'file_time', $10); set_collected_field('', 'sent_time', $11); set_collected_field('', 'resends', $12); set_collected_field('', 'failed_resends', $13); set_collected_field('', 'proxy_info', $14); accept_collected_entry('', false); ); # type 3 else if (matches_regular_expression(current_log_line(), '^([^ ]+) - - \\[([0-9]+/[A-Za-z]+/[0-9]+:[0-9]+:[0-9]+:[0-9]+) [+-][0-9]+\\] *\\"([^"]+)\\" ([0-9]+) ([0-9]+) \\[([^]]+)\\] \\[([^]]+)\\] \\[([^]]+)\\] ([0-9]+) ([0-9]+) ([0-9]+) ([0-9]+) ([0-9]+) \\[([^]]+)\\] \\[([^]]+)\\] ([^ ]+) \\[([^]]+)\\]$')) then ( set_collected_field('', 'hostname', $1); set_collected_field('', 'date_time', $2); if (matches_regular_expression($3, '^(GET|POST) ([^ ]+) ([^$]+)')) then ( set_collected_field('', 'operation', $1); set_collected_field('', 'file', $2); set_collected_field('', 'protocol', $3); ); set_collected_field('', 'server_response', $4); set_collected_field('', 'bytes_sent', $5); set_collected_field('', 'client_info', $6); set_collected_field('', 'client_id', $7); set_collected_field('', 'client_stats_results', $8); set_collected_field('', 'file_size', $9); set_collected_field('', 'file_time', $10); set_collected_field('', 'sent_time', $11); set_collected_field('', 'resends', $12); set_collected_field('', 'failed_resends', $13); set_collected_field('', 'stream_components', $14); set_collected_field('', 'start_time', $15); set_collected_field('', 'server_address', $16); set_collected_field('', 'proxy_info', $17); accept_collected_entry('', false); ); # type 4 else if (matches_regular_expression(current_log_line(), '^([^ ]+) - - \\[([0-9]+/[A-Za-z]+/[0-9]+:[0-9]+:[0-9]+:[0-9]+) [+-][0-9]+\\] *\\"([^"]+)\\" ([0-9]+) ([0-9]+) \\[([^]]+)\\] \\[([^]]+)\\] \\[([^]]+)\\] ([0-9]+) ([0-9]+) ([0-9]+) ([0-9]+) ([0-9]+) \\[([^]]+)\\] \\[([^]]+)\\] ([^ ]+) ([0-9]+) ([0-9]+) \\[([^]]+)\\]$')) then ( set_collected_field('', 'hostname', $1); set_collected_field('', 'date_time', $2); if (matches_regular_expression($3, '^(GET|POST) ([^ ]+) ([^$]+)')) then ( set_collected_field('', 'operation', $1); set_collected_field('', 'file', $2); set_collected_field('', 'protocol', $3); ); set_collected_field('', 'server_response', $4); set_collected_field('', 'bytes_sent', $5); set_collected_field('', 'client_info', $6); set_collected_field('', 'client_id', $7); set_collected_field('', 'client_stats_results', $8); set_collected_field('', 'file_size', $9); set_collected_field('', 'file_time', $10); set_collected_field('', 'sent_time', $11); set_collected_field('', 'resends', $12); set_collected_field('', 'failed_resends', $13); set_collected_field('', 'stream_components', $14); set_collected_field('', 'start_time', $15); set_collected_field('', 'server_address', $16); set_collected_field('', 'average_bitrate', $17); set_collected_field('', 'packets_sent', $18); set_collected_field('', 'proxy_info', $19); accept_collected_entry('', false); ); ` log.field_options = { sessions_page_field = "file" # sessions_visitor_id_field = "client_id" sessions_visitor_id_field = "hostname" sessions_event_field = "clips" } # log.field_options database.numerical_fields = { clips = { label = "$lang_stats.field_labels.clips" default = true requires_log_field = false type = "int" display_format_type = "integer" entries_field = true } # clips visitors = { label = "$lang_stats.field_labels.visitors" default = false requires_log_field = true log_field = "client_id" type = "unique" display_format_type = "integer" } # visitors bytes_sent = { label = "$lang_stats.field_labels.bytes_sent" default = false requires_log_field = true log_field = "bytes_sent" type = "int" integer_bits = 64 display_format_type = "bandwidth" } # bytes_sent file_size = { label = "$lang_stats.field_labels.file_size" default = false requires_log_field = true log_field = "file_size" type = "int" integer_bits = 64 display_format_type = "bandwidth" } # file_size file_time = { label = "$lang_stats.field_labels.file_time" default = false requires_log_field = true log_field = "file_time" type = "int" display_format_type = "duration_milliseconds" } # file_time # Changed to seconds (was milliseconds), because docs say it's seconds: # http://service.real.com/help/library/guides/helixuniversalproxy/htmfiles/tracking.htm # "The sent_time field expresses in seconds the total playing time of the media transmitted to the client." sent_time = { label = "$lang_stats.field_labels.sent_time" default = false requires_log_field = true log_field = "sent_time" type = "int" display_format_type = "duration_compact" } # sent_time resends = { label = "$lang_stats.field_labels.resends" default = false requires_log_field = true log_field = "resends" type = "int" display_format_type = "integer" } # resends failed_resends = { label = "$lang_stats.field_labels.failed_resends" default = false requires_log_field = true log_field = "failed_resends" type = "int" display_format_type = "integer" } # failed_resends } # database.numerical_fields create_profile_wizard_options = { # How the reports should be grouped in the report menu report_groups = "" } # create_profile_wizard_options } # beta_helix_universal