# Copyright (c) 2010 Flowerfire, Inc. All Rights Reserved. symantec_gateway_security = { plugin_version = "1.5.4" # 2006-03-10 - 1.1beta - GMF - added support for German years. # 2006-03-26 - 1.2beta - GMF - added support for operations containing spaces. # 2006-04-14 - 1.3beta - GMF - added support for numerical months. # 2006-05-25 - 1.4beta - GAS - added support for new variation and added some new fields, example log lines are in the parsing filters # 2006-05-29 - 1.5beta - KBB - added support for new variation, restored support for numeric months # 2006-06-06 - 1.5.1beta - KBB - added "/" to valid characters in a service name # 2006-06-15 - 1.5.2beta - KBB - added ability to process repeated lines # 2006-06-27 - 1.5.3beta - KBB - map key Target to url field # 2007-09-14 - 1.5.3 - KBB - renumbered per new beta policy and renamed from # beta_symantec_gateway_security.cfg # 2011-07-20 - 1.5.4 - MSG - Edited info lines. info.1.manufacturer = "Symantec" info.1.device = "Gateway Security" info.1.version.1 = "SGS 2.0/3.0" info.1.version.2 = "SEF 8.0" # This log format is based on a text export of a binary data file on the SGS/SEF device. # To use "remotelogfile8.exe" to extract the text log from the binary data: # 1. Browse to "http://www.symantec.com/search/" # 2. search for document "2004021815290054" # To use the "flatten8" utility to extract the text log from the binary data: # 1. Review page 102 of "Symantec© Security Gateways - Reference Guide" - Version 8, this is an excerpt: # Flatten utility # The flatten8 utility is shipped on the included CD and lets you perform simple log file management from # the command-line. The flatten8 utility reads in the log message information from the system’s XML files, # and then parses in real-time the binary log file, substituting the actual error text message for its binary # counterpart. # Most often, this utility is used to convert the binary log file to a more usable format for a third party utility, # such as an ASCII text editor. This utility is also used to review the most recent messages, or directed to # show just statistics messages. # # usage: flatten8 [-h] [-r|-s|-D] [-f] [-u seconds] [-t n] [-x xmlpath] log file ... # # Where: # # -h Print this message and exit. # -r Only has an effect when -s is used. Do reverse lookups on IP addresses. # -s Output stats only. # -D Do not print out error information. # -f Follow output. (Binary files, default interval 2 seconds). # -u Follow update interval in seconds. (Implies -f). # -t Tail the last 'n' log messages. # -x Next argument specifies path to XML dictionary files. This argument should not need to be used, as the XML files # are placed in the default location during installation. # Format log_file_format value log.format.format_label = "Symantec Gateway Security Log Format (SGS 2.0/3.0 & SEF 8.0)" # This log is the following type log.miscellaneous.log_data_type = "network" log.miscellaneous.log_format_type = "firewall" log.format.treat_brackets_as_quotes = "false" log.format.ignore_format_lines = "true" # We use this to recognise the format and "auto-detect". # added support for another format - restructured for readability - # changed to auto_detect_regular_expression since there is only one - KBB log.format.autodetect_expression = ` matches_regular_expression(volatile.log_data_line, "^([A-Z][a-z][a-z]|[1-9]|[01][0-9])[ ][0-9][0-9],[ ][0-9][0-9][0-9][0-9][ ][0-9][0-9]:[0-9][0-9]:[0-9][0-9]\\.[0-9][0-9][0-9][ ]*[0-9a-z_.-]+[ ][a-zA-Z0-9 ]+[ ]*\\[?[0-9]+\\]?[ ]+(([0-9]+[ ]+)?(INFORMATIONAL|NOTICE|WARNING)([ ]+[0-9]+)?:?|[0-9]+:)[ ]+") ` # log.format.autodetect_expression = ` ## matches_regular_expression(volatile.log_data_line, '^([A-Z][a-z][a-z]|[0-9]+) [0-9][0-9], [0-9][0-9][0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9]\.[0-9][0-9][0-9] [0-9a-z_.-]+ [a-zA-Z0-9]+\\\\[[0-9]+\\\\] [0-9]* (INFORMATIONAL|NOTICE|WARNING): ') ## matches_regular_expression(volatile.log_data_line, '^([A-Z][a-z][a-z]|[0-9]+) [0-9][0-9], [0-9][0-9][0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9]\.[0-9][0-9][0-9] [0-9a-z_.-]+ [a-zA-Z0-9]+\\\\[[0-9]+\\\\] [0-9]*(INFORMATIONAL:|NOTICE:|WARNING:|:)') #matches_regular_expression(volatile.log_data_line, '^([A-Z][a-z][a-z]|[0-9]+)[ ][0-9][0-9],[ ][0-9][0-9][0-9][0-9][ ][0-9][0-9]:[0-9][0-9]:[0-9][0-9]\\\\.[0-9][0-9][0-9][ ]*[0-9a-z_.-]+[ ][a-zA-Z0-9 ]+[[ ]*[0-9]+[] ]*( INFORMATIONAL [0-9]+: | [0-9]+ INFORMATIONAL: | NOTICE [0-9]+: | WARNING [0-9]+: | [0-9]+ INFORMATIONAL | [0-9]*: | )') #` log.format.parse_only_with_filters = "true" log.format.allow_spaces_in_listed_field_values = "false" # Log fields log.fields = { date = "" time = "" logging_device = "" service = "" duration = "" authentication_result = "" id = "" sent = "" received = "" bytes = "" source_interface = "" source_ip.type = "host" source_port = "" source_name = "" server_source = "" server_source_port = "" destination_interface = "" destination_ip = "" destination_port = "" destination_name = "" client_destination = "" url = { type = "page" hierarchy_dividers = "/?" left_to_right = true leading_divider = "false" } # url result = "" protocol = "" rule_id = "" message_type = "" message = "" operation = "" status = "" state = "" rule = "" pid = "" notes = "" adapter = "" alert_destination_mac_addr = "" alert_source_mac_addr = "" class = "" consolidated_message = "" count = "" cve = "" end_time = "" family = "" flag = "" flow_cookie = "" host = "" interface = "" interface_id = "" interval = "" ip_code = "" ip_protocol = "" level = "" outcome = "" packet = "" payload_left_offset = "" payload_right_offset = "" policy_tag = "" program_name = "" reliability = "" request = "" resource = "" response = "" start_time = "" string_value = "" title = "" type = "" vendor = "" vlan_id = "" month = "" user = "" setting = "" key = "" revision = "" domain = "" client_port = "" related_id = "" server = "" ip_address = "" license_exp_date = "" feature_id = "" license_type = "" product = "" version = "" detail = "" av_comfort = "" av_scan = "" context_data = "" context_description = "" probable_probe = "" trace_route_ttl = "" # KBB command = "" error_number = "" information = "" length = "" limit = "" message_count = "" offset = "" #target = "" # it's a url } # log.fields log.filter_initialization = ` # Initialize German-to-English month translation map (the rest of the months are the same in both languages) v.german_months.Mrz = 'Mar'; v.german_months.Mai = 'May'; v.german_months.Okt = 'Oct'; v.german_months.Dez = 'Dec'; ` # Log Parsing Filters log.parsing_filters.parse = ` # supported log format variations # new KBB - e.g., May 22, 2006 17:37:54.343 21.152.1.220 kernel 1039 WARNING 30970 Packet is trying to leave via the wrong interface Source IP 21.152.1.232 Destination IP 21.152.1.239 IP Code TCP Flag RST Source Port 22 Destination Port 4097 Interface eth1 IP Address 21.121.2.220 # new KBB - e.g., May 22, 2006 16:46:58.628 11.151.1.230 IDS/IPS 1020 WARNING 31101 HTTP Nessus Scan Source IP 11.151.1.5 Destination IP 11.151.1.230 IP Code TCP Flag PUSH Flag ACK Source Port 48016 Destination Port 80 Interface eth0 Action Taken Logged Event 20450 Policy Tag Medium_Security # e.g., Jan 23, 2006 00:00:16.283 172.23.1.5 kernel[28518] 20146: Packet dropped by interface, input packet filter, Interface=eth1, Source IP=23.168.127.015, Destination IP=212.168.124.6, IP Code=TCP, Flag=SYN, Source Port=4294, Destination Port=135 # e.g., Jan 31, 2006 00:00:00.218 beaver1.pch.gc.ca httpd[16249] INFORMATIONAL 10487: Connection completed, Source IP=164.33.1.42, Destination IP=16.9.245.101, Source Port=1841, Destination Port=80, Source Name=164.33.1.42, Destination Name=blackberry.sk, Target=http://www.somewhere.net/no/url/specified.xsl, Operation=GET, Source Interface=eth0, Destination Interface=eth1, Protocol=http, Rule=11, Duration=0, ID=d7WVW, Sent=179, Received=2441, Bytes=2620, Server Source=18.103.196.130, Server Source Port=38825, Result=200 OK # e.g., Feb 06, 2006 23:59:58.746 10.10.1.23 kernel[986] 121 INFORMATIONAL: Statistics, Duration=65.00 , Authentication Result=N/A, ID=6ViGG, Sent=39, Received=55, Bytes=94, Source Interface=eth0, Source IP=10.10.1.29, Source Port=64239, Source Name=10.10.1.29, Server Source=192.168.197.129, Server Source Port=52510, Destination Interface=eth2, Destination IP=192.168.1.110, Destination Port=53, Destination Name=192.168.1.10, Operation=N/A, Protocol=53/udp, Rule ID=61, Notes=(Idle timeout) # e.g., Jul 25, 2005 09:05:50.336 www.domain.com pingd[6017] 121 INFORMATIONAL: Statistics, Duration=60.53 , Authentication Result=N/A, ID=g1gi, Sent=280, Received=280, Bytes=560, Source Interface=eth2, Source IP=10.5.14.4, Source Port=N/A, Source Name=10.5.14.4, Destination Interface=N/A, Destination IP=10.5.14.2, Destination Port=N/A, Destination Name=10.5.14.2, Operation=N/A, Protocol=ping, Rule ID=N/A, Notes=(Implicitly allowed local interface) # e.g., Sep 03, 2004 23:59:59.600 192.168.22.1 httpd 1834 121 INFORMATIONAL Statistics Duration 0.00 Authentication Result N/A ID 1quZW Sent 285 Received 141 Bytes 426 Source Interface eth1 Source IP 64.6.8.14 Source Port 45276 Source Name 64.6.8.14 Server Source 81.23.57.49 Server Source Port 50469 Destination Interface eth2 Destination IP 81.23.57.52 Destination Port 80 Destination Name 81.23.57.52 Operation GET Argument http://www.domain.com/navigation/ Result 304 Not Modified Protocol http Rule ID 6 # further modified to accommodate yet another variant - KBB ## modified this regex to accommodate the new log format - GAS ## if (matches_regular_expression(current_log_line(), '^([A-Za-z0-9]+ [0-9]+, [0-9]+) ([0-9:]+)\\\.[0-9]+ ([^ ]+) ([^[]+)\\\\[[0-9]+\\\\] (.*)$')) then ( #) #if (matches_regular_expression(current_log_line(), '^([A-Z][a-z][a-z][ ][0-9][0-9],[ ][0-9][0-9][0-9][0-9])[ ]([0-9][0-9]:[0-9][0-9]:[0-9][0-9])\\\.[0-9][0-9][0-9][ ]*([0-9a-z_.-]+)[ ]([a-zA-Z0-9 ]+)[[ ]*[0-9]+[] ]+(.*)$')) then ( #) # note: [^ ] before final .* below is because [ ]+ is not greedy for some reason - boost bug?! v.line = current_log_line(); # if this is a repeated line, insert it into the log data stream that many times #May 22, 2006 16:39:34.271 19.159.9.239 kernel 1020 NOTICE 20153 Repeated: IP packet dropped due to bad source address Message Count 18 Source IP 19.159.9.239 Destination IP 19.159.9.239 IP Code UDP Source Port 32192 Destination Port 14296 Interface eth0 #May 22, 2006 16:39:34.271 19.159.9.239 kernel[1020] NOTICE 20153: Repeated: IP packet dropped due to bad source address , Message Count=18, Source IP=19.159.9.239, Destination IP=19.159.9.239, IP Code=UDP, Source Port=32192, Destination Port=14296, Interface=eth0 if (matches_regular_expression(v.line, '^(.*,?[ ])Message Count[= ]([0-9]+),?[ ](.*)$')) then ( v.line = $1 . $3; for (int i = 0; i < $2; i++) ( set_subnode_value('volatile.log_line_insertions', i, v.line); ) ); # if message repeated - variation 1 #Jul 25, 2005 09:05:07.469 joe.green.com notifyd: 4610487 message repeated 3 times (121 INFORMATIONAL: Statistics, Duration=21.91 , Authentication Result=N/A, ID=dJBm, Sent=195, Received=644, Bytes=839, Source Interface=eth2, Source IP=19.9.93.29, Source Port=54437, Source Name=19.9.19.29, Server Source=19.9.19.29, Server Source Port=54437, Destination Interface=eth0, Destination IP=19.99.2.159, Destination Port=9004, Destination Name=19.99.2.159, Operation=N/A, Protocol=9004/tcp, Rule ID=22) else if (matches_regular_expression(v.line, '^(.* )message repeated ([0-9]+) times \\\((.*)\\\)$')) then ( v.line = $1 . $3; for (int i = 0; i < $2; i++) ( set_subnode_value('volatile.log_line_insertions', i, v.line); ) ); # if message repeated - variation 2 else ( # process regular lines if (matches_regular_expression(v.line, '^(([A-Z][a-z][a-z]|[01][0-9]|[1-9])[ ][0-9][0-9],[ ][0-9][0-9][0-9][0-9])[ ]([0-9][0-9]:[0-9][0-9]:[0-9][0-9])\\\.[0-9][0-9][0-9][ ]*([0-9a-z_.-]+)[ ]([a-zA-Z0-9/ ]+):?[ ]*\\\[?[0-9]+\\\]?[ ]+([^ ].*)$')) then ( v.date = $1; set_collected_field('', 'time', $3); set_collected_field('', 'logging_device', $4); set_collected_field('', 'service', $5); v.message = $6; # Handle numerical months if (matches_regular_expression(v.date, '^([0-9]+) ([0-9]+), ([0-9]+)$')) then ( v.date = normalize_date($3 . '/' . $1 . '/' . $2, 'auto'); ); # Handle German month names else if (matches_regular_expression(v.date, '^([A-Z][a-z][a-z])( .*)$')) then ( v.translated_month = node_value(subnode_by_name('v.german_months', $1)); if (v.translated_month ne '') then v.date = v.translated_month . $2; ); set_collected_field('', 'date', v.date); # simplify gathering of message_type, message and listed_fields - KBB # assumes all lines in all formats have messages - verified in existing examples # note: two steps below because boost doesn't nest captures #if (matches_regular_expression(v.message, #) #'^(([0-9]+[ ]+)?([A-Z]+)([ ]+[0-9]+)?:?|[0-9]+:)[ ]+(.*)')) then ( #) v.listed_fields = ''; if (matches_regular_expression(v.message, '^([0-9]+[ ]+[A-Z]+:?|[A-Z]+[ ]+[0-9]+:?|[0-9]+:)[ ]+(.*)')) then ( v.message_type = $1; v.listed_fields = $2; if (matches_regular_expression(v.message_type, '([A-Z]+)')) then ( # one format has no message type set_collected_field('', 'message_type', $1); ); ); # comma-tab separated - message may contain ", ", so can't look for sequence of no commas if (contains(v.listed_fields, ', ')) then ( if (matches_regular_expression(v.listed_fields, '^([^ ]+), (.*)')) then ( set_collected_field('', 'message', $1); collect_listed_fields('', $2, ', ', '=', 'Argument=url|Target=url|Date=license_exp_date') ); ); # tab separated else if (contains(v.listed_fields, ' ')) then ( if (matches_regular_expression(v.listed_fields, '^([^ ]+) (.*)')) then ( set_collected_field('', 'message', $1); collect_listed_fields('', $2, ' ', ' ', 'Argument=url|Target=url|Date=license_exp_date') ); ); # comma-space separated - assume no comma in message else if (contains(v.listed_fields, ', ')) then ( if (matches_regular_expression(v.listed_fields, '^([^,]+), (.*)')) then ( set_collected_field('', 'message', $1); collect_listed_fields('', $2, ', ', '=', 'Argument=url|Target=url|Date=license_exp_date') ); ); # fix negative durations to prevent huge numbers due to overflow - KBB if (starts_with(get_collected_field('', 'duration'), '-')) then set_collected_field('', 'duration', 0); ## # added spaces to the front of these three regex's ## # if (matches_regular_expression(v.message, '^[0-9]+: ([^,]+), ([^,]+),[ ](.*)$')) then ( #) ## if (matches_regular_expression(v.message, '^ *[0-9]+: ([^,]+), ([^,]+),[ ](.*)$')) then ( ## set_collected_field('', 'message_type', $1); ## set_collected_field('', 'message', $2); ## v.listed_fields = $3; ## ); ## # else if (matches_regular_expression(v.message, '^[0-9]+: ([^,]+),[ ](.*)$')) then ( #) ## else if (matches_regular_expression(v.message, '^ *[0-9]+: ([^,]+),[ ](.*)$')) then ( ## set_collected_field('', 'message_type', $1); ## v.listed_fields = $2; ## ); ## # else if (matches_regular_expression(v.message, '^[0-9]+ ([A-Z]+): ([^,]+),[ ](.*)$')) then ( ## else if (matches_regular_expression(v.message, '^ *[0-9]* ([A-Z]*): ([^,]*),[ ](.*)$')) then ( ## set_collected_field('', 'message_type', $1); ## set_collected_field('', 'message', $2); ## v.listed_fields = $3; ## ); ## else if (matches_regular_expression(v.message, ' ([A-Z]+) [0-9]+: (.*)$')) then ( ## set_collected_field('', 'message_type', $1); ## v.listed_fields = $2; ## ); ## # added new "else if" statements to collect the new log format - GAS ## else if (matches_regular_expression(v.message, '^[0-9]+ ([A-Z]+) ([^ ]+) (.*)$')) then ( ## set_collected_field('', 'message_type', $1); ## set_collected_field('', 'message', $2); ## v.listed_fields = $3; ## ); ## ## # the duration pair value sometimes ends in a space, this line removes it but it is disabled because I am not sure if it needs to be removed - GAS ## # if (matches_regular_expression(v.listed_fields, '^(Duration=[0-9]*\\\.[0-9]*) (,.*)$')) then v.listed_fields = $1 . $2; ## ## # Collect the listed fields - some use Target, some Argument as the URL field ## # comma and TAB seperated pairs, = divided ## if (matches_regular_expression(v.listed_fields, '^[^,]+, ')) then (collect_listed_fields('', v.listed_fields, ', ', '=', 'Argument=url|Target=url|Date=license_exp_date')); ## ## # TAB seperated pairs, TAB divided - GAS ## else if (matches_regular_expression(v.listed_fields, '^[^ ]+ ')) then (collect_listed_fields('', v.listed_fields, ' ', ' ', 'Argument=url|Date=license_exp_date')); ## ## # comma and space seperated pairs, = divided - GAS ## else if (matches_regular_expression(v.listed_fields, '^[^,]+, ')) then (collect_listed_fields('', v.listed_fields, ', ', '=', 'Argument=url|Date=license_exp_date')); accept_collected_entry('', false); ); ); ` # Database fields database.fields = { date_time = "" day_of_week = "" hour_of_day = "" logging_device = "" service = "" duration = "" authentication_result = "" id = "" sent = "" received = "" bytes = "" source_interface = "" source_ip = "" source_port = "" source_name = "" server_source = "" server_source_port = "" destination_interface = "" destination_ip = "" destination_port = "" destination_name = "" client_destination = "" url = "" result = "" protocol = "" rule_id = "" message_type = "" message = "" operation = "" status = "" state = "" rule = "" pid = "" notes = "" adapter = "" alert_destination_mac_addr = "" alert_source_mac_addr = "" class = "" consolidated_message = "" count = "" cve = "" # end_time = "" family = "" flag = "" flow_cookie = "" host = "" interface = "" interface_id = "" interval = "" ip_code = "" ip_protocol = "" level = "" outcome = "" packet = "" payload_left_offset = "" payload_right_offset = "" policy_tag = "" program_name = "" reliability = "" request = "" resource = "" response = "" # start_time = "" string_value = "" title = "" type = "" vendor = "" vlan_id = "" month = "" user = "" setting = "" key = "" revision = "" domain = "" client_port = "" related_id = "" server = "" ip_address = "" license_exp_date = "" feature_id = "" license_type = "" product = "" version = "" detail = "" av_comfort = "" av_scan = "" context_data = "" context_description = "" probable_probe = "" trace_route_ttl = "" # KBB command = "" error_number = "" information = "" length = "" limit = "" message_count = "" offset = "" #target = "" } # database.fields # Log Filters log.filters = { simplify_url = { label = 'simplify_url' comment = 'simplify_url' value = `if (matches_regular_expression(url, '^([^:]+://[^/]+)')) then url = $1 . '(truncated)'; else if(matches_regular_expression(url, '^([^/]+)')) then url = $1 . '(truncated)';` } # simplify_url simplify_id = { label = 'simplify_id' comment = 'simplify_id' value = `id = ''` } # simplify_id mark_entry = { label = '$lang_admin.log_filters.mark_entry_label' comment = '$lang_admin.log_filters.mark_entry_comment' value = 'events = 1;' } # mark_entry } # log.filters log.field_options = { sessions_page_field = "url" sessions_visitor_id_field = "source_ip" sessions_event_field = "page_views" } # log.field_options database.numerical_fields = { events = { default = true requires_log_field = false entries_field = true } # events sent = { type = "int" integer_bits = 64 display_format_type = "bandwidth" } # sent received = { type = "int" integer_bits = 64 display_format_type = "bandwidth" } # received bytes = { type = "int" integer_bits = 64 display_format_type = "bandwidth" } # bytes duration = { type = "int" integer_bits = 64 display_format_type = "duration_compact" } # duration } # database.numerical_fields create_profile_wizard_options = { # How the reports should be grouped in the report menu report_groups = { date_time_group = "" hour_of_day = true day_of_week = true logging_device = true message_type = true message = true notes = true consolidated_message = true adapter = true operation = true protocol = true status = true state = true rule = true rule_id = true authentication_result = true license_type = true license_exp_date = true feature_id = true product = true version = true source_group = { source_ip = true source_port = true source_name = true source_interface = true user = true client_port = true ip_address = true } content_group = { url = true file_type = true } destination_group = { destination_ip = true destination_port = true destination_name = true destination_interface = true client_destination = true } server_group = { server_source = true server_source_port = true server = true domain = true service = true } other_group = { id = true result = true pid = true alert_destination_mac_addr = true alert_source_mac_addr = true class = true count = true cve = true family = true flag = true flow_cookie = true host = true interface = true interface_id = true interval = true ip_code = true ip_protocol = true level = true outcome = true packet = true payload_left_offset = true payload_right_offset = true policy_tag = true program_name = true reliability = true request = true resource = true response = true string_value = true title = true type = true vendor = true vlan_id = true setting = true key = true revision = true related_id = true av_comfort = true av_scan = true context_data = true context_description = true payload = true probable_probe = true trace_route_ttl = true command = true error_number = true information = true length = true limit = true message_count = true offset = true #target = true } # other_group } # report_groups } # create_profile_wizard_options } # symantec_gateway_security