# Copyright (c) 2010 Flowerfire, Inc. All Rights Reserved. palo_alto_networks_firewall_threat = { plugin_version = "2.3" info.1.manfacturer = "Palo Alto Networks" info.1.device = "Firewall (Threat)" info.1.version.1 = "1.3" info.1.version.2 = "2.0" info.1.version.3 = "3.0" info.1.version.4 = "3.1" # 2008-06-12 - KBB - 1.0 - Initial implementation. # 2008-08-25 - KBB - 1.1 - Added basic sessioning. # 2009-01-24 - KBB - 1.1.1 - Changed field used for date and time from receive_time to time_generated. # 2009-07-08 - KBB - 1.2 - Added support for time format with year. # 2009-07-17 - GMF - 1.2.1 - Switched back to using the receive_time, because time_generated is so very # far off (see below, June 27 example). # 2009-07-17 - GMF - 1.3 - Added support for a variant [ThreadID:620871] # 2009-07-28 - GMF - 2.0 - Added support for many additional fields # 2009-07-29 - GMF - 2.1 - Added support for many additional fields in first format # 2010-10-05 - MSG - 2.1.1 - Edited info lines. # 2010-09-22 - KBB - 2.2 - Restored use of time_generated for time stamp at the request of Palo Alto # Networks. We have been assured that the very odd time in the example below was an abberation. # Combined the two variants, in autodetection and parsing. They are the same except for the 1,date # in the front. This is now in sync with palo_alto_networks_firewall_integrated.cfg. Made quotes optional # on page field since they aren't there when the field is empty. # 2011-01-25 - MSG & KBB - 2.2.1 - Fixed bug where matches_regular_expression reset positional variables # before they were used, causing threatid, category, severity and direction not to be set. # 2011-02-10 - KBB - 2.2.1 - Changed suppress bottom value of page field to 9 for consistency with change # to integrated (2010-12-08 - MSG - 1.3 - Changed suppress bottom value of page field to 9.). # 2011-02-14 - KBB - 2.2.2 - Changed support for missing 1,date to make space following those optional # to fix bug where some logs with those missing do not parse properly. (Only TRAFFIC examples supplied, # but changing THREAT plug-in and THREAT section of integrated plug-in for consistency.) # 2011-02-15 - KBB - 2.3 - Restored sessions by restoring the user database field. # Here's an example line where the syslog received the message on June 27, and the received_time is June 27, # but time_generated is January 22! # This why I (GMF) have switched back to receive_time, pending more information on this from Palo Alto. # 2009-06-27 22:58:27 User.Info 1.2.3.4 Jun 27 23:05:52 1,06/27 23:05:52,0002A100287,THREAT,url,10,01/22 08:26:14,12.34.56.78,98.76.54.32,0.0.0.0,0.0.0.0,Domain Users Default,nt-something\someone,,web-browsing,vsys1,trust,untrust,ethernet1/2,ethernet1/1,Fwd_to_Kiwi_Sawmill,06/27 23:05:51,406938,1,1413,80,0,0,0x0,tcp,alert,googleads.g.doubleclick.net/pagead/ads?,(9999),search-engines,informational,0<000>^M # The name of the log format log.format.format_label = "Palo Alto Networks Firewall Threat Log Format" log.miscellaneous.log_data_type = "syslog_required" log.miscellaneous.log_format_type = "firewall" # (Tabs in example changed to \t.) # The first date after the syslog header is being ignored because I'm not sure it is always there. KBB #2008-05-14 11:47:03\tUUCP.Info\t192.168.66.66\tMay 14 13:47:22 1,05/14 13:47:22,0001a100263,THREAT,url,3,12/31 18:09:46,192.168.66.66,12.34.56.78,0.0.0.0,0.0.0.0,rule1,,,gmail,vsys1,trust,untrust,ethernet1/2,ethernet1/1,Fwd_to_Syslogd,05/14 13:47:22,97,1,2222,80,0,0,0x0,tcp,alert,mail.google.com./mail/,(9999),web-based-e-mail,informational,0<000> #2008-08-25 16:53:36\tUser.Info\t192.168.101.10\tAug 26 14:51:07 1,08/26 14:51:07,0004A100238,THREAT,url,15,08/26 14:51:06,192.168.100.100,66.266.166.166,0.0.0.0,0.0.0.0,Outbound_Test_Nwk,,,web-browsing,vsys1,trust,untrust,ethernet1/2,ethernet1/1,threat to custom syslog,08/26 14:51:07,7364,1,50063,80,0,0,0x0,tcp,alert,safebrowsing.clients.google.com/safebrowsing/downloads?,(9999),unknown,informational,0<000> #2009-07-01 22:45:26 User.Info 10.222.2.222 Jul 1 22:45:26 1,2009/07/01 22:45:26,0002A100461,THREAT,url,5,2009/07/01 22:45:24,10.252.248.207,207.158.47.62,0.0.0.0,0.0.0.0,Less Restrictive,lion\tiger,,web-browsing,vsys1,inside,outside,ethernet1/2,ethernet1/1,Log_to_KIWI_Sawmill,2009/07/01 22:45:25,261303,1,2124,80,0,0,0x0,tcp,alert,"www.lionsandtigersandbearsohmy.com/errorpages/404.html",(9999),shopping,informational,0<000> # Another example, with different start-of-line format (no "1," and only one timestamp) [ThreadID:620871] #Jul 8 06:47:26 abc01-efgfw-01 06:47:26,0001A100332,THREAT,url,14,2009/07/08 06:46:53,12.34.56.78,98.76.54.32,23.45.67.89,87.65.43.21,User_Tracking,,,web-browsing,vsys25,assinet-inside,assinet-outside,ethernet1/2,ethernet1/10,ToSawMill,2009/07/08 06:47:26,981667,1,3557,80,3381,80,0x40,tcp,alert,"yahoo.com/",(9999),internet-portals,informational,0 # log.format.autodetect_expression = ` #matches_regular_expression(volatile.log_data_line, "1,([0-9]{4}/)?[0-9]{2}/[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2},[0-9A-Za-z]+,THREAT,(url|virus|vulnerability|spyware)") or #matches_regular_expression(volatile.log_data_line, "[0-9][0-9]:[0-9][0-9]:[0-9][0-9],[0-9A-Za-z]+,THREAT,(url|virus|vulnerability|spyware)") #` log.format.autodetect_regular_expression = "[0-9]{2}:[0-9]{2}:[0-9]{2},[0-9A-Za-z]+,THREAT,(url|virus|vulnerability|spyware)" log.format.autodetect_lines = 10000 # Log fields log.fields = { type = "" subtype = "" config_ver = "" src = "" dst = "" natsrc = "" natdst = "" rule = "" srcuser = "" dstuser = "" app = "" vsys = "" from = "" to = "" inbound_if = "" outbound_if = "" logset = "" time_received = "" sessionid = "" repeatcnt = "" sport = "" dport = "" natsport = "" natdport = "" flags = "" proto = "" action = "" #misc = "" # call this page since the value is always a page page.type = "page" threatid = "" category = "" severity = "" direction = "" user = "" } # log.fields # Log Filters log.filters = { # remove_query = { # label = "$lang_admin.log_filters.remove_query_label" # comment = "$lang_admin.log_filters.remove_query_comment" # value = "if (contains(page, '?')) then page = substr(page, 0, index(page, '?') + 1) . '(parameters)';" # disabled = true # } # remove_query detect_page_views = { label = '$lang_admin.log_filters.detect_page_views_label' comment = '$lang_admin.log_filters.detect_page_views_comment' value = "if ((subtype ne 'url') or (category eq 'advertisements-and-popups') or (file_type eq 'JPEG') or (file_type eq 'JPG') or (file_type eq 'GIF') or (file_type eq 'ICO') or (file_type eq 'PNG') or (file_type eq 'CSS') or (file_type eq 'SWF') or (file_type eq 'JS')) then page_views = 0; else page_views = 1;" } # detect_page_views # strip_non_page_views = { # label = '$lang_admin.log_filters.strip_non_page_views_label' # comment = '$lang_admin.log_filters.strip_non_page_views_comment' # value = "if (page_views == 0) then page = substr(page, 0, last_index(page, '/') + 1) . '(nonpage)';" # disabled = true # } # strip_non_page_views mark_entry = { label = '$lang_admin.log_filters.mark_entry_label' comment = '$lang_admin.log_filters.mark_entry_comment' value = 'events = 1;' } # mark_entry } # log.filters log.field_options = { sessions_page_field = "page" sessions_visitor_id_field = "user" sessions_event_field = "page_views" } # log.field_options log.parsing_filters.parse = ` # Get the year from the syslog date. v.syslog_date = get_collected_field('', 'date'); v.year = ''; if (matches_regular_expression(v.syslog_date, '([0-9]{4})')) then ( v.year = $1; ); v.session_user = ''; ##Important fields: receive_time, subtype, src, dst, srcuser, dport, action, misc, category #Important fields: subtype, time_generated, src, dst, srcuser, dport, action, misc, category #All fields: domain,receive_time,serial,type,subtype,config_ver,time_generated,src,dst,natsrc,natdst,rule,srcuser,dstuser,app,vsys,from,to,inbound_if,outbound_if,logset,time_received,sessionid,repeatcnt,sport,dport,natsport,natdport,flags,proto,action,misc,threatid,category,severity,direction #2008-08-25 17:07:05\tUser.Info\t192.168.65.65\tAug 26 15:04:36 1,08/26 15:04:36,0004A100238,THREAT,url,15,08/26 15:04:35,192.168.65.66,206.206.236.66,0.0.0.0,0.0.0.0,Outbound_Test_Nwk,testuser,,web-browsing,vsys1,trust,untrust,ethernet1/2,ethernet1/1,threat to test syslog,08/26 15:04:36,8270,1,51502,80,0,0,0x0,tcp,alert,www.securityfocus.com/rss/vulnerabilities.xml,(9999),computing-and-internet,informational,0<000> #2008-05-14 11:47:56\tUUCP.Info\t192.168.66.66\tMay 14 13:48:14 1,05/14 13:48:14,0001a100263,THREAT,spyware,3,12/31 18:10:38,192.168.55.55,55.55.55.55,0.0.0.0,0.0.0.0,rule1,,,web-browsing,vsys1,trust,untrust,ethernet1/2,ethernet1/1,Fwd_to_Syslogd,05/14 13:48:14,125,1,2246,80,0,0,0x0,tcp,alert,d.yimg.com./us.yimg.com/i/us/p/cnn.com.web,(9999),unknown,informational,0<000> #2009-07-01 22:45:26 User.Info 10.222.2.222 Jul 1 22:45:26 1,2009/07/01 22:45:26,0002A100461,THREAT,url,5,2009/07/01 22:45:24,10.252.248.207,207.158.47.62,0.0.0.0,0.0.0.0,Less Restrictive,lion\tiger,,web-browsing,vsys1,inside,outside,ethernet1/2,ethernet1/1,Log_to_KIWI_Sawmill,2009/07/01 22:45:25,261303,1,2124,80,0,0,0x0,tcp,alert,"www.lionsandtigersandbearsohmy.com/errorpages/404.html",(9999),shopping,informational,0<000> # Jul 8 06:47:26 abcdef01-enetfw-02 06:47:26,0001A100332,THREAT,url,14,2009/07/08 06:46:53,10.12.34.56.78,98.76.54.32,23.45.67.89,87.65.43.21,User_Tracking,,,web-browsing,vsys25,assinet-inside,assinet-outside,ethernet1/2,ethernet1/10,ToSawMill,2009/07/08 06:47:26,981207,1,2334,80,40550,80,0x40,tcp,alert,"forums.somewhere.com/index.php?",(9999),unknown,informational,0 # No page, so no quotes #Jun 2 18:00:25 10.0.0.44 Jun 02 18: 00:25 1,06/02 18:00:25,0001a100200,THREAT,vulnerability,4,06/02 18:00:19,550.0.0.22,10.0.0.222,0.0.0.0,0.0.0.0,rule15,laughnetwork\libby,,msrpc,vsys1,l2-lan-trust,l2-lan-untrust,ethernet1/12,ethernet1/11,Forward to Mike,06/02 18:00:25,180718,1,2007,135,0,0,0x8000,tcp,alert,,Microsoft RPC Endpoint Mapper(30845),any,low,0 #if (matches_regular_expression(v.syslog_message, '(([^,]*),([0-9/]+))? ([0-9]{2}:[0-9]{2}:[0-9]{2}),([^,]+),([A-Z]+),([^,]+),([^,]+),([0-9/]+) ([0-9]{2}:[0-9]{2}:[0-9]{2}),([0-9.]+),([0-9.]+),([0-9.]+),([0-9.]+),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([0-9/]+ [0-9]{2}:[0-9]{2}:[0-9]{2}),([^,]*),([0-9]*),([0-9]*),([0-9]*),([0-9]*),([0-9]*),([^,]*),([a-z]+),([^,]*),("[^"]*"|[^,]*),([^,]*),([^,]*),([^,]*),([^,]*)(,([0-9]+))?$')) then ( if (matches_regular_expression(v.syslog_message, '(([^,]*),([0-9/]+) )? *([0-9]{2}:[0-9]{2}:[0-9]{2}),([^,]+),([A-Z]+),([^,]+),([^,]+),([0-9/]+) ([0-9]{2}:[0-9]{2}:[0-9]{2}),([0-9.]+),([0-9.]+),([0-9.]+),([0-9.]+),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([0-9/]+ [0-9]{2}:[0-9]{2}:[0-9]{2}),([^,]*),([0-9]*),([0-9]*),([0-9]*),([0-9]*),([0-9]*),([^,]*),([a-z]+),([^,]*),("[^"]*"|[^,]*),([^,]*),([^,]*),([^,]*),([^,]*)(,([0-9]+))?$')) then ( v.repeatcnt = $27; v.original_repeatcnt = $42; # Insert repeatcnt copies of log line. if (v.repeatcnt > 1) then ( v.in_context = "," . $25 . "," . $26 . "," . $27 . "," . $28 . "," . $29 . ","; v.once_only = "," . $25 . "," . $26 . "," . 1 . "," . $28 . "," . $29 . ","; v.line = replace_first(current_log_line(), v.in_context, v.once_only); for (int i = 0; i < v.repeatcnt; i++) ( set_subnode_value('volatile.log_line_insertions', i, v.line . "," . v.repeatcnt); ); ); # Accept repeated and non-repeated lines. else ( v.user = $16; v.src = $11; v.date = $9; set_collected_field('', 'time', $10); # Commented fields are currently not needed and not specified in log.fields or database.fields. #set_collected_field('', 'domain', $2); #set_collected_field('', 'receive_time', $3 . " " . $4); set_collected_field('', 'serial', $5); #set_collected_field('', 'type', $6); set_collected_field('', 'subtype', $7); set_collected_field('', 'config_ver', $8); # $9 is date # $10 is time set_collected_field('', 'src', $11); set_collected_field('', 'dst', $12); set_collected_field('', 'natsrc', $13); set_collected_field('', 'natdst', $14); set_collected_field('', 'rule', $15); set_collected_field('', 'srcuser', $16); set_collected_field('', 'dstuser', $17); set_collected_field('', 'app', $18); set_collected_field('', 'vsys', $19); set_collected_field('', 'from', $20); set_collected_field('', 'to', $21); set_collected_field('', 'inbound_if', $22); set_collected_field('', 'outbound_if', $23); set_collected_field('', 'logset', $24); #set_collected_field('', 'time_received', $25); set_collected_field('', 'sessionid', $26); #set_collected_field('', 'repeatcnt', $27); if (v.original_repeatcnt eq '') then ( v.original_repeatcnt = "1"; ); set_collected_field('', 'repeatcnt', v.original_repeatcnt); set_collected_field('', 'sport', $28); set_collected_field('', 'dport', $29); set_collected_field('', 'natsport', $30); set_collected_field('', 'natdport', $31); set_collected_field('', 'flags', $32); set_collected_field('', 'proto', $33); set_collected_field('', 'action', $34); #set_collected_field('', 'misc', $35); v.page = $35; set_collected_field('', 'threatid', $36); set_collected_field('', 'category', $37); set_collected_field('', 'severity', $38); set_collected_field('', 'direction', $39); if (matches_regular_expression(v.page, '^"(.*)"$')) then ( v.page = $1; ); set_collected_field('', 'page', v.page); if (matches_regular_expression(v.date, '^[0-9]{4}/[0-9]{2}/[0-9]{2}$')) then ( set_collected_field('', 'date', v.date); ); else if (v.year ne '') then ( set_collected_field('', 'date', v.year . "/" . v.date); ); else ( set_collected_field('', 'date', normalize_date(v.date, 'mm/dd')); ); v.session_user = v.src; if (v.user ne '') then ( v.session_user .= '_' . v.user; ); set_collected_field('', 'user', v.session_user); accept_collected_entry('', false); ); ); #else ( # debug # echo(v.syslog_message); #); ` # Database fields database.fields = { #type = "" subtype = "" config_ver = "" src = "" dst = "" natsrc = "" natdst = "" rule = "" srcuser = "" dstuser = "" user = "" app = "" vsys = "" from = "" to = "" inbound_if = "" outbound_if = "" logset = "" # time_received = "" # sessionid = "" # repeatcnt = "" sport = "" dport = "" natsport = "" natdport = "" flags = "" proto = "" action = "" page = { suppress_bottom = 9 display_format_type = "page" } # page threatid = "" category = "" severity = "" direction = "" } # database.fields database.numerical_fields = { events = { default = true requires_log_field = false entries_field = true } # events page_views = { default = true requires_log_field = false } # page_views } # database.numerical_fields create_profile_wizard_options = { # How the reports should be grouped in the report menu report_groups = { date_time_group = "" source_group = { src = true natsrc = true srcuser = true user = true sport = true natsport = true from = true category_by_srcuser = true page_by_srcuser = true } # source_group destination_group = { dst = true natdst = true dstuser = true dport = true natdport = true to = true } # destination_group content_group = { #type = true subtype = true page = true file_type = true category = true } # content_group other_group = { app = true vsys = true action = true config_ver = true rule = true inbound_if = true outbound_if = true logset = true flags = true proto = true threatid = true severity = true category = true direction = true logging_device = true syslog_priority = true syslog_message_type = true } # other_group } # report_groups } # create_profile_wizard_options } # palo_alto_networks_firewall_threat