# Copyright (c) 2010 Flowerfire, Inc.  All Rights Reserved.
palo_alto_networks_firewall_traffic = {

  plugin_version = "1.4.2"

  info.1.manufacturer = "Palo Alto Networks"
  info.1.device = "Firewall (Traffic)"
  info.1.version.1 = "1.3"
  info.1.version.2 = "2.0"
  info.1.version.3 = "3.0"
  info.1.version.4 = "3.1"
  info.1.version.5 = "4.0"

  # 2009-04-01 - KBB - 1.0 - Initial implementation, based on palo_alto_networks_firewall_threat.
  # 2009-07-08 - KBB - 1.1 - Added support for time format with year.
  # 2010-08-23 - Benson - 1.2 - Fixed for correct log format from syslog-ng.
  # 2010-09-22 - KBB - 1.3 - Backed out 1.2. Greg had added support for the missing 1,date already in
  # palo_alto_networks_firewall_threat.cfg and for threat in palo_alto_networks_firewall_integrated.cfg.
  # I extended it to traffic and added it here. Getting the year from the syslog header and the date and
  # time from the log are supported as long as generated_time (formerly known as time_generated!) is used,
  # which is Palo Alto's preference.
  # This version is now in sync with palo_alto_networks_firewall_integrated.cfg.
  # 2010-10-05 - MSG - 1.2.1 - Edited info lines.
  # 2011-02-11 - KBB - 1.3.1 - Changed support for missing 1,date to make space following those optional
  # to fix bug where some logs with those missing do not parse properly.
  # 2011-08-23 - KBB - 1.4 - Added support for version 4. In version 4, certain fields which previously
  # contained values are now "FUTURE_USE". Since the values are still in the v4 logs, but designated
  # unpredictable by the v4 documentation, they are now suppressed for all versions.
  # 2011-07-14 - gas - 1.4.1 - Slight mod of the parsing regex to allow hostname and IPs in some fields.
  # (KBB - GAS change was only added to integrated, so duplicating here. I changed all instances of '([0-9.]+)'
  # to '([a-z0-9.-]+)', whereas Graham did only some.)
  # 2013-02-15 - MSG - Made this syslog_optional and had it ignore format lines for a Palo Alto version 5 variation 
  # that has a csv header and file extension.
  # 2014-02-05 - GMF - 1.4.2 - Extended parsing regular expression to allow some fields to be empty (ThreadID:1301581)
  # 2014-03-05 - GMF - 1.4.3 - Removed highly unique fields sequence_number and start_time

  # The name of the log format
  log.format.format_label = "Palo Alto Networks Firewall Traffic Log Format"
  log.miscellaneous.log_data_type = "syslog_optional"
  log.miscellaneous.log_format_type = "firewall"

  # The first date after the syslog header is being ignored because I'm not sure it is always there. KBB
  #Jun  3 13:44:20 10.0.10.10 Jun  03 13: 44:20 1,06/03 13:44:20,0001a100200,TRAFFIC,start,8,06/03 13:44:19,10.0.10.11,16.166.16.66,0.0.0.0,0.0.0.0,mike,,,ntp,vsys1,l2-lan-trust,l2-lan-untrust,ethernet1/12,ethernet1/11,Forward to Mike,06/03 13:44:20,490821,1,123,123,0,0,0x0,udp,allow,90,90,90,1,06/03 13:44:20,0,any,0
  #2009-07-01 22:45:26	User.Info	10.222.2.222	Jul  1 22:45:26 1,2009/07/01 22:45:26,0002A100461,TRAFFIC,end,5,2009/07/01 22:45:24,22.122.2.122,10.222.2.122,0.0.0.0,0.0.0.0,Permit Any In,,,incomplete,vsys1,outside,inside,ethernet1/1,ethernet1/2,Log_to_KIWI_Sawmill,2009/07/01 22:45:25,313521,1,1247,81,0,0,0x0,tcp,allow,440,440,440,7,2009/07/01 22:44:52,30,any,0<000>
  # layered syslogs and no 1,[0-9]{4}... 
  #2010-06-22	13:17:50	Local7.Info	192.168.66.66	Jun 22 13:17:59 1,2010/06/22 13:17:59,0003C100949,TRAFFIC,end,117,2010/06/22 13:17:58,192.168.44.44,168.95.2.2,99.120.42.42,169.99.1.1,rule3,,,dns,vsys1,net.14-trust,net.13.14-untru,ethernet1/6,ethernet1/5,traffic-log,2010/06/22 13:17:58,141355,1,50878,53,33043,53,0x40,udp,allow,217,217,217,2,2010/06/22 13:17:27,1,any,0
  #Sep 09 08:58:24 10.30.14.179 Sep  9 09:00:53 1,2010/09/09 09:00:53,0006C100489,TRAFFIC,end,49,2010/09/09 09:00:53,192.168.22.222,25.25.152.122,172.12.22.222,25.25.152.122,rule12,,,mousketeer,vsys1,L3-trust,L3-untrust-2,ethernet1/2,ethernet1/6,Lakeview-229,2010/09/09 09:00:52,17473,1,49192,3544,30264,3544,0x42,udp,allow,254,254,254,2,2010/09/09 08:30:53,0,any,0
  log.format.autodetect_regular_expression = "[0-9]{2}:[0-9]{2}:[0-9]{2},[0-9A-Za-z]+,TRAFFIC,(start|end|drop|deny)"
  #log.format.autodetect_regular_expression = "1,([0-9]{4}/)?[0-9]{2}/[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2},[0-9A-Za-z]+,TRAFFIC,(start|end)"
  log.format.autodetect_lines = 10000

  log.format.ignore_format_lines = "true"

  log.format.parse_only_with_filters = "true"

  # Log fields
  log.fields = {

#    receive_time = ""
    serial_number = ""
#    type = ""
    sub_type = ""
#    config_version = ""
    source_ip = ""
    destination_ip = ""
    nat_source_ip = ""
    nat_destination_ip = ""
    rule_name = ""
    source_user = ""
    destination_user = ""
    application = ""
    virtual_system = ""
    source_zone = ""
    destination_zone = ""
    ingress_interface = ""
    egress_interface = ""
    log_forwarding_profile = ""
#    time_received = ""
#    session_id = ""
    source_port = ""
    destination_port = ""
    nat_source_port = ""
    nat_destination_port = ""
    flags = ""
    protocol = ""
    action = ""
    start_time = ""
    category = ""
#    repeat_count = ""

    bytes = ""
#    bytes_sent = ""
#    bytes_received = ""
    packets = ""
    elapsed_time = ""
    
    # new in v4
    sequence_number = ""
    action_flags = ""
    source_location = ""
    destination_location = ""

  } # log.fields

  # Log Filters
  log.filters = {

    mark_entry = {
      label = '$lang_admin.log_filters.mark_entry_label'
      comment = '$lang_admin.log_filters.mark_entry_comment'
      value = 'events = 1;'
    } # mark_entry

  } # log.filters

  log.parsing_filters.parse = `

# Get the year from the syslog date.
v.syslog_date = get_collected_field('', 'date_time');
v.year = '';
if (matches_regular_expression(v.syslog_date, '([0-9]{4})')) then (
  v.year = $1;
);

#v.session_user = '';

# v3 - These are the v3 field names. The corresponding v4 field names are now used for the database.
#Important fields: receive_time, sub_type, time_generated, src, dst, rule, srcuser, app, from, to, time_received, sessionid, dport, proto, action, bytes, bytes_sent, bytes_received, packets, start, elapsed, bytes, bytes_sent, bytes_received, packets, start, elapsed, category
#All fields: domain, receive_time, serial, type, sub_type, config_ver, time_generated, src, dst, natsrc, natdst, rule, srcuser, dstuser, app, vsys, from, to, inbound_if, outbound_if, logset, time_received, sessionid, repeatcnt, sport, dport, natsport, natdport, flags, proto, action, bytes, bytes_sent, bytes_received, packets, start, elapsed, category, padding
#Jun  3 13:44:20 10.0.0.244 Jun  03 13: 44:20 1,06/03 13:44:20,0001a100200,TRAFFIC,start,8,06/03 13:44:19,10.0.10.10,16.166.16.66,0.0.0.0,0.0.0.0,mike,,,ntp,vsys1,l2-lan-trust,l2-lan-untrust,ethernet1/12,ethernet1/11,Forward to Mike,06/03 13:44:20,490821,1,123,123,0,0,0x0,udp,allow,90,90,90,1,06/03 13:44:20,0,any,0
#2009-07-01 22:45:26	User.Info	10.222.2.222	Jul  1 22:45:26 1,2009/07/01 22:45:26,0002A100461,TRAFFIC,end,5,2009/07/01 22:45:24,22.122.2.122,10.222.2.122,0.0.0.0,0.0.0.0,Permit Any In,,,incomplete,vsys1,outside,inside,ethernet1/1,ethernet1/2,Log_to_KIWI_Sawmill,2009/07/01 22:45:25,313521,1,1247,81,0,0,0x0,tcp,allow,440,440,440,7,2009/07/01 22:44:52,30,any,0<000>
#Sep 09 12:18:03 10.30.14.179 Sep  9 12:20:32 1,2010/09/09 12:20:32,0006C100489,TRAFFIC,end,49,2010/09/09 12:20:32,192.168.22.222,22.222.22.2,172.12.22.222,22.222.22.2,rule12,,,bittorrent,vsys1,L3-trust,L3-untrust-2,ethernet1/2,ethernet1/6,Panorama-229,2010/09/09 12:20:31,31959,1,11171,30638,2542,30638,0x42,udp,allow,464,464,464,2,2010/09/09 12:00:32,0,any,0

# v4
#All fields: FUTURE_USE, receive_time, serial_number, type, sub_type, FUTURE_USE, generated_time, source_ip, destination_ip, nat_source_ip, nat_destination_ip, rule_name, source_user, destination_user, application, virtual_system, source_zone, destination_zone, ingress_interface, egress_interface, log_forwarding_profile, FUTURE_USE, session_id, repeat_count, source_port, destination_port, nat_source_port, nat_destination_port, flags, protocol, action, bytes, FUTURE_USE, FUTURE_USE, packets, start_time, elapsed_time, category, FUTURE_USE, sequence_number, action_flags, source_location, destination_location, FUTURE_USE
#Aug 23 20:38:25 10.20.30.40 1,2011/08/23 20:33:14,0001C100768,TRAFFIC,end,1,2011/08/23 20:33:13,172.16.2.222,172.16.3.133,0.0.0.0,0.0.0.0,testco policy,,,incomplete,vsys3,testco trust,testco trust,ethernet1/13,ethernet1/13,ubuntu_test_logs,2011/08/23 20:33:14,1353314,1,24410,45823,0,0,0x0,tcp,allow,198,198,198,4,2011/08/23 20:33:04,0,any,0,0,0x0,172.16.0.0-172.31.255.255,172.16.0.0-172.31.255.255,0

#if (matches_regular_expression(v.syslog_message, '(([^,]*),([0-9/]+) )? *([0-9]{2}:[0-9]{2}:[0-9]{2}),([^,]+),(TRAFFIC),([a-z]*),([0-9]*),([0-9/]+) ([0-9]{2}:[0-9]{2}:[0-9]{2}),([0-9.]+),([0-9.]+),([0-9.]+),([0-9.]+),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([0-9/]+ [0-9]{2}:[0-9]{2}:[0-9]{2}),([0-9]*),([0-9]*),([0-9]*),([0-9]*),([0-9]*),([0-9]*),([^,]*),([^,]*),([^,]*),([0-9]*),([0-9]*),([0-9]*),([0-9]*),([0-9/]+ [0-9]{2}:[0-9]{2}:[0-9]{2}),([0-9]*),([^,]*),([^,]*)(,([0-9]+))?')) then (
#if (matches_regular_expression(v.syslog_message, '(([^,]*),([0-9/]+) )? *([0-9]{2}:[0-9]{2}:[0-9]{2}),([^,]+),(TRAFFIC),([a-z]*),([0-9]*),([0-9/]+) ([0-9]{2}:[0-9]{2}:[0-9]{2}),([0-9.]+),([0-9.]+),([0-9.]+),([0-9.]+),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([0-9/]+ [0-9]{2}:[0-9]{2}:[0-9]{2}),([0-9]*),([0-9]*),([0-9]*),([0-9]*),([0-9]*),([0-9]*),([^,]*),([^,]*),([^,]*),([0-9]*),([0-9]*),([0-9]*),([0-9]*),([0-9/]+ [0-9]{2}:[0-9]{2}:[0-9]{2}),([0-9]*),([^,]*),([^,]*)(,([^,]*),([^,]*),([^,]*),([^,]*))?')) then (

if (matches_regular_expression(v.syslog_message, '(([^,]*),([0-9/]+) )? *([0-9]{2}:[0-9]{2}:[0-9]{2}),([^,]+),(TRAFFIC),([a-z]*),([0-9]*),([0-9/]+) ([0-9]{2}:[0-9]{2}:[0-9]{2}),([a-z0-9.-]*),([a-z0-9.-]*),([a-z0-9.-]*),([a-z0-9.-]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([0-9/]+ [0-9]{2}:[0-9]{2}:[0-9]{2}),([0-9]*),([0-9]*),([0-9]*),([0-9]*),([0-9]*),([0-9]*),([^,]*),([^,]*),([^,]*),([0-9]*),([0-9]*),([0-9]*),([0-9]*),([0-9/]+ [0-9]{2}:[0-9]{2}:[0-9]{2}),([0-9]*),([^,]*),([^,]*)(,([^,]*),([^,]*),([^,]*),([^,]*))?')) then (

  v.repeat_count = $27;
#  v.original_repeat_count = $44;

  # Insert repeat_count copies of log line.
  if (v.repeat_count > 1) then (
    v.in_context = "," . $25 . "," . $26 . "," . $27 . "," . $28 . "," . $29 . ",";
    v.once_only = "," . $25 . "," . $26 . "," . 1 . "," . $28 . "," . $29 . ",";
    v.line = replace_first(current_log_line(), v.in_context, v.once_only); 

    for (int i = 0; i < v.repeat_count; i++) (
#      set_subnode_value('volatile.log_line_insertions', i, v.line . "," . v.repeat_count);
      set_subnode_value('volatile.log_line_insertions', i, v.line);
    );
  );
  # Accept repeated and non-repeated lines.
  else (

    # Commented fields are currently not needed and not specified in log.fields or database.fields.
    v.src = $11;
    v.date = $9;
    #set_collected_field('', 'domain', $2); # future_use in v4
#    set_collected_field('', 'receive_time', $3 . " " . $4);
    set_collected_field('', 'serial_number', $5);
    #set_collected_field('', 'type', $6);
    set_collected_field('', 'sub_type', $7);
#    set_collected_field('', 'config_version', $8); # future_use in v4
    set_collected_field('', 'time', $10);
    set_collected_field('', 'source_ip', v.src);
    set_collected_field('', 'destination_ip', $12);
    set_collected_field('', 'nat_source_ip', $13);
    set_collected_field('', 'nat_destination_ip', $14);
    set_collected_field('', 'rule_name', $15);
    set_collected_field('', 'source_user', $16);
    set_collected_field('', 'destination_user', $17);
    set_collected_field('', 'application', $18);
    set_collected_field('', 'virtual_system', $19);
    set_collected_field('', 'source_zone', $20);
    set_collected_field('', 'destination_zone', $21);
    set_collected_field('', 'ingress_interface', $22);
    set_collected_field('', 'egress_interface', $23);
    set_collected_field('', 'log_forwarding_profile', $24);
#    set_collected_field('', 'time_received', $25); # future_use in v4
#    set_collected_field('', 'session_id', $26);
# Don't store repeat_count. Mechanism for remembering it breaks with varying line lengths.
#    set_collected_field('', 'repeat_count', $27);
#    if (v.original_repeat_count eq '') then (
#      v.original_repeat_count = "1";
#    );
#    set_collected_field('', 'repeat_count', v.original_repeat_count);
    set_collected_field('', 'source_port', $28);
    set_collected_field('', 'destination_port', $29);
    set_collected_field('', 'nat_source_port', $30);
    set_collected_field('', 'nat_destination_port', $31);
    set_collected_field('', 'flags', $32);
    set_collected_field('', 'protocol', $33);
    set_collected_field('', 'action', $34);
    set_collected_field('', 'bytes', $35);
#    set_collected_field('', 'bytes_sent', $36); # future_use in v4
#    set_collected_field('', 'bytes_received', $37); # future_use in v4
    set_collected_field('', 'packets', $38);
    set_collected_field('', 'start_time', $39);
    set_collected_field('', 'elapsed_time', $40);
    set_collected_field('', 'category', $41);
    #set_collected_field('', 'padding', $42); # future_use in v4
    set_collected_field('', 'sequence_number', $44);
    set_collected_field('', 'action_flags', $45);
    set_collected_field('', 'source_location', $46);
    set_collected_field('', 'destination_location', $47);

    if (matches_regular_expression(v.date, '^[0-9]{4}/[0-9]{2}/[0-9]{2}$')) then (
      set_collected_field('', 'date', v.date);
    );
    else if (v.year ne '') then (
      set_collected_field('', 'date', v.year . "/" . v.date);
    );
    else (
      set_collected_field('', 'date', normalize_date(v.date, 'mm/dd'));
    );

    accept_collected_entry('', false);
  );

);

#else ( # debug
#  echo(v.syslog_message);
#);
`

  # Database fields
  database.fields = {

#    receive_time = ""
    serial_number = ""
#    type = ""
    sub_type = ""
#    config_version = ""
    source_ip = ""
    destination_ip = ""
    nat_source_ip = ""
    nat_destination_ip = ""
    rule_name = ""
    source_user = ""
    destination_user = ""
    application = ""
    virtual_system = ""
    source_zone = ""
    destination_zone = ""
    ingress_interface = ""
    egress_interface = ""
    log_forwarding_profile = ""
#    time_received = ""
#    session_id = ""
    source_port = ""
    destination_port = ""
    nat_source_port = ""
    nat_destination_port = ""
    flags = ""
    protocol = ""
    action = ""

# 2014-03-05 - GMF - Removing highly unique field start_time
#    start_time = ""

    category = ""
#    repeat_count = ""
    
    # new in v4
# This is a *highly* unique field; disabling it by default
#    sequence_number = ""
    action_flags = ""
    source_location = ""
    destination_location = ""

  } # database.fields

  database.numerical_fields = {

    events = {
      default = true
      requires_log_field = false
      entries_field = true
    } # events

    bytes = {
      default = true
      type = "int"
      integer_bits = 64
      display_format_type = "bandwidth"
    } # bytes

#    bytes_sent = {
#      type = "int"
#      integer_bits = 64
#      display_format_type = "bandwidth"
#    } # bytes_sent
#
#    bytes_received = {
#      type = "int"
#      integer_bits = 64
#      display_format_type = "bandwidth"
#    } # bytes_received

    packets = {
      default = true
      type = "int"
      integer_bits = 64
      display_format_type = "bandwidth"
    } # packets

    elapsed_time = {
      default = true
      type = "int"
      integer_bits = 64
      display_format_type = "duration_compact"
    } # elapsed_time

  } # database.numerical_fields

  create_profile_wizard_options = {

    # How the reports should be grouped in the report menu
    report_groups = {

      date_time_group = ""

      source_group = {
        source_ip = true
        nat_source_ip = true
        source_user = true
        source_port = true
        nat_source_port = true
        source_zone = true
        egress_interface = true
        source_location = true
      } # source_group

      destination_group = {
        destination_ip = true
        nat_destination_ip = true
        destination_user = true
        destination_port = true
        nat_destination_port = true
        destination_zone = true
        ingress_interface = true
        destination_location = true
      } # destination_group

      other_group = {
        logging_device = true
        serial_number = true
        sub_type = true
#        config_version = true
        rule_name = true
        application = true
        virtual_system = true
        log_forwarding_profile = true
#        time_received = true
#        session_id = true
        flags = true
        protocol = true
        action = true
        start_time = true
        category = true
#        repeat_count = true
        sequence_number = true
        action_flags = true
      } # other_group

    } # report_groups

  } # create_profile_wizard_options

} # palo_alto_networks_firewall_traffic