Deep recursion postfix patterns (#69)

retoo · anthonycorbacho · commit 551fadba59e0 · 2017-08-04T10:48:02.000+09:00
* Add failing test for Postfix patterns The Postfix grok patterns have been downloaded from https://github.com/whyscream/postfix-grok-patterns/blob/f0ec34dcc6250463a30ba2077d8afa89ee1a17a1/postfix.grok Refs Graylog2/graylog2-server#3949 * Grok Compiler: Abort when unable to find definition for rule
diff --git a/patterns/postfix b/patterns/postfix
@@ -0,0 +1,125 @@
+# common postfix patterns
+POSTFIX_QUEUEID ([0-9A-F]{6,}|[0-9a-zA-Z]{15,})
+POSTFIX_CLIENT_INFO %{HOSTNAME:postfix_client_hostname}?\[%{IP:postfix_client_ip}\](:%{INT:postfix_client_port})?
+POSTFIX_RELAY_INFO %{HOSTNAME:postfix_relay_hostname}?\[(%{IP:postfix_relay_ip}|%{DATA:postfix_relay_service})\](:%{INT:postfix_relay_port})?|%{WORD:postfix_relay_service}
+POSTFIX_SMTP_STAGE (CONNECT|HELO|EHLO|STARTTLS|AUTH|MAIL( FROM)?|RCPT( TO)?|(end of )?DATA|RSET|UNKNOWN|END-OF-MESSAGE|VRFY|\.)
+POSTFIX_ACTION (accept|defer|discard|filter|header-redirect|reject)
+POSTFIX_STATUS_CODE \d{3}
+POSTFIX_STATUS_CODE_ENHANCED \d\.\d\.\d
+POSTFIX_DNSBL_MESSAGE Service unavailable; .* \[%{GREEDYDATA:postfix_status_data}\] %{GREEDYDATA:postfix_status_message};
+POSTFIX_PS_ACCESS_ACTION (DISCONNECT|BLACKLISTED|WHITELISTED|WHITELIST VETO|PASS NEW|PASS OLD)
+POSTFIX_PS_VIOLATION (BARE NEWLINE|COMMAND (TIME|COUNT|LENGTH) LIMIT|COMMAND PIPELINING|DNSBL|HANGUP|NON-SMTP COMMAND|PREGREET)
+POSTFIX_TIME_UNIT %{NUMBER}[smhd]
+POSTFIX_KEYVALUE_DATA [\w-]+=[^;]*
+POSTFIX_KEYVALUE %{POSTFIX_QUEUEID:postfix_queueid}: %{POSTFIX_KEYVALUE_DATA:postfix_keyvalue_data}
+POSTFIX_WARNING_LEVEL (warning|fatal|info)
+
+POSTFIX_TLSCONN (Anonymous|Trusted|Untrusted|Verified) TLS connection established (to %{POSTFIX_RELAY_INFO}|from %{POSTFIX_CLIENT_INFO}): %{DATA:postfix_tls_version} with cipher %{DATA:postfix_tls_cipher} \(%{DATA:postfix_tls_cipher_size} bits\)
+POSTFIX_DELAYS %{NUMBER:postfix_delay_before_qmgr}/%{NUMBER:postfix_delay_in_qmgr}/%{NUMBER:postfix_delay_conn_setup}/%{NUMBER:postfix_delay_transmission}
+POSTFIX_LOSTCONN (lost connection|timeout|SSL_accept error)
+POSTFIX_LOSTCONN_REASONS (receiving the initial server greeting|sending message body|sending end of data -- message may be sent more than once)
+POSTFIX_PROXY_MESSAGE (%{POSTFIX_STATUS_CODE:postfix_proxy_status_code} )?(%{POSTFIX_STATUS_CODE_ENHANCED:postfix_proxy_status_code_enhanced})?.*
+
+# helper patterns
+GREEDYDATA_NO_COLON [^:]*
+GREEDYDATA_NO_SEMICOLON [^;]*
+
+# warning patterns
+POSTFIX_WARNING_WITH_KV (%{POSTFIX_QUEUEID:postfix_queueid}: )?%{POSTFIX_WARNING_LEVEL:postfix_message_level}: %{GREEDYDATA:postfix_message}; %{POSTFIX_KEYVALUE_DATA:postfix_keyvalue_data}
+POSTFIX_WARNING_WITHOUT_KV (%{POSTFIX_QUEUEID:postfix_queueid}: )?%{POSTFIX_WARNING_LEVEL:postfix_message_level}: %{GREEDYDATA:postfix_message}
+POSTFIX_WARNING %{POSTFIX_WARNING_WITH_KV}|%{POSTFIX_WARNING_WITHOUT_KV}
+
+# smtpd patterns
+POSTFIX_SMTPD_CONNECT connect from %{POSTFIX_CLIENT_INFO}
+POSTFIX_SMTPD_DISCONNECT disconnect from %{POSTFIX_CLIENT_INFO}
+POSTFIX_SMTPD_LOSTCONN %{POSTFIX_LOSTCONN:postfix_smtpd_lostconn_data}( after %{POSTFIX_SMTP_STAGE:postfix_smtp_stage}( \(%{INT} bytes\))?)? from %{POSTFIX_CLIENT_INFO}(: %{GREEDYDATA:postfix_smtpd_lostconn_reason})?
+POSTFIX_SMTPD_NOQUEUE NOQUEUE: %{POSTFIX_ACTION:postfix_action}: %{POSTFIX_SMTP_STAGE:postfix_smtp_stage} from %{POSTFIX_CLIENT_INFO}:( %{POSTFIX_STATUS_CODE:postfix_status_code} %{POSTFIX_STATUS_CODE_ENHANCED:postfix_status_code_enhanced})?( <%{DATA:postfix_status_data}>:)? (%{POSTFIX_DNSBL_MESSAGE}|%{GREEDYDATA:postfix_status_message};) %{POSTFIX_KEYVALUE_DATA:postfix_keyvalue_data}
+POSTFIX_SMTPD_PIPELINING improper command pipelining after %{POSTFIX_SMTP_STAGE:postfix_smtp_stage} from %{POSTFIX_CLIENT_INFO}: %{GREEDYDATA:postfix_improper_pipelining_data}
+POSTFIX_SMTPD_PROXY proxy-%{POSTFIX_ACTION:postfix_proxy_result}: (%{POSTFIX_SMTP_STAGE:postfix_proxy_smtp_stage}): %{POSTFIX_PROXY_MESSAGE:postfix_proxy_message}; %{POSTFIX_KEYVALUE_DATA:postfix_keyvalue_data}
+
+# cleanup patterns
+POSTFIX_CLEANUP_MILTER %{POSTFIX_QUEUEID:postfix_queueid}: milter-%{POSTFIX_ACTION:postfix_milter_result}: %{GREEDYDATA:postfix_milter_message}; %{GREEDYDATA_NO_COLON:postfix_keyvalue_data}(: %{GREEDYDATA:postfix_milter_data})?
+
+# qmgr patterns
+POSTFIX_QMGR_REMOVED %{POSTFIX_QUEUEID:postfix_queueid}: removed
+POSTFIX_QMGR_ACTIVE %{POSTFIX_QUEUEID:postfix_queueid}: %{POSTFIX_KEYVALUE_DATA:postfix_keyvalue_data} \(queue active\)
+POSTFIX_QMGR_EXPIRED %{POSTFIX_QUEUEID:postfix_queueid}: from=<%{DATA:postfix_from}>, status=%{WORD:postfix_status}, returned to sender
+
+# pipe patterns
+POSTFIX_PIPE_ANY %{POSTFIX_QUEUEID:postfix_queueid}: %{POSTFIX_KEYVALUE_DATA:postfix_keyvalue_data}, status=%{WORD:postfix_status} \(%{GREEDYDATA:postfix_pipe_response}\)
+
+# error patterns
+POSTFIX_ERROR_ANY %{POSTFIX_QUEUEID:postfix_queueid}: %{POSTFIX_KEYVALUE_DATA:postfix_keyvalue_data}, status=%{WORD:postfix_status} \(%{GREEDYDATA:postfix_error_response}\)
+
+# discard patterns
+POSTFIX_DISCARD_ANY %{POSTFIX_QUEUEID:postfix_queueid}: %{POSTFIX_KEYVALUE_DATA:postfix_keyvalue_data} status=%{WORD:postfix_status} %{GREEDYDATA}
+
+# postsuper patterns
+POSTFIX_POSTSUPER_ACTIONS (removed|requeued|placed on hold|released from hold)
+POSTFIX_POSTSUPER_ACTION %{POSTFIX_QUEUEID:postfix_queueid}: %{POSTFIX_POSTSUPER_ACTIONS:postfix_postsuper_action}
+POSTFIX_POSTSUPER_SUMMARY_ACTIONS (Deleted|Requeued|Placed on hold|Released from hold)
+POSTFIX_POSTSUPER_SUMMARY %{POSTFIX_POSTSUPER_SUMMARY_ACTIONS:postfix_postsuper_summary_action}: %{NUMBER:postfix_postsuper_summary_count} messages?
+
+# postscreen patterns
+POSTFIX_PS_CONNECT CONNECT from %{POSTFIX_CLIENT_INFO} to \[%{IP:postfix_server_ip}\]:%{INT:postfix_server_port}
+POSTFIX_PS_ACCESS %{POSTFIX_PS_ACCESS_ACTION:postfix_postscreen_access} %{POSTFIX_CLIENT_INFO}
+POSTFIX_PS_NOQUEUE %{POSTFIX_SMTPD_NOQUEUE}
+POSTFIX_PS_TOOBUSY NOQUEUE: reject: CONNECT from %{POSTFIX_CLIENT_INFO}: %{GREEDYDATA:postfix_postscreen_toobusy_data}
+POSTFIX_PS_DNSBL %{POSTFIX_PS_VIOLATION:postfix_postscreen_violation} rank %{INT:postfix_postscreen_dnsbl_rank} for %{POSTFIX_CLIENT_INFO}
+POSTFIX_PS_CACHE cache %{DATA} full cleanup: retained=%{NUMBER:postfix_postscreen_cache_retained} dropped=%{NUMBER:postfix_postscreen_cache_dropped} entries
+POSTFIX_PS_VIOLATIONS %{POSTFIX_PS_VIOLATION:postfix_postscreen_violation}( %{INT})?( after %{NUMBER:postfix_postscreen_violation_time})? from %{POSTFIX_CLIENT_INFO}(( after %{POSTFIX_SMTP_STAGE:postfix_smtp_stage})?(: %{GREEDYDATA:postfix_postscreen_data})?| in tests (after|before) SMTP handshake)
+
+# dnsblog patterns
+POSTFIX_DNSBLOG_LISTING addr %{IP:postfix_client_ip} listed by domain %{HOSTNAME:postfix_dnsbl_domain} as %{IP:postfix_dnsbl_result}
+
+# tlsproxy patterns
+POSTFIX_TLSPROXY_CONN (DIS)?CONNECT( from)? %{POSTFIX_CLIENT_INFO}
+
+# anvil patterns
+POSTFIX_ANVIL_CONN_RATE statistics: max connection rate %{NUMBER:postfix_anvil_conn_rate}/%{POSTFIX_TIME_UNIT:postfix_anvil_conn_period} for \(%{DATA:postfix_service}:%{IP:postfix_client_ip}\) at %{SYSLOGTIMESTAMP:postfix_anvil_timestamp}
+POSTFIX_ANVIL_CONN_CACHE statistics: max cache size %{NUMBER:postfix_anvil_cache_size} at %{SYSLOGTIMESTAMP:postfix_anvil_timestamp}
+POSTFIX_ANVIL_CONN_COUNT statistics: max connection count %{NUMBER:postfix_anvil_conn_count} for \(%{DATA:postfix_service}:%{IP:postfix_client_ip}\) at %{SYSLOGTIMESTAMP:postfix_anvil_timestamp}
+
+# smtp patterns
+POSTFIX_SMTP_DELIVERY %{POSTFIX_KEYVALUE} status=%{WORD:postfix_status}( \(%{GREEDYDATA:postfix_smtp_response}\))?
+POSTFIX_SMTP_CONNERR connect to %{POSTFIX_RELAY_INFO}: (Connection timed out|No route to host|Connection refused|Network is unreachable)
+POSTFIX_SMTP_LOSTCONN %{POSTFIX_QUEUEID:postfix_queueid}: %{POSTFIX_LOSTCONN:postfix_smtp_lostconn_data} with %{POSTFIX_RELAY_INFO}( while %{POSTFIX_LOSTCONN_REASONS:postfix_smtp_lostconn_reason})?
+POSTFIX_SMTP_TIMEOUT %{POSTFIX_QUEUEID:postfix_queueid}: conversation with %{POSTFIX_RELAY_INFO} timed out( while %{POSTFIX_LOSTCONN_REASONS:postfix_smtp_lostconn_reason})?
+POSTFIX_SMTP_RELAYERR %{POSTFIX_QUEUEID:postfix_queueid}: host %{POSTFIX_RELAY_INFO} said: %{GREEDYDATA:postfix_smtp_response} \(in reply to %{POSTFIX_SMTP_STAGE:postfix_smtp_stage} command\)
+
+# master patterns
+POSTFIX_MASTER_START (daemon started|reload) -- version %{DATA:postfix_version}, configuration %{PATH:postfix_config_path}
+POSTFIX_MASTER_EXIT terminating on signal %{INT:postfix_termination_signal}
+
+# bounce patterns
+POSTFIX_BOUNCE_NOTIFICATION %{POSTFIX_QUEUEID:postfix_queueid}: sender (non-delivery|delivery status|delay) notification: %{POSTFIX_QUEUEID:postfix_bounce_queueid}
+
+# scache patterns
+POSTFIX_SCACHE_LOOKUPS statistics: (address|domain) lookup hits=%{INT:postfix_scache_hits} miss=%{INT:postfix_scache_miss} success=%{INT:postfix_scache_success}%
+POSTFIX_SCACHE_SIMULTANEOUS statistics: max simultaneous domains=%{INT:postfix_scache_domains} addresses=%{INT:postfix_scache_addresses} connection=%{INT:postfix_scache_connection}
+POSTFIX_SCACHE_TIMESTAMP statistics: start interval %{SYSLOGTIMESTAMP:postfix_scache_timestamp}
+
+# aggregate all patterns
+POSTFIX_SMTPD %{POSTFIX_SMTPD_CONNECT}|%{POSTFIX_SMTPD_DISCONNECT}|%{POSTFIX_SMTPD_LOSTCONN}|%{POSTFIX_SMTPD_NOQUEUE}|%{POSTFIX_SMTPD_PIPELINING}|%{POSTFIX_TLSCONN}|%{POSTFIX_WARNING}|%{POSTFIX_SMTPD_PROXY}|%{POSTFIX_KEYVALUE}
+POSTFIX_CLEANUP %{POSTFIX_CLEANUP_MILTER}|%{POSTFIX_WARNING}|%{POSTFIX_KEYVALUE}
+POSTFIX_QMGR %{POSTFIX_QMGR_REMOVED}|%{POSTFIX_QMGR_ACTIVE}|%{POSTFIX_QMGR_EXPIRED}|%{POSTFIX_WARNING}
+POSTFIX_PIPE %{POSTFIX_PIPE_ANY}
+POSTFIX_POSTSCREEN %{POSTFIX_PS_CONNECT}|%{POSTFIX_PS_ACCESS}|%{POSTFIX_PS_NOQUEUE}|%{POSTFIX_PS_TOOBUSY}|%{POSTFIX_PS_CACHE}|%{POSTFIX_PS_DNSBL}|%{POSTFIX_PS_VIOLATIONS}|%{POSTFIX_WARNING}
+POSTFIX_DNSBLOG %{POSTFIX_DNSBLOG_LISTING}|%{POSTFIX_WARNING}
+POSTFIX_ANVIL %{POSTFIX_ANVIL_CONN_RATE}|%{POSTFIX_ANVIL_CONN_CACHE}|%{POSTFIX_ANVIL_CONN_COUNT}
+POSTFIX_SMTP %{POSTFIX_SMTP_DELIVERY}|%{POSTFIX_SMTP_CONNERR}|%{POSTFIX_SMTP_LOSTCONN}|%{POSTFIX_SMTP_TIMEOUT}|%{POSTFIX_SMTP_RELAYERR}|%{POSTFIX_TLSCONN}|%{POSTFIX_WARNING}
+POSTFIX_DISCARD %{POSTFIX_DISCARD_ANY}|%{POSTFIX_WARNING}
+POSTFIX_LMTP %{POSTFIX_SMTP}
+POSTFIX_PICKUP %{POSTFIX_KEYVALUE}
+POSTFIX_TLSPROXY %{POSTFIX_TLSPROXY_CONN}|%{POSTFIX_WARNING}
+POSTFIX_MASTER %{POSTFIX_MASTER_START}|%{POSTFIX_MASTER_EXIT}|%{POSTFIX_WARNING}
+POSTFIX_BOUNCE %{POSTFIX_BOUNCE_NOTIFICATION}
+POSTFIX_SENDMAIL %{POSTFIX_WARNING}
+POSTFIX_POSTDROP %{POSTFIX_WARNING}
+POSTFIX_SCACHE %{POSTFIX_SCACHE_LOOKUPS}|%{POSTFIX_SCACHE_SIMULTANEOUS}|%{POSTFIX_SCACHE_TIMESTAMP}
+POSTFIX_TRIVIAL_REWRITE %{POSTFIX_WARNING}
+POSTFIX_TLSMGR %{POSTFIX_WARNING}
+POSTFIX_LOCAL %{POSTFIX_KEYVALUE}
+POSTFIX_VIRTUAL %{POSTFIX_SMTP_DELIVERY}
+POSTFIX_ERROR %{POSTFIX_ERROR_ANY}
+POSTFIX_POSTSUPER %{POSTFIX_POSTSUPER_ACTION}|%{POSTFIX_POSTSUPER_SUMMARY}
diff --git a/src/main/java/io/thekraken/grok/api/Grok.java b/src/main/java/io/thekraken/grok/api/Grok.java
@@ -15,6 +15,8 @@
  *******************************************************************************/
 package io.thekraken.grok.api;
 
+import static java.lang.String.format;
+
 import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileNotFoundException;
@@ -383,14 +385,19 @@ public void compile(String pattern, boolean namedOnly) throws GrokException {
             addPattern(group.get("pattern"), group.get("definition"));
             group.put("name", group.get("name") + "=" + group.get("definition"));
           } catch (GrokException e) {
-            // Log the exeception
+            throw new RuntimeException(e);
           }
         }
         int count = StringUtils.countMatches(namedRegex, "%{" + group.get("name") + "}");
         for (int i = 0; i < count; i++) {
-          String replacement = String.format("(?<name%d>%s)", index, grokPatternDefinition.get(group.get("pattern")));
+          String definitionOfPattern = grokPatternDefinition.get(group.get("pattern"));
+          if (definitionOfPattern == null) {
+              throw new GrokException(format("No definition for key '%s' found, aborting",
+                  group.get("pattern")));
+          }
+          String replacement = String.format("(?<name%d>%s)", index, definitionOfPattern);
           if (namedOnly && group.get("subname") == null) {
-            replacement = String.format("(?:%s)", grokPatternDefinition.get(group.get("pattern")));
+            replacement = String.format("(?:%s)", definitionOfPattern);
           }
           namedRegexCollection.put("name" + index,
               (group.get("subname") != null ? group.get("subname") : group.get("name")));
diff --git a/src/test/java/io/thekraken/grok/api/GrokTest.java b/src/test/java/io/thekraken/grok/api/GrokTest.java
@@ -18,6 +18,12 @@
 import org.junit.Test;
 import org.junit.runners.MethodSorters;
 
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.util.*;
+
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.junit.Assert.*;
 import io.thekraken.grok.api.exception.GrokException;
 
 
@@ -593,4 +599,47 @@ public void testDisablingAutomaticConversion() throws GrokException {
         gm.captures();
         assertEquals("\"foo\" \"bar\"", gm.toMap().get("clientid"));
     }
+
+    @Test
+    public void test020_postfix_patterns() throws Throwable {
+        final Grok grok = Grok.create("patterns/postfix");
+        grok.addPatternFromFile("patterns/patterns");
+        grok.compile("%{POSTFIX_SMTPD}", false);
+
+        assertTrue(grok.getPatterns().containsKey("POSTFIX_SMTPD"));
+    }
+
+    @Test
+    public void test021_postfix_patterns_with_named_captures_only() throws Throwable {
+        final Grok grok = Grok.create("patterns/postfix");
+        grok.addPatternFromFile("patterns/patterns");
+        grok.compile("%{POSTFIX_SMTPD}", true);
+
+        assertTrue(grok.getPatterns().containsKey("POSTFIX_SMTPD"));
+    }
+
+    @Test
+    public void test022_named_captures_with_missing_definition() throws Throwable {
+        ensureAbortsWithDefinitionMissing("FOO %{BAR}", "%{FOO}", true);
+    }
+
+    @Test
+    public void test023_captures_with_missing_definition() throws Throwable {
+        ensureAbortsWithDefinitionMissing("FOO %{BAR}", "%{FOO:name}", false);
+    }
+
+    @Test
+    public void test024_captures_with_missing_definition() throws Throwable {
+        ensureAbortsWithDefinitionMissing("FOO %{BAR}", "%{FOO}", false);
+    }
+
+    private void ensureAbortsWithDefinitionMissing(String pattern, String compilePattern, boolean namedOnly) {
+        try {
+            final Grok grok = Grok.create(ResourceManager.PATTERNS, pattern);
+            grok.compile(compilePattern, namedOnly);
+            fail("should abort due to missing definition");
+        } catch (GrokException e) {
+            assertThat(e.getMessage(), containsString("No definition for key"));
+        }
+    }
 }
