Dataset set postmatch 5576 v4

tests/datasets-delayed-multi-postmatch/README.md

@@ -0,0 +1,14 @@
+Test
+====
+
+Test datasets only sets when there is a full signature match.
+Test is with a signature using different keywords matching at different stages,
+and pcap having different packets making the transaction progress step by step.
+And test is using a multi-buffer to test that we only save the right occurences.
+
+https://redmine.openinfosecfoundation.org/issues/5576
+
+PCAP
+====
+
+Pcap crafted with some http server and some python client that delays or not the writing of the headers

tests/datasets-delayed-multi-postmatch/expected/http_match.csv

@@ -0,0 +1,2 @@
+WC1maXJzdDogc2VjcmV0
+SGVhZGVyMTogZmlyc3Q=

tests/datasets-delayed-multi-postmatch/test.rules

@@ -0,0 +1 @@
+alert http any any -> any any ( sid: 1; http.stat_code; content: "200"; fast_pattern; http.response_header; content: "first"; dataset:set,http_match,type string,save http_match.csv; file.data; content: "later";)

tests/datasets-delayed-multi-postmatch/test.yaml

@@ -0,0 +1,15 @@
+requires:
+  min-version: 8
+
+args:
+- -k none --no-random --data-dir=${OUTPUT_DIR}
+
+checks:
+- filter:
+    count: 1
+    match:
+      event_type: alert
+      alert.signature_id: 1
+- file-compare:
+    filename: http_match.csv
+    expected: expected/http_match.csv

tests/datasets-delayed-postmatch/README.md

@@ -0,0 +1,14 @@
+Test
+====
+
+Test datasets only sets when there is a full signature match.
+Test is with a signature using different keywords matching at different stages,
+and pcap having different packets making the transaction progress step by step.
+
+https://redmine.openinfosecfoundation.org/issues/5576
+
+PCAP
+====
+
+Pcap crafted with some http server and some python client that delays or not the writing of the headers
+

tests/datasets-delayed-postmatch/expected/http_match.csv

@@ -0,0 +1 @@
+L3BhZ2U/cGFyYW09dmFsdWU=

tests/datasets-delayed-postmatch/test.rules

@@ -0,0 +1,2 @@
+alert http any any -> any any ( sid: 1; http.uri; content: "param"; fast_pattern; pcre: "/param=(.*)/,flow:paramval"; http.request_header; content: "Header1";)
+alert http any any -> any any ( sid: 2; http.uri; content: "param"; fast_pattern; dataset:set,http_match,type string,save http_match.csv; http.method; content: "GET"; http.request_header; content: "Header1";)

tests/datasets-delayed-postmatch/test.yaml

@@ -0,0 +1,21 @@
+requires:
+  min-version: 8
+
+args:
+- -k none --no-random --data-dir=${OUTPUT_DIR}
+
+checks:
+- filter:
+    count: 1
+    match:
+      event_type: alert
+      alert.signature_id: 1
+      #TODO ticket 7197 metadata.flowvars[0].paramval: value
+- filter:
+    count: 1
+    match:
+      event_type: alert
+      alert.signature_id: 2
+- file-compare:
+    filename: http_match.csv
+    expected: expected/http_match.csv

tests/datasets-multibuf-postmatch/README.md

@@ -0,0 +1,8 @@
+# Description
+
+Test datasets only sets when there is a full signature match.
+Test is with multibuffer where some are matching, but not other.
+
+# Ticket
+
+https://redmine.openinfosecfoundation.org/issues/5576

tests/datasets-multibuf-postmatch/expected/http_match.csv

@@ -0,0 +1,2 @@
+WC10aXRpOiB0b3RvMg==
+WC10b3RvOiB2YWx1ZTE=

tests/datasets-multibuf-postmatch/test.rules

@@ -0,0 +1,2 @@
+# this will match, and thus write some headers to the dataset
+alert http any any -> any any (msg:"HTTP learning"; http.request_header; content:"toto"; dataset:set,http_match,type string,save http_match.csv; sid:1;)

tests/datasets-multibuf-postmatch/test.yaml

@@ -0,0 +1,15 @@
+requires:
+  min-version: 8
+
+args:
+  - -k none --no-random --data-dir=${OUTPUT_DIR}
+
+checks:
+  - filter:
+      count: 1
+      match:
+        event_type: alert
+        alert.signature_id: 1
+  - file-compare:
+      filename: http_match.csv
+      expected: expected/http_match.csv

tests/datasets-validate-postmatch/README.md

@@ -0,0 +1,7 @@
+# Description
+
+Test datasets only sets when there is a full signature match.
+
+# Ticket
+
+https://redmine.openinfosecfoundation.org/issues/5576

tests/datasets-validate-postmatch/test.rules

@@ -0,0 +1,4 @@
+# this will not match, and thus not write to the dataset
+alert http any any -> $HOME_NET any (msg:"HTTP learning"; flow:established,to_client; http.content_type; content:"noone"; http.server; content:"ECS"; fast_pattern; dataset:set,http_none,type string,state output/none.intel; sid:2; rev:1; priority:2;)
+# this will match, and thus write to the dataset
+alert http any any -> $HOME_NET any (msg:"HTTP learning"; flow:established,to_client; http.content_type; content:"ocsp"; http.server; content:"ECS"; fast_pattern; dataset:set,http_match,type string,state output/match.intel; sid:3; rev:1; priority:2;)

tests/datasets-validate-postmatch/test.yaml

@@ -0,0 +1,25 @@
+requires:
+  min-version: 8
+
+args:
+  - -k none
+
+checks:
+  # only sid 3 should trigger
+  - filter:
+      count: 1
+      match:
+        event_type: alert
+        alert.signature_id: 3
+  - filter:
+      count: 0
+      match:
+        event_type: alert
+        alert.signature_id: 2
+  # data should only be written to one dataset
+  - shell:
+      args: cat none.intel | wc -l | xargs
+      expect: 0
+  - shell:
+      args: cat match.intel | wc -l | xargs
+      expect: 1