From 8755bc5e0aca5d4a0bf8bc391f196f26c27015a3 Mon Sep 17 00:00:00 2001 From: Andreas Herz Date: Tue, 11 Jun 2024 17:44:02 +0200 Subject: [PATCH 1/4] tests: add test to verify unmatched signatures does not write to dataset Related to https://redmine.openinfosecfoundation.org/issues/5576 --- tests/datasets-validate-postmatch/README.md | 7 ++++++ tests/datasets-validate-postmatch/input.pcap | Bin 0 -> 1929 bytes tests/datasets-validate-postmatch/test.rules | 4 +++ tests/datasets-validate-postmatch/test.yaml | 25 +++++++++++++++++++ 4 files changed, 36 insertions(+) create mode 100644 tests/datasets-validate-postmatch/README.md create mode 100644 tests/datasets-validate-postmatch/input.pcap create mode 100644 tests/datasets-validate-postmatch/test.rules create mode 100644 tests/datasets-validate-postmatch/test.yaml diff --git a/tests/datasets-validate-postmatch/README.md b/tests/datasets-validate-postmatch/README.md new file mode 100644 index 000000000..a8ccb5a9a --- /dev/null +++ b/tests/datasets-validate-postmatch/README.md @@ -0,0 +1,7 @@ +# Description + +Test datasets only sets when there is a full signature match. + +# Ticket + +https://redmine.openinfosecfoundation.org/issues/5576 diff --git a/tests/datasets-validate-postmatch/input.pcap b/tests/datasets-validate-postmatch/input.pcap new file mode 100644 index 0000000000000000000000000000000000000000..a554c13c8469891db8ed26baf6c4a2625cb99f89 GIT binary patch literal 1929 zcmaKtc~BEq9LL}0Lej1Q2P)8ETn{v$A-hWsvbDt|M1l|uC6U9UlP1fug4vj?fG|@j zvGmZwP)o7xjA*B{cI-^OYKvnlRXpnWhgUu7(GEI9rFLx9X^V{XZBi-*#+|&G&D-}r z@B96}_uD;pc=t9DkbwWM1ppxM58Cyf>l9D`6TBB1NL-G@CxWRD0f?CdEC6VRXDt9t zGJT2s)i~1NFv>fh{cl-L?W6DiJi$2T9_DSMYs{7 zBLsj}# ziNmj}`ZRK36<)#Dq>-kE>=M?VudlO|E~(G0DK2(8*hS8K z6T3varf7|pfg^Qrl)t(}UtC_cvZ2~+F&28O%dC3FZ1vgUqrwVCQ)F}6jA>c9nAPF1 ztBFhkm76)Qmv-|k=cTX$n)a!THEb=7%8hO}?c*sdL!E)jodMdfGBUImcGy_Ap9^r6 zyvpof>*G12Jzq`WnK&x9asl|9a|e8x9+qL^RSgH z?U`sToWcqCi;OOz{k628!Yt+zOzEz2sj~@#Asv;Mslt{i%$g4s3C0C~g|wI9t0+vP zhvx>BAa7-~H&~F;(OGgsNLHtpnYqUu{gj+HARd`os_Q z^|a-*hEKR}{l3zDv}$TkZ>T`C{f+$w(a*2V|Lx#O#ftkG)th&&B7e-jL}_k{+qUMg zpEv#^Px)eZ>vuV+-<-LoRurFGFTH*5kh|0L(x7GFoofR((|7frH9b3|yZ5|r(*y8k zCVihyn0~eI^2#PdSD!vtasHh0dh;Vjksq+{t3BFzr0o6nwik>9*%3PS_oIf=tPOEj z(>C3D*t_06UntdGbC9RKDCdM$COSS#*Xu)40A{66_F_h>Ji*i^Q*0S_zTGl`hiloTCm}aFjkR%26M5bip1Ef>$ylln+|QP{wP?2x}>i(sBW&dnBChZUFuTp_xv{ literal 0 HcmV?d00001 diff --git a/tests/datasets-validate-postmatch/test.rules b/tests/datasets-validate-postmatch/test.rules new file mode 100644 index 000000000..a028fa089 --- /dev/null +++ b/tests/datasets-validate-postmatch/test.rules @@ -0,0 +1,4 @@ +# this will not match, and thus not write to the dataset +alert http any any -> $HOME_NET any (msg:"HTTP learning"; flow:established,to_client; http.content_type; content:"noone"; http.server; content:"ECS"; fast_pattern; dataset:set,http_none,type string,state output/none.intel; sid:2; rev:1; priority:2;) +# this will match, and thus write to the dataset +alert http any any -> $HOME_NET any (msg:"HTTP learning"; flow:established,to_client; http.content_type; content:"ocsp"; http.server; content:"ECS"; fast_pattern; dataset:set,http_match,type string,state output/match.intel; sid:3; rev:1; priority:2;) diff --git a/tests/datasets-validate-postmatch/test.yaml b/tests/datasets-validate-postmatch/test.yaml new file mode 100644 index 000000000..4f4e4b73e --- /dev/null +++ b/tests/datasets-validate-postmatch/test.yaml @@ -0,0 +1,25 @@ +requires: + min-version: 8 + +args: + - -k none + +checks: + # only sid 3 should trigger + - filter: + count: 1 + match: + event_type: alert + alert.signature_id: 3 + - filter: + count: 0 + match: + event_type: alert + alert.signature_id: 2 + # data should only be written to one dataset + - shell: + args: cat none.intel | wc -l | xargs + expect: 0 + - shell: + args: cat match.intel | wc -l | xargs + expect: 1 From 2d83288f7db99ff4316d5c4c55ad8ed27da768c7 Mon Sep 17 00:00:00 2001 From: Philippe Antoine Date: Tue, 15 Oct 2024 09:46:40 +0200 Subject: [PATCH 2/4] datasets: test with multibuffer and set postmatch Ticket: 5576 --- tests/datasets-multibuf-postmatch/README.md | 8 ++++++++ .../expected/http_match.csv | 2 ++ tests/datasets-multibuf-postmatch/input.pcap | Bin 0 -> 1708 bytes tests/datasets-multibuf-postmatch/test.rules | 2 ++ tests/datasets-multibuf-postmatch/test.yaml | 15 +++++++++++++++ 5 files changed, 27 insertions(+) create mode 100644 tests/datasets-multibuf-postmatch/README.md create mode 100644 tests/datasets-multibuf-postmatch/expected/http_match.csv create mode 100644 tests/datasets-multibuf-postmatch/input.pcap create mode 100644 tests/datasets-multibuf-postmatch/test.rules create mode 100644 tests/datasets-multibuf-postmatch/test.yaml diff --git a/tests/datasets-multibuf-postmatch/README.md b/tests/datasets-multibuf-postmatch/README.md new file mode 100644 index 000000000..c0a25a38b --- /dev/null +++ b/tests/datasets-multibuf-postmatch/README.md @@ -0,0 +1,8 @@ +# Description + +Test datasets only sets when there is a full signature match. +Test is with multibuffer where some are matching, but not other. + +# Ticket + +https://redmine.openinfosecfoundation.org/issues/5576 diff --git a/tests/datasets-multibuf-postmatch/expected/http_match.csv b/tests/datasets-multibuf-postmatch/expected/http_match.csv new file mode 100644 index 000000000..752b88213 --- /dev/null +++ b/tests/datasets-multibuf-postmatch/expected/http_match.csv @@ -0,0 +1,2 @@ +WC10aXRpOiB0b3RvMg== +WC10b3RvOiB2YWx1ZTE= diff --git a/tests/datasets-multibuf-postmatch/input.pcap b/tests/datasets-multibuf-postmatch/input.pcap new file mode 100644 index 0000000000000000000000000000000000000000..1c4ed0f5760eb42e59bcc4ea4cd750e4876136f4 GIT binary patch literal 1708 zcmbW1(QDH{9LLkH%yAHLzF2UwgZUuanxtL3)*!fH%LZ<2tPyo*%%J= zFWB}6%m-n(7s2V1;KoE>6@2s`5EP$8olkcDu4(8ttE{{B;4b&O?|pxt{4Uw|FCNE* zgwSr`_)YR`%^XMwMOffHYH=8p1Yv^zCwc|pfgr?qZ;TACZ*A<{=W*NdgM&s^5aNmJ zA7ec|y|Gvyx-kCXB40~@0M~xjh4$cjV`Q^$;V0;?Y@h#I`%dclcdj34zXI(7oW*<8 zTE4+mN8c-;zyI)D@@wM?B-8TCb-CBKUFfyE?sKn)^&*d44_!(KJ7A3WsI~mP{I9$%CJlqQ@Nifj7SZh#bC{#zMcoPrXu(`(x{6uAg651e3Nwhwh)6^{r)!YY zNtw49?Fzu?OT_#zqm{t~ya%3ZLjDn0~Fv3zk^N3z=$ITGC#!MBY@*hYZ-rL%M7so;W zItqFS7m06p)a3vW(umJ+1uDlU%*T#xE#u~B$s{&rCh-WSKGtzHuq^D;nys?{p4D|` zm{PbK)|V(VaXqjYy&F`k#KW*1qI{=LupWg|yEcmN1`OK{!w%P$5E{Z7HSQ6Hkja?q ztFr8oN}bw7a$I6lyXyGFs5*fS9>K8Wcr}^&en5OVm&=bg>-x#5aBHi8`%@>}ue;C- z>sniF;_mN!Pu_K**DeQb?O*;b`W5>P=KcHj>{s8uY BlAZtn literal 0 HcmV?d00001 diff --git a/tests/datasets-multibuf-postmatch/test.rules b/tests/datasets-multibuf-postmatch/test.rules new file mode 100644 index 000000000..9636b1e5b --- /dev/null +++ b/tests/datasets-multibuf-postmatch/test.rules @@ -0,0 +1,2 @@ +# this will match, and thus write some headers to the dataset +alert http any any -> any any (msg:"HTTP learning"; http.request_header; content:"toto"; dataset:set,http_match,type string,save http_match.csv; sid:1;) diff --git a/tests/datasets-multibuf-postmatch/test.yaml b/tests/datasets-multibuf-postmatch/test.yaml new file mode 100644 index 000000000..cc30d2721 --- /dev/null +++ b/tests/datasets-multibuf-postmatch/test.yaml @@ -0,0 +1,15 @@ +requires: + min-version: 8 + +args: + - -k none --no-random --data-dir=${OUTPUT_DIR} + +checks: + - filter: + count: 1 + match: + event_type: alert + alert.signature_id: 1 + - file-compare: + filename: http_match.csv + expected: expected/http_match.csv From ef03d675bcdaa418be75c5a8e9f70a96b3969681 Mon Sep 17 00:00:00 2001 From: Philippe Antoine Date: Thu, 1 Aug 2024 21:21:13 +0200 Subject: [PATCH 3/4] datasets: test with delayed set postmatch Ticket: 5576 Signature full match does not happen on first packet inspected, but signature gets stored as partially matching so far, and then postmatch retreieves the buffer to set in the dataset. --- tests/datasets-delayed-postmatch/README.md | 14 ++++++++++++ .../expected/http_match.csv | 1 + tests/datasets-delayed-postmatch/input.pcap | Bin 0 -> 1182 bytes tests/datasets-delayed-postmatch/test.rules | 2 ++ tests/datasets-delayed-postmatch/test.yaml | 21 ++++++++++++++++++ 5 files changed, 38 insertions(+) create mode 100644 tests/datasets-delayed-postmatch/README.md create mode 100644 tests/datasets-delayed-postmatch/expected/http_match.csv create mode 100644 tests/datasets-delayed-postmatch/input.pcap create mode 100644 tests/datasets-delayed-postmatch/test.rules create mode 100644 tests/datasets-delayed-postmatch/test.yaml diff --git a/tests/datasets-delayed-postmatch/README.md b/tests/datasets-delayed-postmatch/README.md new file mode 100644 index 000000000..a073a6dd2 --- /dev/null +++ b/tests/datasets-delayed-postmatch/README.md @@ -0,0 +1,14 @@ +Test +==== + +Test datasets only sets when there is a full signature match. +Test is with a signature using different keywords matching at different stages, +and pcap having different packets making the transaction progress step by step. + +https://redmine.openinfosecfoundation.org/issues/5576 + +PCAP +==== + +Pcap crafted with some http server and some python client that delays or not the writing of the headers + diff --git a/tests/datasets-delayed-postmatch/expected/http_match.csv b/tests/datasets-delayed-postmatch/expected/http_match.csv new file mode 100644 index 000000000..352340277 --- /dev/null +++ b/tests/datasets-delayed-postmatch/expected/http_match.csv @@ -0,0 +1 @@ +L3BhZ2U/cGFyYW09dmFsdWU= diff --git a/tests/datasets-delayed-postmatch/input.pcap b/tests/datasets-delayed-postmatch/input.pcap new file mode 100644 index 0000000000000000000000000000000000000000..41fc99473f59ea6b92a2b1c74e4157aaa7c365a8 GIT binary patch literal 1182 zcmb7^L2DC17>2*3q3S?DS`UE=otse-lT90vu7qZ5OIM;;%AyAkGOW|AuqJ7`5rv>V zOalC4eGb#c_a#03pim)U>g856+iQMhY zld+%sagz;mox4?AxDMVjdhD^sqig8XUqJmIK@EX^962|i0hp*xQTjLm z#bL+3(5sX5v3tMHLX%jg6t`qu(|ONZW5j)q+)>BW=eIiOr)}k0h2~l0Tzd;}3Ee2l zuQ!=P#&Y$y-(28lINgr6<@;rnUxqOPw-Edvg7%NEYY6bB|BAy#s>fM5g`Cp>zX0SpB}o7P literal 0 HcmV?d00001 diff --git a/tests/datasets-delayed-postmatch/test.rules b/tests/datasets-delayed-postmatch/test.rules new file mode 100644 index 000000000..d9706cc67 --- /dev/null +++ b/tests/datasets-delayed-postmatch/test.rules @@ -0,0 +1,2 @@ +alert http any any -> any any ( sid: 1; http.uri; content: "param"; fast_pattern; pcre: "/param=(.*)/,flow:paramval"; http.request_header; content: "Header1";) +alert http any any -> any any ( sid: 2; http.uri; content: "param"; fast_pattern; dataset:set,http_match,type string,save http_match.csv; http.method; content: "GET"; http.request_header; content: "Header1";) diff --git a/tests/datasets-delayed-postmatch/test.yaml b/tests/datasets-delayed-postmatch/test.yaml new file mode 100644 index 000000000..f7317888e --- /dev/null +++ b/tests/datasets-delayed-postmatch/test.yaml @@ -0,0 +1,21 @@ +requires: + min-version: 8 + +args: +- -k none --no-random --data-dir=${OUTPUT_DIR} + +checks: +- filter: + count: 1 + match: + event_type: alert + alert.signature_id: 1 + #TODO ticket 7197 metadata.flowvars[0].paramval: value +- filter: + count: 1 + match: + event_type: alert + alert.signature_id: 2 +- file-compare: + filename: http_match.csv + expected: expected/http_match.csv \ No newline at end of file From e842b6a17ee8ea65f2ba9574077747037f617759 Mon Sep 17 00:00:00 2001 From: Philippe Antoine Date: Tue, 15 Oct 2024 14:41:16 +0200 Subject: [PATCH 4/4] datasets: test with multi-buffer and occurences in different packets Ticket: 5576 --- tests/datasets-delayed-multi-postmatch/README.md | 14 ++++++++++++++ .../expected/http_match.csv | 2 ++ .../datasets-delayed-multi-postmatch/input.pcap | Bin 0 -> 1845 bytes .../datasets-delayed-multi-postmatch/test.rules | 1 + tests/datasets-delayed-multi-postmatch/test.yaml | 15 +++++++++++++++ 5 files changed, 32 insertions(+) create mode 100644 tests/datasets-delayed-multi-postmatch/README.md create mode 100644 tests/datasets-delayed-multi-postmatch/expected/http_match.csv create mode 100644 tests/datasets-delayed-multi-postmatch/input.pcap create mode 100644 tests/datasets-delayed-multi-postmatch/test.rules create mode 100644 tests/datasets-delayed-multi-postmatch/test.yaml diff --git a/tests/datasets-delayed-multi-postmatch/README.md b/tests/datasets-delayed-multi-postmatch/README.md new file mode 100644 index 000000000..b794e7276 --- /dev/null +++ b/tests/datasets-delayed-multi-postmatch/README.md @@ -0,0 +1,14 @@ +Test +==== + +Test datasets only sets when there is a full signature match. +Test is with a signature using different keywords matching at different stages, +and pcap having different packets making the transaction progress step by step. +And test is using a multi-buffer to test that we only save the right occurences. + +https://redmine.openinfosecfoundation.org/issues/5576 + +PCAP +==== + +Pcap crafted with some http server and some python client that delays or not the writing of the headers diff --git a/tests/datasets-delayed-multi-postmatch/expected/http_match.csv b/tests/datasets-delayed-multi-postmatch/expected/http_match.csv new file mode 100644 index 000000000..9a2e45ed0 --- /dev/null +++ b/tests/datasets-delayed-multi-postmatch/expected/http_match.csv @@ -0,0 +1,2 @@ +WC1maXJzdDogc2VjcmV0 +SGVhZGVyMTogZmlyc3Q= diff --git a/tests/datasets-delayed-multi-postmatch/input.pcap b/tests/datasets-delayed-multi-postmatch/input.pcap new file mode 100644 index 0000000000000000000000000000000000000000..fcbd9a32917c3bdb70db8025c7046ddbc0ef4dc0 GIT binary patch literal 1845 zcmbW1&u`LT7{^~2fu$LX^TdrBFJ=m~fbz@CBFrrrn~Y)Hfsict%9b!#+D-=-J(;jS z!J8+qi-~{05(%f>y|~EHnRxUd+2-?<_l1IBQXY7crfoj`e4oB=!)Kp9*olJ-s&{<1 zFkk+>?;>fqfX&Me2g`&g_*a~SydlJfecJRU+YR*(-gjcZzP<_*Vs||HYMYsH+HA8z zEAj9*UUL8d*F>kOM>)i$Z`P&cpl)b^b&I(%PEOLNfMu~30qWsMODnC z>9u`B|GNeK-on+^VF6s4Ul=YZz5p(O2Np(;=%*&=U$z@eKQf|EcV19cj*Bn%R82Q@ ze@&$unln$E>)wO6_vOe%*1!{+7n?SuP=@wxZ`z@`zHWl+W^>3))v7uy8J}K1pm9;B zN4kKIaMrDEF`RJ7JFKj9V#RC@dAEUA09R~YOnG~PvbV3CF)J&bZJ#mPxTw*RN_Bxp z$k1w8--V447K{x3=`y6oW$;0ULeoW_fIK!Ymf`V)45@yG2RuW5sg+PV8l_^rb|_YM zbgJG$`g@>T`=}Wc-vrwqd`q?x>9bfdzl1#J@szxU@w}u39$R#qkbc8FO>{#?^s7wE XTQGlu2E30Jc any any ( sid: 1; http.stat_code; content: "200"; fast_pattern; http.response_header; content: "first"; dataset:set,http_match,type string,save http_match.csv; file.data; content: "later";) diff --git a/tests/datasets-delayed-multi-postmatch/test.yaml b/tests/datasets-delayed-multi-postmatch/test.yaml new file mode 100644 index 000000000..c83451c21 --- /dev/null +++ b/tests/datasets-delayed-multi-postmatch/test.yaml @@ -0,0 +1,15 @@ +requires: + min-version: 8 + +args: +- -k none --no-random --data-dir=${OUTPUT_DIR} + +checks: +- filter: + count: 1 + match: + event_type: alert + alert.signature_id: 1 +- file-compare: + filename: http_match.csv + expected: expected/http_match.csv \ No newline at end of file