From a76e659989fcedf16b1fdd1403d1806bd5878792 Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 4 Aug 2023 14:09:53 +0200 Subject: [PATCH 01/10] Improve timeouts of compatibility check --- compat/module.go | 6 +++--- compat/notify.go | 2 +- compat/selfcheck.go | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/compat/module.go b/compat/module.go index 77478f7e..ea5fcd69 100644 --- a/compat/module.go +++ b/compat/module.go @@ -16,7 +16,7 @@ var ( module *modules.Module selfcheckTask *modules.Task - selfcheckTaskRetryAfter = 5 * time.Second + selfcheckTaskRetryAfter = 15 * time.Second // selfCheckIsFailing holds whether or not the self-check is currently // failing. This helps other failure systems to not make noise when there is @@ -34,7 +34,7 @@ var ( // selfcheckFailThreshold holds the threshold of how many times the selfcheck // must fail before it is reported. -const selfcheckFailThreshold = 5 +const selfcheckFailThreshold = 10 func init() { module = modules.Register("compat", prep, start, stop, "base", "network", "interception", "netenv", "notifications") @@ -60,7 +60,7 @@ func start() error { Schedule(time.Now().Add(selfcheckTaskRetryAfter)) module.NewTask("clean notify thresholds", cleanNotifyThreshold). - Repeat(10 * time.Minute) + Repeat(1 * time.Hour) return module.RegisterEventHook( netenv.ModuleName, diff --git a/compat/notify.go b/compat/notify.go index 5cf6fccc..a92e654f 100644 --- a/compat/notify.go +++ b/compat/notify.go @@ -236,7 +236,7 @@ func (issue *appIssue) notify(proc *process.Process) { } const ( - notifyThresholdMinIncidents = 11 + notifyThresholdMinIncidents = 10 notifyThresholdResetAfter = 2 * time.Minute ) diff --git a/compat/selfcheck.go b/compat/selfcheck.go index 4515d93c..fd4a22bc 100644 --- a/compat/selfcheck.go +++ b/compat/selfcheck.go @@ -28,12 +28,12 @@ var ( systemIntegrationCheckDialNet = fmt.Sprintf("ip4:%d", uint8(SystemIntegrationCheckProtocol)) systemIntegrationCheckDialIP = SystemIntegrationCheckDstIP.String() systemIntegrationCheckPackets = make(chan packet.Packet, 1) - systemIntegrationCheckWaitDuration = 40 * time.Second + systemIntegrationCheckWaitDuration = 45 * time.Second // DNSCheckInternalDomainScope is the domain scope to use for dns checks. DNSCheckInternalDomainScope = ".self-check." + resolver.InternalSpecialUseDomain dnsCheckReceivedDomain = make(chan string, 1) - dnsCheckWaitDuration = 40 * time.Second + dnsCheckWaitDuration = 45 * time.Second dnsCheckAnswerLock sync.Mutex dnsCheckAnswer net.IP ) From bf6bf0705d44a0105b1c19094fe3b755dc03d7a6 Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 4 Aug 2023 14:10:25 +0200 Subject: [PATCH 02/10] Fix eBPD PID detection for UDP on linux and recompile eBPF programs --- .../interception/ebpf/bandwidth/bpf_bpfeb.o | Bin 35576 -> 35544 bytes .../interception/ebpf/bandwidth/bpf_bpfel.o | Bin 35576 -> 35544 bytes .../ebpf/connection_listener/bpf_bpfeb.o | Bin 36472 -> 36560 bytes .../ebpf/connection_listener/bpf_bpfel.o | Bin 36472 -> 36560 bytes .../interception/ebpf/programs/bandwidth.c | 4 ++-- firewall/interception/ebpf/programs/monitor.c | 20 +++++++++--------- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/firewall/interception/ebpf/bandwidth/bpf_bpfeb.o b/firewall/interception/ebpf/bandwidth/bpf_bpfeb.o index 82069f79c918416b840435302ee1fea0820da665..4ceee0d952171086916737035c977f85427fa7af 100644 GIT binary patch delta 2753 zcmbW1UufG^6vxlKzio}$waq$A)y>r9-!EOTnJrjZ|BZ^v{yYSnGpK{egay~ubkDAJtsF?ld?1^CtAhS&8WvE+U8=ezBciVx_a0vK3CUq|5&Xa_PNbEpe2&D0hmu+ z>-1zg>MgJwuhNcd2ReIew8q_hj_K1(r-7Ii|m0`i9!hW3}X3V&- zOf(ei0Ae@ta+y`_p!dici0>ak>tEU$XarH(g}i063F5AyZ%Ee&41m}m#-mugp!x^B zVnJOU3}vV_bYxtX?>CTp30#bKA~$Qs7D+!{DXke>B+bee$w+03WQ5`;biDrr$S7;k z3zmTba(2Kl$o9$}$#$nl)FRnY*&^9d(IWJ#`oVpfHIT8&F3FRuOTjqEcx9L5Db}Un zB*@qBk5M$~gYEbTn+9O#T9kC;u{I@)s@c13ykaPm^EQ0T$EbOE%-==S}1d##CUl%lABQaB*hRCB3YZ{QX-rX@2SBb9_u9OmYA?{^h+ObyXi_-A&@2G335J#c zGjO?dj>QO}nWGh)QOn(^5hr|4P7HRH23;`FRjRsJ!} z_^gct<>xfZ(2^P+zV!@Uh8A`d4g&c5EV4Rialg&jFpIBg4%}Ahx4ie89b8n=5%00b z?AkG1J5=UG6&E%8%V)u6^Rt9zsoIfp_I+1V?~gRk&5xX|7jLMhOG~1lN|!RCs1oBB z)#-7!x;LIycg7pkxold!m~Bw^vIX(2%3Xe1-66W%DqsIsHC^cuDV4DHDcna@=}Pne D7n_jN delta 2604 zcmbW1UuauZ9LLW&cWxDAOYJaK)Wqh0TD#PhnX$HMiLex$6!jr8l#b~r7Ta`oQrJU^ zGIKkI4gM_D7m46QkO`UT=mm!cY+aE(tYr`B5Umfo;DdAzVqgmXe)ryZe|L{N4=3Nx z@B2IFp7Z1jPokoI2LG6?-w-WI88!C zPkcP!&A)QM1k*T0GmbrIp4?|?oTI0ho?-d}rY|x5Dbr;~KegY{(JPjIl%5j3%8YNJ zHsBrIVv(PjZZQ3qqhlJ=>@l$*_y+^|6U=>$>9?4kVtUTe9rqj^D>(Y#VMoW7m}VD_ zeQm|}(j{Zppfc#~yTwv}Vfs&||8;cyVQ3C^JrVC5gJ2>`nH|OUeUY;j^*c495~-xSL@Mbn(Zn>BNM+n5QZKR+g(rm?bC*cH%1RWj zz{a#sR-iHcblj{7*1&viy6~tt%v#+MN+)R5>JB7wzI;lkFFG(fW@SZ)>3~_A&kI#1 zf1EM-6D*(n9Aon5SU&mljLFBN(5dp~11Z5S`Tqx4W(&wKF{Ta5Hn%QNVNAZk@@ajI zG5H%Nr$rE3SfLi)v=U6F7S?UX7Oq5*cNo(KjTS?3#%$plworCjQGCnQ2DVW8ZB7ey zgIbobSr_Ui@vzOCuscIu;0$1Q@v(`jzL-s6nP3mt6&kM~@{y1auCs;oDImYDAv3&C98I!MDOdYfiP`q!0WwwAeC^4oD$}FG!3S;sO zmQQ|-G5H%Nn;q1m?hb0^S6;Uk;76$SL~+3mW7?q6VldvN4r&AL4r;@!gZL3@FHpx4 z)++p;(lW%u7UM_QIZGd55@+np?Dt3e@EJ6)Frpj(px-dbEQHAgoAKf%e`Rs#F8nj= zd(hS4Ex3HnH+sb0J7et~X>s;G#v;1Tf;Ho3iLBFZBdygByc>LeAz1z7z0(KeA~Y|2 zC+lE*n1|JZ2aXq};fUT2*9$pV)sc1lUGTtl-H^Y-{KE5c0*r+rHQ5Qxg_P7Gd6DKh KoR`74=>HEkz?KUD diff --git a/firewall/interception/ebpf/bandwidth/bpf_bpfel.o b/firewall/interception/ebpf/bandwidth/bpf_bpfel.o index f7974b55888cb2f21b1cd45d1062cd0475b17f72..e52039fc80359be3c6a6da1d13094debbe788375 100644 GIT binary patch delta 2606 zcmbW1UrgIo6vxlKKN-|Nl&%Y9nOi4i&?>s&Ohj1=%KpIgp;KeX+!Bl?qGLcA5@Mpo zn2bWQxb39Mh$c1qz%WJ@2-L(#lktGcOo_Un?qM1=7#}q8g+!Nl&ad}J&u@?WrNH-d z?m72&@AsUtp4O{reKxGdFGaSMsl77?%3Ak&zS;$#0PZ}BiKc+(Vr_wrF{$_P%chVo z90q>MJ1LDm@F3o33M3mTO629G=Q&KzZS>Pp$S!3*DD<;JcL|*k`VFDa2wlFNcLRIR zcM9>O4HF{iL!l>yo)`ME&{u^1QRp?H{}I{~DIOvaxuXanEQy914vXcdg+4Cyzy4xX zHzuNiU9XANGeW;B^auC}Wqu|ii$Z@Z^e?y)4V;fvZCw|^(tC?9-HZ*fQ2TZfsTVpb z^kJco3jLDMr||t)py4eMd0Xg=&>su^DgGAQUH*lLlyAasVwHD&FGAPwK=bbC?;`S- z&^Ls>{l4P${6g2xTx#~#%>4AizEZ3@vHzZ{g*msApQCx5FHE}4$BCzAUO(}tN*=NR zDy9|m(ZUZJU_C@LKC58Z@QxaQKvqFYa0T&M!%y(~=L*IJR}g0m572sLPQj#LFY$DN zt6_o?!ix&BCV|;UoO2jGJ;BGWP;~2e3jQ`xKizKAs)BXHVPfkVU5h?^jg!pR6;v3W z!XPhCfa>_++&_q6i-t16CB!=o+ZNSmw?$zM!)D#Ks9w7*B6eA{M>`fh z1?*V`1u|v>+oOlH+aqG*QNf}{?Y4+GXX+>L&0b?sn|53Dq~5S7P6J@Y>AupUXY__e zY5ci2sM4tVf)TcaE#RT4rnm6JRxJ}ls%X_Rw)X{-HFW^~k2J(h7PF7I!*D0BPip8C z>?7_H`CQ*8nCpi`KGzQm=6W-eNpKF}IPM%^TomvI8Ns}PGn7ufW4KeF75QAB6U_BX zHn#%U9h}yXHwnyM;#I@;kk4wkF4#xB<}hANm<}##s4^Y0hkS_!(Xc(_%Nq8`4(2tq z7@k8<;d<+#MwDGV8X1 zl<(>w@z726!-9QOpDM8FAQiYeNIWhI{8W$;%o|J^b`Id`Ak}9@KG)|2bN!OdP6xx5 zyMx55roirCz2)v8@tU)aE2m5c_gU@^wpbe-w7-M?U=mD5SJghxh_7$4b!zm9Zcp8md= t>c)7AI5n%j#tWklW1eZWUjO$BZXSzaezXib#u`-@o*QdcL%1?l_djy7i6a02 delta 2565 zcmbW1Uuau(6vuzR-`$@UT$9kc+o2=1Y1VXqoB6M!f3lJmTd=aJgX&aTq%y2)o4J;X z4=IZFw&+$k6zYSL2OlzFkeC(&n;Ni{Dn6`b59tuy9_k7*Sp~H)2A*^CyUDqG+$FvD z`#HaJe)-++Ica`YZOy8gkj$nV9xIbCWID^DPnNF-g^)r#@+!od&GN;41$vE0egi)> z2Yk_xpsS(})9@pD;C{0id}?Hn_prz-Ob;+U#Pm4R_bN*} zPeQEK49>FEB-39reHAXZnn$m*$nQ-5#q@oq%lE^-t+jh9_m{r1mgz$fZVPlbvB+~w z_cA@e^eEFGFns|&Z!^PRu*g?TUt#)3rhkU@wj<#+7OB`#I_)hMzsvN0&>1%4 z*&VDtP$~#AeVFO?OgiGJQ#Cc;_usG>EThoWS?x z7QL*B7NuZm*e_GCKJ0HGOUMEmnp*r8y2z^A$igaFbsG+i_~Vg!oSQF|IH9wMeaO9< z->3DY5`Bz)$Y)qS)sHZy`f-*|^%IP#Ue9D)I0tYXcMdSe3TT4_W7@zON~hj2+^Ns9 ze5%herut=@qe9pnOet|qClGs(^P24;pI720V;}N{!|-WLcW_aOYTY4w$d_>tHQPh} zixQ2zgV&Teu6YIS#AwKOo%1(Pr?y*4ETQ+jgEx?I4vNEm15TaO9emtycM!Q(pU>`K zjp6Pf@)^Bu8;JR?4kC{~P(Q)ghxJy0bqBG))j{MrR#1rr3C6TRQnPaaR|l~^%krr{ z$C&DuZFV{sGTa?R&g%lZg9i;cd*59cksAT?4bP{>@eIN>@tcSOu?h) z={NW^-mx?1e>fJ#NurC4p?~^sXijLJ)J!i<>z6fe!pTwpLszfDrBQ$PGxn?q(WUox zxc31j6Y0C1vyf-}EKA@U^S3+I54{`2#sZm{u|AKy2%8^$FY}Q8I1Z`FD(IMe2O6y^ wSj1-vpB)xHr;x3A`3Jl=^(-__mqB)_NmUQR-&1YUg79<%?=ig3L3+CWe|Dgkr~m)} diff --git a/firewall/interception/ebpf/connection_listener/bpf_bpfeb.o b/firewall/interception/ebpf/connection_listener/bpf_bpfeb.o index 9545daa42e82509fae977097f5c4200b6351a09f..e8e7cdb7b368369546e5e3dbddb1b97f06c90825 100644 GIT binary patch delta 4602 zcmZ9Oe{7Z28OP6g+tMF^md)B;ndQ<;f7~DKEvU3oXs<;E2-AwV)lKCt(+#AUy5Rur zdT&s%{0w-aA-XMhhPf^?4px#inUkBz5+;yzrjCmLG~$*fq7umxwq zD0SVApbo!KG_tr8uGmsD{pA0{(sz&rsCgf z-90i{l1yfrh(q&RuT;a^#e}9*NeVuX?VezqhCjoPDYXdMaN${{s+B6v!jss0lwZoP zQqi|Ii~X-~6`X@Vg=26Y{s+83tJ*V4HO>k+Q~M{xmu#%(I2)QgA|Leegrg@<4R=i!&= zzyM)PjjzI+;S~I93>%fI6Z@N7n0;LyK1YVQYmo%^7wmKSQySl=@4ZUN+BHscVbV1z z?%((^;$1`<{v2jDHU03vVRlPRR;i{kYd;J#3%vn*9u8sW#!@vkabcFBFr2WAt?5?F zt*~TKeA?kV;Y>`cPzu5NE8#m}X617?{E+39FlqDcbZpvYc@@0N@>k)k<<;G&J1e6f2&~aG{GjDW;O|)845Ot)x6*Moz&42Pq?XVr(Py{_hv7riOg-bz z^<@O1t27Nix6FXi-&pR4Pg!QbC{LzzR$2Ic>OK5(@J9skO*ju$JkJ#BgE_jpWRdCue z%i=F9RqTHA;Ih^F0ds6wQa^0Qmqq42$}qENBHP@nxFfQ-FYxzL_}lw>Y50fOnTD5z z2W^D1dK*n|M?}ZXK*uwB&kM_|Ge@}jR-DB#2Q#2f!LM5u`>67RlKmKa8;Fo}}s%>qGWp*I*VkJcq$KGG!M2 zLoWldknt}m27;e}Z?gP-d^&`Sl=lOU%U@VKvyazV`wp1Q_(>#!aR*$4t@@n)aT*DmM3Ax)J&uZQ~Xyj z{fkn)h*1PHDax`wA()vI<+1+}PmTmuPsPV9rwEQMs<&g$V!uGEWRDxhFH!UCNQr-9 zF0V-Hf10m!MlxSuU@}q?C6<>A-&EcmU-c#V~ItrY8$>CQW9tl_u zQk6LAG`wZ{S0?p&lUrFGde5N3QPMOX8pHwRHn$p9oV2;fhE#b>? zJKfGBs1gRc73lg|YPG)1dF-E0+@S6)KASju z>)UWh@|S|WeuMJLu=@v<=-s?+y=+c(+Q1^Fccq_c8S*zAceLD~H_#1tEAOcIryVW+n;k9w{nI*QJM5e`pru3B(b8ea(Q<>sju!u; zju!u%qs9NY)xT8Uu(Xd_eXm3;1LU0!;&3L=iM`4@$?I581)8g0^c5;eAzyQ2>`|}G z>J6rS-~+R6ps{u#-Tad03NK9iD%if|=E^{1OD<>$_XaKgM!_@uGpi*C>6x>7?0>bq z)1|$R?+7XWGPs)l`fkp*-A3;Vo8rNjJ*`Y`FsaMT*kHAuGE;;1gevK|go~kXFu?&B#AQ5;=rjiFWU4yzrRX8Tx1v8ma;z4X3o&Ms?!kPNHZc9JfoT)H}gmiLWJ3=L{;?+Vkq@01=jHTzfVN|WB-rq7t8`@71A{cg&uF_YLP zOwEBVoj1K)Y3Dj&CJuC!PXu0%zhlA&+n8wY!LIUSfvtSN4ch5ri#2tA+x delta 4528 zcmYk9e{hsX8OL|uBqX6gLv2igZMbrzxw~9)sm$bJ0^~x{LTPG2`-4`h838A;v`yPc zQ+~Zytr|*T((dRm*v?!>hEnYx=L97qRc|cJLI5#N@F zQj1>7wwhGYuEnbm^ZTpkJloBgqNO@%t`#L0HjC$MgR$1)6I!>8O_wAReu$1We?3aw zezKUclqyNVr?K59t>E9lUsr0m@VOV1s#2;r4PU^%0VApWBObJ^O=JHPTnT63_uv?u zgZ~aM)v9&^zMf~lmd@*Muob_BYb^f`uCsgwUSatxt9$}x619`q*IJ%}Win!)h9AoF zQi}R7^I#H7D+pJHNpFLzv2RnpQr|SF4sACo#ki`T$SF3lI>q?)?;_44df^XXR!}_v z{}a9yPAe4w1N$gkMn_-3o`V--XJ83hB*cSN)=)-duY_qBiCb=lC5mEig;&CUOskp{ zqFXB9PMEV0@EZ65%U$rJmhp`c!ocTF_$kYG!9$kWmdNv#*TLVnd^dd5@_MD#ML)3lp7y)9^)@(A8vMBB#Fy=isa6ax&ph7YCk-IMqH*txTU3`2^-f z=$ukf4>!Sq8quBp!it@Ks8DWNk6cgaimpY!Ub+m=volB};LmtHy=z&;-z;iur6 zEI*CU3gIH_Y~oeGvN`OkKcJFnwzLiZC_&5N5oh6kFMlMX-{hG#f;i zG%G2}VgCi5TnVyw4HK4AOpYU}2eGHIpQBdZ4Ys7=B4vb3YPe$lw<@9kVdAU9{=YG> zn85zAWn^4B7CY_Z^4|Y zMjthYI@;^rq-GU?3DFM$mHSPn;GbJw2fuA*I#&2+Z=?zr^xrxaMSm8(SQ1bGXalbq+t|@D_(3 zcX(%MFn%!a#$l(zIQ)jgCmep;;qwmv+2Icy{>b5vXW1OT$<0yjPEaG%4^IQ-2#KO*aIa+ch5#Butm!>1fR=diQnrVkzaf6VBba7|Ic zlDNs?Tg{m@VL#$noP{Nvy-Ymd_-u3dX@~a~cu{+S69*kW?C_8CESIN>zv)!`%%r;# z`n1V*Rn?qz?3WyV*GzRKRyj*eIOn{%xZv4r&$*1!?CfWA<5*AkU&1EQw`6QX-(w3( zxJye%O>VG3C&sP~e&XqBbM0B*kJ7G>{0*hTN0mM%(;aX$ucR8X`l`~$Ti~>#xe3*< zqow_5UN4U+J-!^~hfewrwA5#;-lVjD>T`%=1;Z(RtyI?0@^v+Cbw=q4@t?5zveJL} zG(73lOZy9smiALtUsw8?T;DT!z3i7NsW)@i)+I1BktH$Hy7kH{!ya5rqIdG^Dg#M0 z0&TL-B^JM-yjJXqF zKui1CfW-Lil!(p-M6}e~fYkj}dH#s}SlEb&VWb*;fh-djEen};wD`|B8vo$JcKqdY zb*?@B^2s`<#h+}o9}J*Fa|TZ)q$7@&2F;FULc#xv40!up$~z?foAUL67XMyHi+`V^ z#eZN{`*a`;hGq?DX^?ibG#GZY_>VeT{0})={Eu1vn(~fGeb%X$_TyG}$_m6Gmp8

!JJ!ppqI(_3MtqnSLd*Op6hL= z+4y3}tlWFpQ#0nm-h{jrbLm6KURZj{g#7z-nHlh_bj8@PUtdx(Np;!S@fR+7m2ZJ^ z^oq%((lb1?Ir828m0KY`gl22X21T3h;d?5@r_8CjB-$Jq9vo}kw?liwS4{TBU+J_N z*uR!>$M+}osF~j1QJ%q%V8+6)~? oO7($`@{xSQ^37)EK+Al8%1UKg54L#2Q)b}cYH!c`CVMdSf5mn_WB>pF diff --git a/firewall/interception/ebpf/connection_listener/bpf_bpfel.o b/firewall/interception/ebpf/connection_listener/bpf_bpfel.o index e4f9f3f88655d6f1bc52b7d06af6de5fb4d59bfb..5241f58ebd540dab34c823871c8d4fafd3b05c6e 100644 GIT binary patch delta 4520 zcmY+G4{TM{9mmhT^slt8Ezs8|t9!Pzbgl5H%37%OJz5wrP@tBrY#=p5QiPf!$bWjD z{9A!SN=|f4r>RCamKg^t#x8U4GBN1{l6KSaXIWB)7#3u6HYRKfy3g;vbK&gdop(Rq z-}#+$e&=`YJ@?-HsXO?x+f(5#+JC$<;pP`BS1Gl6T@L?BjdR8yo?kO&tS|QPe>Zg# zHKK31Z&%l=yWrK&1mJ%2{9m0=&y`1U>|F zl5n#jUPMT^t<*ORx52+Q+yS35ycs?V%Q2LoUWE5@?WKMUf1eW-ynZUG3DBRQ?k^SYgeAf$ zjM=Q0)Rj6_dQ06dXRp3h7xE6%x`x>kwD!Vl;W+h8!?KEgSfWb8@4+(iFnkX7Q{k%2 z3$Q%-nXbx~hfC~vu5$ineDYnDal`n8@VN;SoTnoEyH%;}j>9?d(+u7U&oTpbFi-(@ z8FU?dkKt!vIR?bO-c_T1&bvi;BlZ&Gvk7~Z;jP#oGQ16LsN&d=4%-p(=nLTM?X4rp{;XSZqC<*r%9)@3mJ6M?FF#RRG4vx1V`Vr3{60khJ z>)|B)KK3Q}%kB*tj>97aw?X{j^Vk_XO(who)8v;M`tLKM&VO{czSR2)yDWy{q~D)n zx;r_B>F8tg1Wt*@NMp_;oK{_OMxx?h7ic{C=*|AdRb-g+^-g!&!iPe>zpw|pq zBCM<*_C4X>8=ecF(N|*+hCjlB_!`b?DqgT`BGfhP6xPdI-{LPV z&YL8yDW9;#kKi;r7;YSP?0 z5uLFbE?NAE#h+XJUyIWx>q9dOXN;YEXSiY6Pm`Py%RbfOn8ow-<(VPx3Cq%9@ivS1 zTl|v6Z(962i_cno-r`Rz{=%@o9ZpntHk5=%NAIlQ9*fKLZyHLyDV8N>@gj?tTikB( zR*UyqTzB`K8DFsMuUXue;sQTXc?u{#pVaM*QRkd~xiJ*}$m;O1#n<(@#^{VMEX$V` z=g2D~xSuU9v3P3Vv{_e6_4BjK`yOsvnUlVqxsT~LHdHw^eM1|*a-Faq+T?kaM;+D1 zSkF>_R;tu&bcdx2(DIHB+Vju}qet-n)(J;-S$ZtG$I=DpBSt?kf!A`sM8=9%`Lde4?<{&B3~TGu}a#p{ZG zf6Iej7yZF~yS^{b@`q6S0aK!aL%*0=G5xUK4FGR}=-Ea3yhs}Y(bVyiQI&@iDCg`!W zOmM`~;@@j&@jqttFBre@C|hFm;vmAL)jP~%m$2)v}=@&aEhb2I}-0(%Fx~4C8E_CvAwHI=V`sR5P z({l&XQjT)T=-aXTPj2xs&L=5i%AzK1t=oUJt9Y51DdNH0YOSqxZdYURY0Hv<$A`Ao z?Op5hm+}20C75Mx-%yw5xHI5E_T9P!Gb zY7@_B3)QH5kS$x&e%0o~jC*y=T+4`MPaAUY{eIw>)>XaNGo#T&QHfGoshkELCnhPG zDG<+8>JItPc9+LjC`bpc*%eZI`{`>Nywr9Q@9fO`-8Pq-ZJgKxt3 z!u=DJ{wBpws?&(<$*G3Fz=f7Sgi9=+foE7g3qQj;2|^7p=zQ$LgYaU@H{fNKWyU9r zo*7Q8#_}qjaV*bTPQpppVDE#w;3s8CMM}L6H^XswAN(k0RjkxeST>{&J_*xp+dzg= ze}!|!hw>4;M8+9InqYP-q2h?ki1XN+;4feay%oL%kB5`6DoxqD;UN4&2I_-DuuQB9 zi4T^%7QquOm%>vmPlIPzu7qcoCQ=>Zg-Xx04RvsXxT5f_@SzZaVr~nhIR@gra@D|HO@JsM0|F9u`jgWAgsBc#-rVx*DG_YcSPa4df zS&@YiEGuwLQYXV)H<@^->F^VFh#34+%QbM5<$18|q?CtEOVxb6#2l%L>c`BdRgv5k zv<_n~Kf5MW9htt$`o~FDdfq1Z8SMU2;f=6FD0gIoSzR5`5wpGeMcrX?XNMEpY5h5~ zCu!XcKLy9BZ&{XA9EK&TKKKMIBM-u-U_TX}8a@Ean?KxBS@Lp;JF zPZ*yYFu^5M5zqgngo?v~DTL49P4H|xP%8u7gIxw)3+G#22g@}e_Vu31*f%1@8?leI zKF?u~SZ>2U)$$g&Y6{nebbcNoufA|QEU&n52i#ygB;e&Rv8p7&wOH%i2Mo&Gx8GSljjmKk1;wag*b#g;jO zy4><`c!uR5Tx~fEo}1!w`EKinY(q9Ii;;kG;76^UPmBJUx%EJG$+K7pBanwjIvogGK zhwIGEx^Uug$I|TZdWTwJeZT3N zQ(Tx49a=%Y!{g2AIgv!EV~IJuz~QA1uXcEy!yOJ+j~JS<%dx-i@b6Qc??)u!ns!>CfYo15SIG_6v?KKo45|4gTK0wYqKf`^7PS1MvGNTB20yV5L^N2%Q=9{Od5PaTjlN+g6fOSgjHLbLB9oz1)(w2FDfOS&$!yW{i_$puL=%z8U}7~wVBshKmLqk5xnGT^LFR@-pcK*+Pl(e z4!-=pPMVe-^L5Z1*%8y-=IV~xte%Y240&ey&Y12oEjw%VVRL}zadVYtzbV^Qo7JD0 t^7^w`yDKK`*miD6*_NA|yJA^`yaIkP)4O8cV5eEr^?>(zyE)i3>HiD8J-+|| diff --git a/firewall/interception/ebpf/programs/bandwidth.c b/firewall/interception/ebpf/programs/bandwidth.c index b56ce763..04222353 100644 --- a/firewall/interception/ebpf/programs/bandwidth.c +++ b/firewall/interception/ebpf/programs/bandwidth.c @@ -105,7 +105,7 @@ int BPF_PROG(udp_sendmsg, struct sock *sk, struct msghdr *msg, size_t len) { struct sock_common *skc = &sk->__sk_common; // Create a key for the map and set all the nececery information. - struct sk_key key = {0}; + struct sk_key key = {0}; key.protocol = PROTOCOL_UDP; key.src_ip[0] = skc->skc_rcv_saddr; key.dst_ip[0] = skc->skc_daddr; @@ -134,7 +134,7 @@ int BPF_PROG(udp_recvmsg, struct sock *sk, struct msghdr *msg, size_t len, int f struct sock_common *skc = &sk->__sk_common; // Create a key for the map and set all the nececery information. - struct sk_key key = {0}; + struct sk_key key = {0}; key.protocol = PROTOCOL_UDP; key.src_ip[0] = skc->skc_rcv_saddr; key.dst_ip[0] = skc->skc_daddr; diff --git a/firewall/interception/ebpf/programs/monitor.c b/firewall/interception/ebpf/programs/monitor.c index b084a5d8..70bb19de 100644 --- a/firewall/interception/ebpf/programs/monitor.c +++ b/firewall/interception/ebpf/programs/monitor.c @@ -76,7 +76,7 @@ int BPF_PROG(tcp_connect, struct sock *sk) { tcp_info->ipVersion = 6; } - // Send event + // Send event bpf_ringbuf_submit(tcp_info, 0); return 0; }; @@ -95,15 +95,15 @@ int BPF_PROG(udp_v4_connect, struct sock *sk) { return 0; } - // Allocate space for the event. + // Allocate space for the event. struct Event *udp_info; udp_info = bpf_ringbuf_reserve(&pm_connection_events, sizeof(struct Event), 0); if (!udp_info) { return 0; } - // Read PID - udp_info->pid = __builtin_bswap32((u32)bpf_get_current_pid_tgid()); + // Read PID (Careful: This is the Thread Group ID in kernel speak!) + udp_info->pid = __builtin_bswap32((u32)(bpf_get_current_pid_tgid() >> 32)); // Set src and dist ports udp_info->sport = __builtin_bswap16(sk->__sk_common.skc_num); @@ -119,7 +119,7 @@ int BPF_PROG(udp_v4_connect, struct sock *sk) { // Set protocol. No way to detect udplite for ipv4 udp_info->protocol = UDP; - // Send event + // Send event bpf_ringbuf_submit(udp_info, 0); return 0; } @@ -138,21 +138,21 @@ int BPF_PROG(udp_v6_connect, struct sock *sk) { return 0; } - // Make sure its udp6 socket + // Make sure its udp6 socket struct udp6_sock *us = bpf_skc_to_udp6_sock(sk); if (!us) { return 0; } - // Allocate space for the event. + // Allocate space for the event. struct Event *udp_info; udp_info = bpf_ringbuf_reserve(&pm_connection_events, sizeof(struct Event), 0); if (!udp_info) { return 0; } - // Read PID - udp_info->pid = __builtin_bswap32((u32)bpf_get_current_pid_tgid()); + // Read PID (Careful: This is the Thread Group ID in kernel speak!) + udp_info->pid = __builtin_bswap32((u32)(bpf_get_current_pid_tgid() >> 32)); // Set src and dist ports udp_info->sport = __builtin_bswap16(sk->__sk_common.skc_num); @@ -176,7 +176,7 @@ int BPF_PROG(udp_v6_connect, struct sock *sk) { udp_info->protocol = UDPLite; } - // Send event + // Send event bpf_ringbuf_submit(udp_info, 0); return 0; } \ No newline at end of file From b645e440adf8d21f7176795e7aa2226355ec4bab Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 4 Aug 2023 21:41:37 +0200 Subject: [PATCH 03/10] Copy resolver IPScope to DNS connection IPScope --- intel/entity.go | 1 + nameserver/nameserver.go | 16 +++++++++++++--- network/connection.go | 10 ++++++---- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/intel/entity.go b/intel/entity.go index 0394d96c..c1685377 100644 --- a/intel/entity.go +++ b/intel/entity.go @@ -60,6 +60,7 @@ type Entity struct { //nolint:maligned IP net.IP // IPScope holds the network scope of the IP. + // For DNS requests, this signifies in which scope the DNS request was resolved. IPScope netutils.IPScope // Country holds the country the IP address (ASN) is diff --git a/nameserver/nameserver.go b/nameserver/nameserver.go index a13e98cd..19251007 100644 --- a/nameserver/nameserver.go +++ b/nameserver/nameserver.go @@ -22,7 +22,7 @@ import ( var hostname string func handleRequestAsWorker(w dns.ResponseWriter, query *dns.Msg) { - err := module.RunWorker("dns request", func(ctx context.Context) error { + err := module.RunWorker("handle dns request", func(ctx context.Context) error { return handleRequest(ctx, w, query) }) if err != nil { @@ -187,6 +187,13 @@ func handleRequest(ctx context.Context, w dns.ResponseWriter, request *dns.Msg) if rrCache != nil { conn.DNSContext = rrCache.ToDNSRequestContext() conn.Resolver = rrCache.Resolver + conn.Entity.IPScope = rrCache.Resolver.IPScope + } else { + // Get resolvers for this query to determine the resolving scope. + resolvers, _, _ := resolver.GetResolversInScope(ctx, q) + if len(resolvers) > 0 { + conn.Entity.IPScope = resolvers[0].Info.IPScope + } } switch conn.Verdict.Active { @@ -297,11 +304,14 @@ func handleRequest(ctx context.Context, w dns.ResponseWriter, request *dns.Msg) return reply(nsutil.ServerFailure("internal error: empty reply")) case rrCache.RCode == dns.RcodeNameError: // Try alternatives domain names for unofficial domain spaces. - rrCache = checkAlternativeCaches(ctx, q) - if rrCache == nil { + altRRCache := checkAlternativeCaches(ctx, q) + if altRRCache != nil { + rrCache = altRRCache + } else { // Return now if NXDomain. return reply(nsutil.NxDomain("no answer found (NXDomain)")) } + } // Check with firewall again after resolving. diff --git a/network/connection.go b/network/connection.go index f19422d8..1d9edd11 100644 --- a/network/connection.go +++ b/network/connection.go @@ -325,8 +325,9 @@ func NewConnectionFromDNSRequest(ctx context.Context, fqdn string, cnames []stri Scope: fqdn, PID: proc.Pid, Entity: &intel.Entity{ - Domain: fqdn, - CNAME: cnames, + Domain: fqdn, + CNAME: cnames, + IPScope: netutils.Global, // Assign a global IP scope as default. }, process: proc, ProcessContext: getProcessContext(ctx, proc), @@ -367,8 +368,9 @@ func NewConnectionFromExternalDNSRequest(ctx context.Context, fqdn string, cname Scope: fqdn, PID: process.NetworkHostProcessID, Entity: &intel.Entity{ - Domain: fqdn, - CNAME: cnames, + Domain: fqdn, + CNAME: cnames, + IPScope: netutils.Global, // Assign a global IP scope as default. }, process: remoteHost, ProcessContext: getProcessContext(ctx, remoteHost), From cbb4a0f2538a05a47e859debafdf264618c2cdf5 Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 4 Aug 2023 21:42:30 +0200 Subject: [PATCH 04/10] Fix expiry times --- core/api.go | 2 +- resolver/ipinfo.go | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/core/api.go b/core/api.go index be5fffad..6a653909 100644 --- a/core/api.go +++ b/core/api.go @@ -224,7 +224,7 @@ func authorizeApp(ar *api.Request) (interface{}, error) { Title: "An app requests access to the Portmaster", Message: "Allow " + appName + " (" + proc.Profile().LocalProfile().Name + ") to query and modify the Portmaster?\n\nBinary: " + proc.Path, ShowOnSystem: true, - Expires: time.Now().Add(time.Minute).UnixNano(), + Expires: time.Now().Add(time.Minute).Unix(), AvailableActions: []*notifications.Action{ { ID: "allow", diff --git a/resolver/ipinfo.go b/resolver/ipinfo.go index 58bfc98a..6c1eed31 100644 --- a/resolver/ipinfo.go +++ b/resolver/ipinfo.go @@ -4,6 +4,7 @@ import ( "fmt" "strings" "sync" + "time" "github.com/safing/portbase/database" "github.com/safing/portbase/database/record" @@ -170,7 +171,7 @@ func (info *IPInfo) Save() error { } // Calculate and set cache expiry. - var expires int64 = 86400 // Minimum TTL of one day. + expires := time.Now().Unix() + 86400 // Minimum TTL of one day. for _, rd := range info.ResolvedDomains { if rd.Expires > expires { expires = rd.Expires From 9e4bdfb341220f1ef039e81ec70091af19c63534 Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 4 Aug 2023 21:43:46 +0200 Subject: [PATCH 05/10] Add support for DNS RRs SVCB and HTTP; improve DNS conn handling --- firewall/dns.go | 27 +++++++++++++++++++++ network/dns.go | 64 +++++++++++++++++++++++++++++-------------------- 2 files changed, 65 insertions(+), 26 deletions(-) diff --git a/firewall/dns.go b/firewall/dns.go index 394e334a..5c3542f1 100644 --- a/firewall/dns.go +++ b/firewall/dns.go @@ -45,6 +45,7 @@ func filterDNSSection( ip = v.AAAA default: // add non A/AAAA entries + // TODO: Add support for dns.SVCB and dns.HTTPS goodEntries = append(goodEntries, rr) continue } @@ -257,6 +258,32 @@ func UpdateIPsAndCNAMEs(q *resolver.Query, rrCache *resolver.RRCache, conn *netw case *dns.AAAA: ips = append(ips, v.AAAA) + + case *dns.SVCB: + if len(v.Target) >= 2 { // Ignore "" and ".". + cnames[v.Hdr.Name] = v.Target + } + for _, pair := range v.Value { + switch svcbParam := pair.(type) { + case *dns.SVCBIPv4Hint: + ips = append(ips, svcbParam.Hint...) + case *dns.SVCBIPv6Hint: + ips = append(ips, svcbParam.Hint...) + } + } + + case *dns.HTTPS: + if len(v.Target) >= 2 { // Ignore "" and ".". + cnames[v.Hdr.Name] = v.Target + } + for _, pair := range v.Value { + switch svcbParam := pair.(type) { + case *dns.SVCBIPv4Hint: + ips = append(ips, svcbParam.Hint...) + case *dns.SVCBIPv6Hint: + ips = append(ips, svcbParam.Hint...) + } + } } } diff --git a/network/dns.go b/network/dns.go index bf048d46..2a1628fc 100644 --- a/network/dns.go +++ b/network/dns.go @@ -8,6 +8,7 @@ import ( "time" "github.com/miekg/dns" + "golang.org/x/exp/slices" "github.com/safing/portbase/log" "github.com/safing/portmaster/nameserver/nsutil" @@ -18,6 +19,13 @@ import ( var ( openDNSRequests = make(map[string]*Connection) // key: /fqdn openDNSRequestsLock sync.Mutex + + supportedDomainToIPRecordTypes = []uint16{ + dns.TypeA, + dns.TypeAAAA, + dns.TypeSVCB, + dns.TypeHTTPS, + } ) const ( @@ -30,6 +38,13 @@ const ( openDNSRequestLimit = 3 * time.Second ) +// IsSupportDNSRecordType returns whether the given DSN RR type is supported +// by the network package, as in the requests are specially handled and can be +// "merged" into the resulting connection. +func IsSupportDNSRecordType(rrType uint16) bool { + return slices.Contains[[]uint16, uint16](supportedDomainToIPRecordTypes, rrType) +} + func getDNSRequestCacheKey(pid int, fqdn string, qType uint16) string { return strconv.Itoa(pid) + "/" + fqdn + dns.Type(qType).String() } @@ -39,28 +54,28 @@ func removeOpenDNSRequest(pid int, fqdn string) { defer openDNSRequestsLock.Unlock() // Delete PID-specific requests. - delete(openDNSRequests, getDNSRequestCacheKey(pid, fqdn, dns.TypeA)) - delete(openDNSRequests, getDNSRequestCacheKey(pid, fqdn, dns.TypeAAAA)) + for _, dnsType := range supportedDomainToIPRecordTypes { + delete(openDNSRequests, getDNSRequestCacheKey(pid, fqdn, dnsType)) + } // If process is known, also check for non-attributed requests. if pid != process.UnidentifiedProcessID { - delete(openDNSRequests, getDNSRequestCacheKey(process.UnidentifiedProcessID, fqdn, dns.TypeA)) - delete(openDNSRequests, getDNSRequestCacheKey(process.UnidentifiedProcessID, fqdn, dns.TypeAAAA)) + for _, dnsType := range supportedDomainToIPRecordTypes { + delete(openDNSRequests, getDNSRequestCacheKey(process.UnidentifiedProcessID, fqdn, dnsType)) + } } } // SaveOpenDNSRequest saves a dns request connection that was allowed to proceed. func SaveOpenDNSRequest(q *resolver.Query, rrCache *resolver.RRCache, conn *Connection) { - // Only save requests that actually went out to reduce clutter. - if rrCache == nil || rrCache.ServedFromCache { + // Only save requests that actually went out (or triggered an async resolve) to reduce clutter. + if rrCache == nil || (rrCache.ServedFromCache && !rrCache.RequestingNew) { return } - // Try to "merge" A and AAAA requests into the resulting connection. + // Try to "merge" supported requests into the resulting connection. // Save others immediately. - switch uint16(q.QType) { - case dns.TypeA, dns.TypeAAAA: - default: + if !IsSupportDNSRecordType(uint16(q.QType)) { conn.Save() return } @@ -68,18 +83,12 @@ func SaveOpenDNSRequest(q *resolver.Query, rrCache *resolver.RRCache, conn *Conn openDNSRequestsLock.Lock() defer openDNSRequestsLock.Unlock() - // Check if there is an existing open DNS requests for the same domain/type. - // If so, save it now and replace it with the new request. - key := getDNSRequestCacheKey(conn.process.Pid, conn.Entity.Domain, uint16(q.QType)) - if existingConn, ok := openDNSRequests[key]; ok { - // End previous request and save it. - existingConn.Lock() - existingConn.Ended = conn.Started - existingConn.Unlock() - existingConn.Save() - } + // Do not check for an existing open DNS request, as duplicates in such quick + // succession are not worth keeping. + // DNS queries are usually retried pretty quick. // Save to open dns requests. + key := getDNSRequestCacheKey(conn.process.Pid, conn.Entity.Domain, uint16(q.QType)) openDNSRequests[key] = conn } @@ -103,12 +112,15 @@ func writeOpenDNSRequestsToDB() { threshold := time.Now().Add(-openDNSRequestLimit).Unix() for id, conn := range openDNSRequests { - conn.Lock() - if conn.Ended < threshold { - conn.Save() - delete(openDNSRequests, id) - } - conn.Unlock() + func() { + conn.Lock() + defer conn.Unlock() + + if conn.Ended < threshold { + conn.Save() + delete(openDNSRequests, id) + } + }() } } From a98505b7da4880036017609a9251c608f310ee40 Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 4 Aug 2023 21:44:18 +0200 Subject: [PATCH 06/10] Fix and document netquery api endpoint permissions --- netquery/module_api.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/netquery/module_api.go b/netquery/module_api.go index 1a559dbf..860f4e4c 100644 --- a/netquery/module_api.go +++ b/netquery/module_api.go @@ -89,8 +89,8 @@ func (m *module) prepare() error { if err := api.RegisterEndpoint(api.Endpoint{ Path: "netquery/query", MimeType: "application/json", - Read: api.PermitUser, - Write: api.PermitUser, + Read: api.PermitUser, // Needs read+write as the query is sent using POST data. + Write: api.PermitUser, // Needs read+write as the query is sent using POST data. BelongsTo: m.Module, HandlerFunc: queryHander.ServeHTTP, Name: "Query Connections", @@ -102,7 +102,6 @@ func (m *module) prepare() error { if err := api.RegisterEndpoint(api.Endpoint{ Path: "netquery/charts/connection-active", MimeType: "application/json", - Read: api.PermitUser, Write: api.PermitUser, BelongsTo: m.Module, HandlerFunc: chartHandler.ServeHTTP, @@ -116,7 +115,6 @@ func (m *module) prepare() error { Path: "netquery/history/clear", MimeType: "application/json", Write: api.PermitUser, - Read: api.PermitUser, BelongsTo: m.Module, HandlerFunc: func(w http.ResponseWriter, r *http.Request) { var body struct { From aa0b42dd01d37c57a3d7033bed622d739839ae90 Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 4 Aug 2023 21:44:45 +0200 Subject: [PATCH 07/10] Fix race condition when setting firewall handler and starting worker --- network/connection.go | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/network/connection.go b/network/connection.go index 1d9edd11..dd4d7b22 100644 --- a/network/connection.go +++ b/network/connection.go @@ -784,15 +784,6 @@ func (conn *Connection) SetFirewallHandler(handler FirewallHandler) { return } - // Start packet handler worker when first handler is set. - if conn.firewallHandler == nil { - // start handling - module.StartWorker("packet handler", conn.packetHandlerWorker) - } - - // Set new handler. - conn.firewallHandler = handler - // Initialize packet queue, if needed. conn.pktQueueLock.Lock() defer conn.pktQueueLock.Unlock() @@ -800,6 +791,14 @@ func (conn *Connection) SetFirewallHandler(handler FirewallHandler) { conn.pktQueue = make(chan packet.Packet, 100) conn.pktQueueActive = true } + + // Start packet handler worker when new handler is set. + if conn.firewallHandler == nil { + module.StartWorker("packet handler", conn.packetHandlerWorker) + } + + // Set new handler. + conn.firewallHandler = handler } // UpdateFirewallHandler sets the firewall handler if it already set and the From d0f4392b9dc076b6178fdfbd7b2d2050a8ae7634 Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 4 Aug 2023 21:49:57 +0200 Subject: [PATCH 08/10] Fix netquery connection ID generation --- firewall/packet_handler.go | 6 ++-- netquery/database.go | 14 ++++---- netquery/manager.go | 70 ++++++++++++++++++++++---------------- process/process.go | 6 ++-- 4 files changed, 55 insertions(+), 41 deletions(-) diff --git a/firewall/packet_handler.go b/firewall/packet_handler.go index cf5e57bf..7c364451 100644 --- a/firewall/packet_handler.go +++ b/firewall/packet_handler.go @@ -628,9 +628,9 @@ func bandwidthUpdateHandler(ctx context.Context) error { return nil case bwUpdate := <-interception.BandwidthUpdates: if bwUpdate != nil { - updateBandwidth(ctx, bwUpdate) // DEBUG: // log.Debugf("filter: bandwidth update: %s", bwUpdate) + updateBandwidth(ctx, bwUpdate) } else { return errors.New("received nil bandwidth update from interception") } @@ -653,6 +653,8 @@ func updateBandwidth(ctx context.Context, bwUpdate *packet.BandwidthUpdate) { // Do not wait for connections that are locked. // TODO: Use atomic operations for updating bandwidth stats. if !conn.TryLock() { + // DEBUG: + // log.Warningf("filter: failed to lock connection for bandwidth update: %s", conn) return } defer conn.Unlock() @@ -675,7 +677,7 @@ func updateBandwidth(ctx context.Context, bwUpdate *packet.BandwidthUpdate) { if err := netquery.DefaultModule.Store.UpdateBandwidth( ctx, conn.HistoryEnabled, - conn.Process().GetID(), + conn.Process().GetKey(), conn.ID, conn.BytesReceived, conn.BytesSent, diff --git a/netquery/database.go b/netquery/database.go index 00e4f006..7823a700 100644 --- a/netquery/database.go +++ b/netquery/database.go @@ -2,8 +2,6 @@ package netquery import ( "context" - "crypto/sha256" - "encoding/hex" "encoding/json" "fmt" "io" @@ -395,12 +393,8 @@ func (db *Database) MarkAllHistoryConnectionsEnded(ctx context.Context) error { // UpdateBandwidth updates bandwidth data for the connection and optionally also writes // the bandwidth data to the history database. func (db *Database) UpdateBandwidth(ctx context.Context, enableHistory bool, processKey string, connID string, bytesReceived uint64, bytesSent uint64) error { - data := connID + "-" + processKey - hash := sha256.Sum256([]byte(data)) - dbConnID := hex.EncodeToString(hash[:]) - params := map[string]any{ - ":id": dbConnID, + ":id": makeNqIDFromParts(processKey, connID), } parts := []string{} @@ -481,6 +475,12 @@ func (db *Database) Save(ctx context.Context, conn Conn, enableHistory bool) err // and save some CPU cycles for the user dbNames := []DatabaseName{LiveDatabase} + // TODO: Should we only add ended connection to the history database to save + // a couple INSERTs per connection? + // This means we need to write the current live DB to the history DB on + // shutdown in order to be able to pick the back up after a restart. + + // Save to history DB if enabled. if enableHistory { dbNames = append(dbNames, HistoryDatabase) } diff --git a/netquery/manager.go b/netquery/manager.go index b5db2ca0..9116cbad 100644 --- a/netquery/manager.go +++ b/netquery/manager.go @@ -100,35 +100,42 @@ func (mng *Manager) HandleFeed(ctx context.Context, feed <-chan *network.Connect if !ok { return } - if !conn.DataIsComplete() { - continue - } - model, err := convertConnection(conn) - if err != nil { - log.Errorf("netquery: failed to convert connection %s to sqlite model: %s", conn.ID, err) + func() { + conn.Lock() + defer conn.Unlock() - continue - } + if !conn.DataIsComplete() { + return + } - // DEBUG: - // log.Tracef("netquery: updating connection %s", conn.ID) + model, err := convertConnection(conn) + if err != nil { + log.Errorf("netquery: failed to convert connection %s to sqlite model: %s", conn.ID, err) - if err := mng.store.Save(ctx, *model, conn.HistoryEnabled); err != nil { - log.Errorf("netquery: failed to save connection %s in sqlite database: %s", conn.ID, err) + return + } - continue - } + // DEBUG: + // log.Tracef("netquery: updating connection %s", conn.ID) - // we clone the record metadata from the connection - // into the new model so the portbase/database layer - // can handle NEW/UPDATE correctly. - cloned := conn.Meta().Duplicate() + // Save to netquery database. + // Do not include internal connections in history. + if err := mng.store.Save(ctx, *model, conn.HistoryEnabled && !conn.Internal); err != nil { + log.Errorf("netquery: failed to save connection %s in sqlite database: %s", conn.ID, err) + return + } - // push an update for the connection - if err := mng.pushConnUpdate(ctx, *cloned, *model); err != nil { - log.Errorf("netquery: failed to push update for conn %s via database system: %s", conn.ID, err) - } + // we clone the record metadata from the connection + // into the new model so the portbase/database layer + // can handle NEW/UPDATE correctly. + cloned := conn.Meta().Duplicate() + + // push an update for the connection + if err := mng.pushConnUpdate(ctx, *cloned, *model); err != nil { + log.Errorf("netquery: failed to push update for conn %s via database system: %s", conn.ID, err) + } + }() } } } @@ -155,18 +162,16 @@ func (mng *Manager) pushConnUpdate(_ context.Context, meta record.Meta, conn Con } // convertConnection converts conn to the local representation used -// to persist the information in SQLite. convertConnection attempts -// to lock conn and may thus block for some time. +// to persist the information in SQLite. +// The caller must hold the lock to the given network.Connection. func convertConnection(conn *network.Connection) (*Conn, error) { - conn.Lock() - defer conn.Unlock() direction := "outbound" if conn.Inbound { direction = "inbound" } c := Conn{ - ID: genConnID(conn), + ID: makeNqIDFromConn(conn), External: conn.External, IPVersion: conn.IPVersion, IPProtocol: conn.IPProtocol, @@ -265,6 +270,13 @@ func convertConnection(conn *network.Connection) (*Conn, error) { return &c, nil } -func genConnID(conn *network.Connection) string { - return conn.ID + "-" + conn.Process().GetID() +// makeNqIDFromConn creates a netquery connection ID from the network connection. +func makeNqIDFromConn(conn *network.Connection) string { + return makeNqIDFromParts(conn.Process().GetKey(), conn.ID) +} + +// makeNqIDFromParts creates a netquery connection ID from the given network +// connection ID and the process key. +func makeNqIDFromParts(processKey string, netConnID string) string { + return processKey + "-" + netConnID } diff --git a/process/process.go b/process/process.go index 733ba819..e982c901 100644 --- a/process/process.go +++ b/process/process.go @@ -314,10 +314,10 @@ func loadProcess(ctx context.Context, key string, pInfo *processInfo.Process) (* return process, nil } -// GetID returns the key that is used internally to identify the process. -// The ID consists of the PID and the start time of the process as reported by +// GetKey returns the key that is used internally to identify the process. +// The key consists of the PID and the start time of the process as reported by // the system. -func (p *Process) GetID() string { +func (p *Process) GetKey() string { return p.processKey } From 53c2b82d2e65b9e4ef9ddec67d5bec76776f121f Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 4 Aug 2023 21:50:03 +0200 Subject: [PATCH 09/10] Improve logging --- resolver/resolve.go | 18 +++++++++++++----- updates/os_integration_linux.go | 4 ++-- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/resolver/resolve.go b/resolver/resolve.go index 94a8f583..09ee2685 100644 --- a/resolver/resolve.go +++ b/resolver/resolve.go @@ -293,11 +293,19 @@ func startAsyncQuery(ctx context.Context, q *Query, currentRRCache *RRCache) { // Set flag and log that we are refreshing this entry. currentRRCache.RequestingNew = true - log.Tracer(ctx).Tracef( - "resolver: cache for %s will expire in %s, refreshing async now", - q.ID(), - time.Until(time.Unix(currentRRCache.Expires, 0)).Round(time.Second), - ) + if currentRRCache.Expired() { + log.Tracer(ctx).Tracef( + "resolver: cache for %s has expired %s ago, refreshing async now", + q.ID(), + time.Since(time.Unix(currentRRCache.Expires, 0)).Round(time.Second), + ) + } else { + log.Tracer(ctx).Tracef( + "resolver: cache for %s will expire in %s, refreshing async now", + q.ID(), + time.Until(time.Unix(currentRRCache.Expires, 0)).Round(time.Second), + ) + } // resolve async module.StartWorker("resolve async", func(asyncCtx context.Context) error { diff --git a/updates/os_integration_linux.go b/updates/os_integration_linux.go index 4f6554c0..b20e3da4 100644 --- a/updates/os_integration_linux.go +++ b/updates/os_integration_linux.go @@ -130,8 +130,8 @@ func upgradeSystemIntegrationFileContents( } // Check if we are allowed to upgrade from the existing file. - if !slices.Contains[string](permittedUpgradeHashes, existingHexSum) { - return fmt.Errorf("%s at %s %w, as it is not a previously published version and cannot be automatically upgraded - try installing again", name, filePath, ErrRequiresManualUpgrade) + if !slices.Contains[[]string, string](permittedUpgradeHashes, existingHexSum) { + return fmt.Errorf("%s at %s (sha256:%s) %w, as it is not a previously published version and cannot be automatically upgraded - try installing again", name, filePath, existingHexSum, ErrRequiresManualUpgrade) } // Start with upgrade! From ac24e3cf874abba7822e3983489b9eef7a7e8585 Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 4 Aug 2023 22:07:49 +0200 Subject: [PATCH 10/10] Update deps --- go.mod | 18 +++++++++--------- go.sum | 30 ++++++++++++++++-------------- 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/go.mod b/go.mod index a816bee4..7affbd42 100644 --- a/go.mod +++ b/go.mod @@ -1,11 +1,11 @@ module github.com/safing/portmaster -go 1.19 +go 1.20 require ( github.com/agext/levenshtein v1.2.3 github.com/cilium/ebpf v0.11.0 - github.com/coreos/go-iptables v0.6.0 + github.com/coreos/go-iptables v0.7.0 github.com/florianl/go-conntrack v0.4.0 github.com/florianl/go-nfqueue v1.3.1 github.com/ghodss/yaml v1.0.0 @@ -16,9 +16,9 @@ require ( github.com/hashicorp/go-version v1.6.0 github.com/jackc/puddle/v2 v2.2.1 github.com/miekg/dns v1.1.55 - github.com/oschwald/maxminddb-golang v1.11.0 + github.com/oschwald/maxminddb-golang v1.12.0 github.com/safing/jess v0.3.1 - github.com/safing/portbase v0.17.0 + github.com/safing/portbase v0.17.1 github.com/safing/portmaster-android/go v0.0.0-20230605085256-6abf4c495626 github.com/safing/spn v0.6.10 github.com/shirou/gopsutil v3.21.11+incompatible @@ -28,10 +28,10 @@ require ( github.com/tannerryan/ring v1.1.2 github.com/tevino/abool v1.2.0 github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26 - golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 - golang.org/x/net v0.12.0 + golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b + golang.org/x/net v0.13.0 golang.org/x/sync v0.3.0 - golang.org/x/sys v0.10.0 + golang.org/x/sys v0.11.0 zombiezen.com/go/sqlite v0.13.0 ) @@ -89,12 +89,12 @@ require ( golang.org/x/crypto v0.11.0 // indirect golang.org/x/mod v0.12.0 // indirect golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.11.0 // indirect + golang.org/x/tools v0.11.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect gvisor.dev/gvisor v0.0.0-20220817001344-846276b3dbc5 // indirect modernc.org/libc v1.24.1 // indirect modernc.org/mathutil v1.6.0 // indirect modernc.org/memory v1.6.0 // indirect - modernc.org/sqlite v1.24.0 // indirect + modernc.org/sqlite v1.25.0 // indirect ) diff --git a/go.sum b/go.sum index ec51652f..899fb7a5 100644 --- a/go.sum +++ b/go.sum @@ -36,8 +36,8 @@ github.com/cilium/ebpf v0.11.0 h1:V8gS/bTCCjX9uUnkUFUpPsksM8n1lXBAvHcpiFk1X2Y= github.com/cilium/ebpf v0.11.0/go.mod h1:WE7CZAnqOL2RouJ4f1uyNhqr2P4CCvXFIqdRDUgWsVs= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= -github.com/coreos/go-iptables v0.6.0 h1:is9qnZMPYjLd8LYqmm/qlE+wwEgJIkTYdhV3rfZo4jk= -github.com/coreos/go-iptables v0.6.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q= +github.com/coreos/go-iptables v0.7.0 h1:XWM3V+MPRr5/q51NuWSgU0fqMad64Zyxs8ZUoMsamr8= +github.com/coreos/go-iptables v0.7.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= @@ -186,8 +186,8 @@ github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zx github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= -github.com/oschwald/maxminddb-golang v1.11.0 h1:aSXMqYR/EPNjGE8epgqwDay+P30hCBZIveY0WZbAWh0= -github.com/oschwald/maxminddb-golang v1.11.0/go.mod h1:YmVI+H0zh3ySFR3w+oz8PCfglAFj3PuCmui13+P9zDg= +github.com/oschwald/maxminddb-golang v1.12.0 h1:9FnTOD0YOhP7DGxGsq4glzpGy5+w7pq50AS6wALUMYs= +github.com/oschwald/maxminddb-golang v1.12.0/go.mod h1:q0Nob5lTCqyQ8WT6FYgS1L7PXKVVbgiymefNwIjPzgY= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -208,6 +208,8 @@ github.com/safing/portbase v0.15.2/go.mod h1:5bHi99fz7Hh/wOsZUOI631WF9ePSHk57c4f github.com/safing/portbase v0.16.2/go.mod h1:mzNCWqPbO7vIYbbK5PElGbudwd2vx4YPNawymL8Aro8= github.com/safing/portbase v0.17.0 h1:RsDzbCGxRIbgaArri3y7MZskfxytEvvkzJpiboDUERQ= github.com/safing/portbase v0.17.0/go.mod h1:eKCRqsfMFLVhNpd2sY/fKvnbuk+LrIYnQEZCg1i86Ho= +github.com/safing/portbase v0.17.1 h1:q2aNHjJw4aoqTqKOxZpxRhYCciHw1exZ7lfGuB78i1E= +github.com/safing/portbase v0.17.1/go.mod h1:1cVgDZIsPiqM5b+K88Kshir5PGIvsftYkx7y1x925+8= github.com/safing/portmaster-android/go v0.0.0-20230605085256-6abf4c495626 h1:olc/REnUdpJN/Gmz8B030OxLpMYxyPDTrDILNEw0eKs= github.com/safing/portmaster-android/go v0.0.0-20230605085256-6abf4c495626/go.mod h1:abwyAQrZGemWbSh/aCD9nnkp0SvFFf/mGWkAbOwPnFE= github.com/safing/spn v0.6.10 h1:4fFBb7UvUzoCcOSd8immOz1Buiuasy5C1/lxfVFacBQ= @@ -307,8 +309,8 @@ golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20221010152910-d6f0a8c073c2/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= -golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 h1:MGwJjxBy0HJshjDNfLsYO8xppfqWlA5ZT9OhtUUhTNw= -golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b h1:r+vk0EmXNmekl0S0BascoeeoHk/L7wmaW2QF90K+kYI= +golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -338,8 +340,8 @@ golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220107192237-5cfca573fb4d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220927171203-f486391704dc/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= -golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= -golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/net v0.13.0 h1:Nvo8UFsZ8X3BhAC9699Z1j7XQ3rsZnUUm7jfBEk1ueY= +golang.org/x/net v0.13.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -391,8 +393,8 @@ golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= -golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210503060354-a79de5458b56/go.mod h1:tfny5GFUkzUvx4ps4ajbZsCe5lw1metzhBm9T3x7oIY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -408,8 +410,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= -golang.org/x/tools v0.11.0 h1:EMCa6U9S2LtZXLAMoWiR/R8dAQFRqbAitmbJ2UKhoi8= -golang.org/x/tools v0.11.0/go.mod h1:anzJrxPjNtfgiYQYirP2CPGzGLxrH2u2QBhn6Bf3qY8= +golang.org/x/tools v0.11.1 h1:ojD5zOW8+7dOGzdnNgersm8aPfcDjhMp12UfG93NIMc= +golang.org/x/tools v0.11.1/go.mod h1:anzJrxPjNtfgiYQYirP2CPGzGLxrH2u2QBhn6Bf3qY8= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -438,7 +440,7 @@ modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= modernc.org/memory v1.6.0 h1:i6mzavxrE9a30whzMfwf7XWVODx2r5OYXvU46cirX7o= modernc.org/memory v1.6.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= -modernc.org/sqlite v1.24.0 h1:EsClRIWHGhLTCX44p+Ri/JLD+vFGo0QGjasg2/F9TlI= -modernc.org/sqlite v1.24.0/go.mod h1:OrDj17Mggn6MhE+iPbBNf7RGKODDE9NFT0f3EwDzJqk= +modernc.org/sqlite v1.25.0 h1:AFweiwPNd/b3BoKnBOfFm+Y260guGMF+0UFk0savqeA= +modernc.org/sqlite v1.25.0/go.mod h1:FL3pVXie73rg3Rii6V/u5BoHlSoyeZeIgKZEgHARyCU= zombiezen.com/go/sqlite v0.13.0 h1:iEeyVqcm3fk5PCA8OQBhBxPnqrP4yYuVJBF+XZpSnOE= zombiezen.com/go/sqlite v0.13.0/go.mod h1:Ht/5Rg3Ae2hoyh1I7gbWtWAl89CNocfqeb/aAMTkJr4=