From fb481bebff01cc49f579be4ede3a572463cd533f Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Fri, 1 Mar 2024 17:20:18 +0530 Subject: [PATCH 1/2] Support sending source PURLs to purldb Signed-off-by: Ayan Sinha Mahapatra --- scanpipe/pipes/purldb.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/scanpipe/pipes/purldb.py b/scanpipe/pipes/purldb.py index 1b46c7a0e9..be78a23c0c 100644 --- a/scanpipe/pipes/purldb.py +++ b/scanpipe/pipes/purldb.py @@ -285,11 +285,19 @@ def get_unique_unresolved_purls(project): def populate_purldb_with_discovered_packages(project, logger=logger.info): """Add DiscoveredPackage to PurlDB.""" discoveredpackages = project.discoveredpackages.all() - packages = [{"purl": pkg.purl} for pkg in discoveredpackages] + packages_to_populate = [] + for pkg in discoveredpackages: + package = {"purl": pkg.purl} + if pkg.source_packages: + package["source_purl"] = pkg.source_packages + packages_to_populate.append(package) - logger(f"Populating PurlDB with {len(packages):,d} PURLs from DiscoveredPackage") + logger( + f"Populating PurlDB with {len(packages_to_populate):,d}" + f" PURLs from DiscoveredPackage" + ) feed_purldb( - packages=packages, + packages=packages_to_populate, chunk_size=100, logger=logger, ) From 3a912f860df3f19b60eaddcefc1b3237894fe559 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Mon, 25 Mar 2024 15:46:04 +0530 Subject: [PATCH 2/2] Add namespace from distro-id Signed-off-by: Ayan Sinha Mahapatra --- scanpipe/pipes/rootfs.py | 21 ++++++++++++ scanpipe/tests/data/basic-rootfs.tar.gz | Bin 4091 -> 4172 bytes .../data/basic-rootfs_root_filesystems.json | 32 ++++++++++-------- 3 files changed, 38 insertions(+), 15 deletions(-) diff --git a/scanpipe/pipes/rootfs.py b/scanpipe/pipes/rootfs.py index b587daa4ca..2cddf1cc9f 100644 --- a/scanpipe/pipes/rootfs.py +++ b/scanpipe/pipes/rootfs.py @@ -23,6 +23,7 @@ import fnmatch import logging import os +from collections import Counter from django.core.exceptions import ObjectDoesNotExist from django.db.models import Q @@ -263,10 +264,30 @@ def scan_rootfs_for_system_packages(project, rootfs): logger.info(f"rootfs location: {rootfs.location}") installed_packages = rootfs.get_installed_packages(package_getter) + + created_system_packages = [] + seen_namespaces = [] for index, (purl, package) in enumerate(installed_packages): logger.info(f"Creating package #{index}: {purl}") + created_system_packages.append(package) + seen_namespaces.append(package.namespace) _create_system_package(project, purl, package) + namespace_counts = Counter(seen_namespaces) + # we overwite namespace only when there are multiple + # namespaces in the packages + if not len(namespace_counts.keys()) > 1: + return + + most_seen_namespace = max(namespace_counts) + # if the distro_id is different from the namespace + # most seen in packages, we update all the package + # namespaces to the distro_id + if most_seen_namespace != distro_id: + for package in created_system_packages: + if package.namespace != distro_id: + package.update(namespace=distro_id) + def get_resource_with_md5(project, status): """ diff --git a/scanpipe/tests/data/basic-rootfs.tar.gz b/scanpipe/tests/data/basic-rootfs.tar.gz index e0b5a05fca4cb124b87b37070c4132c7c98de81c..3bc4595ca9ff53f7b14da6b047a30a1eefc76191 100644 GIT binary patch literal 4172 zcmV-S5VP+eiwFP!000001MOVvcG}33&)4mzsN*kV?}97|32ev7F$USz3rYota%aIYvm`)phG%RbPZi-Thb3TvRNLhDo1V z!_?M4%koMybW_z0!!&BIR86m$>MJmxn=vX`lJXef6~EwdJ`Z+2w0<*nj--VAns<@5P1{3?RrcH$?q zy;ajzuWd=9pQs8$N|_{%S06j}Z`K=6jC~%mi{7R6aWv?>*_oy3B01XMzqz?l^16x} z&G&bhGwSzygW+banFtn})lNs}9|zX`n&WJmC;`txcedc35{h&`Z-;IA1N&XG_2FZ` z*Rk>AYkUM^%7gt>#Pg)eL$A7s0^jvTvcHJ^Ywq6WH*R;>=ycQ_j_xYp=h{C@;=c#} z*Xs2~y`eSgb;5rGVPK2@FHp$-$&AP18NJ|1_NRJU&8*e8_J4`8vcDHSS0?xb`|BFr z$7Hav&HrAcY}!BY$Dy0WiAc;J=nGG=zmC_cRyTERYyX!hkK5mk7PqlKouz-dkFV$d znCv6iZ`fZ~jk>nY|6igEXMO_sKb;9^%fuTNu~=~Ticf_H*CI~*D1>MtYZT7G&`)9> z-vXw)K#8v(P9n%sKk!pbe3i1^{_@!kPar~X@SCI;a(w)W05R2s#G9D-BBwUi*nZ z4n);Y0_0pyd&<60J9bL$Fy;wzmqc@MGvg_s2i!#Q6?jo7NJFP_<_hQwWFmoM`R%0d ze@5=4X;_AN$IPn*aCw4->AGrr@wCoeWNH=vra zhkbH>g-?lKe0B0UG0>HiFu*D}*+gN7ag;6C@uyEIU7TP?3kBWOQ@E>SS5!^8P|&g3 zJV`_dy90i^ht)m#_2kA^I>PrvI12sioGjV(oCjFJf$16lezE4=uN1+ zw=6<4z%`HkC`;g0@Hjy|vQ~+ouX{X=LswaA_OJg1L_>}A-A6Hm zSm0psc)%I!)3{3H$V6kI+X}2SMHfwlS3#%39?&C5Z&Oi0-$qexVS&y5x0c;lzD z2nSd!k~H?+Tv9)DgUmx7Sos*oxnJ}^&7}qsBAO*A0r6RZdF1&M`V>;EMK%unWL9CG zPo&3LiZ@r+vgf>w?@G!^1rloGqSp#(-u)MkVvP3m_8<(sx>DP_lJ1 z!-2aYC_>RBi$ml{$TnVt8kIeM5-tIZtUif?Ai5##yHV)*lC2}cBvktt9SN3U7s4n- zR&y34WUY)|Ji@>o1TYqbK843n`*f=mC#LQ(z=f%grddRBx{ekl>x79H7C61L;rr&m z0^5Q9p!d#hTW#2BI{1C30`KkNMQ=0&tTAYIhaaGK2F>mV_|5LND`5TJA6SmVdIPX8 z`yJcDdv>?g8MW>1IhCX#91FW|BiXLqJsV&z)}_@QD$MTUJ+R*47jQ0`osR6P zdD?qtL95sQFtE=rhH%mAw5@>yrxr5RJndL{FDOr|)3h%U!J3!Nb7E`&JtQ!Y)e6Sm zUs&=McGblHt)boPk~&(w?r?x#D=6S#h)tPn`rdY|3N#0{L%KK{^e!tvn#2}8Nd((< zt(*#J88(c8ROtK2u~xjat!4*lIn;Q)G8)qQ?21K-!O zOF{%TOO`y{!-)mLXuM1ic!qal|4SLW*HN`4QPqIop@mLOzMai~7&J;{zvpNV}mQoj@X;4b#D@4`NS2f>g1P6*>5B-x|O zM-dPH9OALBBOW{j@v7`m%!B8`ywd*bps_+Z_H>kkM^LWHo`7?(fpht+V4ce zDi72Op`a%J>hf=m{Ra)HftVsPEghcHTAD4SGU00>3{N5)u^hdoiNY7Cf>0#lYvEy6 z1F-}T;)#YL35f#bvkQ4=j1xMX2O_CCW>+%fBB^p2*?RI-k&}@eOb9wrg*YuBxS$Ef zWAMPJ(OjfQOjBT!BPm(IiW_-Y4X5%b06D;eF%4kN8_*<(kg1FX%iapSS!V@%bOY`q zt}0Bw&jNO}rh<|Ml+B|F(lc;T1Q8;$8GMRtHg~`|MyD8IFIlNs+J>mE3!t!?0$t(D zMt6tEn#Y9E>aoUw)xSjDTl2D-vn%H+ulxB`ednz_1`0`gb$ZV@$NTNfi`H31~Mp-{Lbjq z8QR0q(1P<`uPtW@jx~5^w=Cy1bb5}|(uiV4yE$yij*$ea2p{A1=?LQpDa-B-t-)Z_ zr?9cdE_&}#8^~i5o3^D^dtE6iDyBD(QTpMmeHFGG)L3iC)~ZqLp;)$(5xUlS$3``` ztc?h_w~TOh-tGJ52Udi+W%#1YH4m*w1A>kDL;IagY!{W$=8j$X2Tq2ZQR||J$eXB- z_+GE-s@kAsfgvVh81*a#+P-@wZW)c92Ma;`Acy!t0rA^agvVDA9$6;%MykyIT&fIT zPnFqrncycc6Lf5cy1%y6d^uU4o1{{}k1cK`P! z${FprrAI(#8;hfjtKB#06ZuXlf^c=t!wlz?EAOaIWb1uRQTN$sf%%i*^J*7E#S!GL z#2h(6&*{_I$tl8iHkRimKP~+DDk^-Au`@~ukKjL7$?g%X@?KrlYVfNc`_QLWaQq3c zPcR(}vmo%5EP2a1dB%1Go%4RDsw>)mEs5qL?LSW2fM)2$;s%a(TM6 z48G^x5p+a?X*`^ZP{f$C^<}~e1vel!wPjw5$=31USgj)1yxhY6; zxj&#`6J#&&V8$^$iiIoWGDp6f@TbkkFgcHk4S*u2&z1Qa{0Svj6`Ajda$Wt=9c3oB zlq+CrScivKm`Z_zgH#}eGk=2A&w?n5aS|UTDb1)Zn*ge=X=+ugsTyDw*_4v?!t!uP za+{=r*81cu1;-w`=I8XAqOqmw)}4$-!MHr3Po~fzY0u<74s;p789~$zZ_}5 zZ1cOnUatAy==^uhG>uw*{#)Ii|9+7|_QwePtUmB```0v6-`f8rN@;&@aW#D|CwP+m z4a)xwwXwDTiV`7+%p{v9f2ohJ zpZ`|XT20%uzpmD`#&-Vy62xFEq3W5Y>zdxACMITmtN}ba_H!qp z@oy@J?C@aRP$&99t)?5|;LsbJje`bPwF7P%+$%@yW5j47v7{twT%DMjX&mYY9@p`I z4KLN2QRA*=jPvzKs7h$`drRW^8}|;YCV6e-A2OuP*QJc_}>Brwvy>y*(X{1}Aoz1*u=9Cr6O*xmfNv zl<%s&%NHDvpsr|DEhncb+JT}PtSuHI^b*NbEXu+2bi(sGJr3HCpM9|Awat%2aJV8T^10iiTU+#~u zR_lw`;fg~5`TKzPBnX8ol7tvcBZ#xGT$fH5(A!piP@Xd^V*|U0=3*iJtK0zk>icJ` zSG*%wU3ZvP%>5}ib8^LFH=)nYlFD~l?h>Qd7CA$&P36mr%jS3861}pR%b+_w?1Fz{ckpK; W`L=A!wrtDO%D(~M6{ZmYPyhf=`bh@> literal 4091 zcmVv^x3O2YRV~RMe1=RQGw;wzuq<^~S5;R%Rn=;eCGqYr&)ig1Z8iz~U8TD{R|Hkw+q(EwGe*Y(CPU_3Wt)UqVyF|x9V=Hpm+{JwsVJul=~e*R6` zCI1Vpf88+jdcOYJ>-xV)qxBE`@vgVHnC|-FB%*Jjo5hJpi~~OR8oDy~j3k>U-#rKF zqtVdsUw_?LTYp^zwN}%0?H5pgpq%%%fB5>>>h2!juNxDtPuym0U-b;bbq#M)7Zal~ z)&R&lzU(zVHS}uWhuN3vG|bkFL`fpWQ1;pW-ngkwYJ2s1O&5Fn-q>jFHMy$oaYN_c zL>a?yoP`4c&XO)I(IdFTzi<(T!dLkiSO5%qUm8= zG3w*-o-r|ab59>PjJj8^H@!zFEqEw`lGM*$r{`)L{{KvKJPH5F|1|?a;T8X1qLBk6 zGaidwFLI?@dkggk8^ROvznW$=as3;HuD-7SOSHS=Kia?#iv83m{_79Kf6Vrl>t8c8 zjQ^TZe~tey(hRTG(CfOG2-j8jRDF++o3*;pgR-3Li+2f+F?yG_~Ci-4Z>NeQ8GagPw5KWbi82da_re9foq6^pa8coj_@9VXu z>$#1_p57dLo~F5FHgN~7XmK6;(^>j&8?b+HEB*gD=XiSkrz2pz;{QvuO*eX;Ii42( zHOl{LYVCFXU!twn|M}*4!uo5}5&d5`Uh}^fX&d<;sv^z)T&{Oldu0CyQ@+~L|Ld42 zV*XzlaOSA!ZFZfg(z}mJSbEF*{fFGiAf&ik6@=|zg5c^T=r`HD%EmA)UdB7au z(v>8VFstlZb^NbLxwcnl=iCod{3qfA7>%2_T;d8;9D2rWuTu8&-;|!9yhN4xq zoSdd;dy1;Fj#!A$OC;kdRX+x7zrCa5PKA274cLD77HW#7?=V{+OY`OE>0(|(a)}%J zi(D5HRXutZ^Vk;&OrjW45zl>88DP%qVJbakFr0~!*P`<^#8HOq;0+`Jd^xJ>ho0nE zgelBLI*UArCJeHL$FctuFqMwEi`@Yk5h6_B%1>ty3!dOYjRPUqZ0>*M#G}GMq51s; z7P%RK8_8u#AVscjNe~KGBndH?Mi6HqEaGSy^LfH>;C$iZn&%UidxPvOnu~?BuUVQd zl7n5WF)#hZM^>tS5}*>4P^7y>|6SI{n+3`x*9|%d#?GP0_2hoSLd=4YW!{+c4mjc^W&Lrr0gaiY%}vusj;$MF-i8 zE(P8m#8Dc#QNW5h0S$sLZBd#nT7xyXY&}5~5{KxGN;B1O+ci#b}uR+dNbi5d zwFl=49E-4e#shy2hhOJo5zcsUgszN0D#0{I3O`{uRgCcvsm?^;0g@z6PcZy}6HU@9 z3ZW-a7J5a@V;cdF7+Yj12DB;0Q}k{e|DOBa?3#x>#V8G@aL%v&Am-te&)MOne7GKe zBqP5?D8;`5;a(6zh-9(Rr2?6Rx3_VApEIJ!;Jdx()L=Z!{W1Q zD5`>{s!YEstewkTE>Ch}ead-c`Y<8cfu}5==E3-j+ObkHhcQo(y98nKYQ|FnQVF46 z6bjPNahxFv4@68U#-aRnH1NM5cXDZ1ha-N2N!xmG=+T!WjFn2{C9(qsiE|%w6@1!4 z9-tG&a~`t8xhF3+HD^=L*uyzFzrd$NFupkYlIUnkN*G|5-w=Qk7-H76V27VSr*v^d z8Hs{s>M7h+vJ2{_oGEBn9iCw9fNh&!?_hUNem%PKm9D^`6OKavGAB#xdddUrd&a+t z;3~p&>u^dpW=C`1jnSG=dG8p-LBK-FkFo@=1%@@$BWsuV3ES=tX#4wn)f!ed_gZEJ ztk6}~hW+3F0-~Wt`W~PdLM+f(JRWdNc*w63>6y4#Xtn}7VJ3zV&a0qN;S6XIq_?T4 zP_o9uYoJtxTv^(ljGT-xKDQm8jHcj;;1+IF zGM}3OGO-|ir-cS3TURr5+zmkyiY8ecB1b}w@gmfyobfYed~}DpPof}*u4JwrVg4f7 zIv`9!wU5z|U>SBHj8bGZXF)>NYSD{F7`QRH9g9MrnA@WE=~gLDOw(h43sWDLW)a2d zI$D&h6DFRS;Pg+1A6m8vmIDL3|K94D9oT9)_4Q_VYhn2kI+AXR_`PH z%j$J1VE#U^O~+w<8?5s|*D~>*)oXW09jkW=$2hLnA41nUw}wb`*aw=hAZ3{j5@YA4 z-9E#wtz)Ze4L?@k#2WUHCY&JA77SYU&}xsmEgJ?Sd(d}GoW6r(dsgqn##zjBvo}H?=ZIjf z^VTUbWzO$f(lTr;22!E#Bgb6v(lJ|Iq~*}y_0G7E<`-8iQgq)k1xI6) zE{O(|TTLa86bPBy$xdksH*Am1qQIldCbuvgNLH35IWg`b;9cN2_D1~xgrUP0_D|(yfemvF5e@XV|FDoE|Myjk*z0R6;GqMjm#mOTGJP?rf4!6eq;boL?`q1oRORVl|h0Md>cNUT`Sp7D(GG z4Sg-b5~0#Xl7(=|es1{6Di%oaM#R1oy^lXioUqMPL_ncR zdc+6_S3q7Sv1>5BhUt|%K)oW7_V%zl&|R|f7{D_Ur(-!1oXyUC*uE4G$!tisgn3rN ze8;j)LTZdz%d0kO3i;|*z!{iri!RLHO%$SKBkJcg9rM3O*a;sllj7}1RDo;I9@#P| zk^Ii+*cn>G(a?m`e!nBD1jn@BTW!;M1KqwOwKSrb(P<4^a$+QbD#FKjeLTWALdvpw zL({fL0}2~E?5zI*wShdgaA-$rwcnGXqGI~CjM8^!?W?fmpvKxmw$_be55=;TjL$m&g|NXc*o^t+AzW>vjdcFR7{_7>$?eqUXevT)te}4Y6Nx%R1y8h4K9yvz;L*{tw z{9moPdjGFA=>5N{H}%)||1Z%_=#6W-JO^`ib)~G{zDk?O-;`qzuFiRw@gP}QM|C1c z?`n#=%SH>#p9P;+dzdOLk1v&{nabl|Qw-bLnBD^KeqQ+TMfv+ijGa(PZ~*_gDs~QF zRrl0Xtq#BWu@3_p1&5#U`UuN`Fbe`-$&z=hn`i6?&^;Y=t2IUYuO-o3q}}^z>)&$1 z=dKCvxL2I0%a6v%e4i?*vMzc9*HH#853!b6ds&HPLs4kt^$Ff!kcD2cFGB|@mI?W3 zZvOIeA`~&^>;qY_K*0^jLk+U7!DRojBN1<)*Z?XId!Q8MRNb-)ied)aJSFDi z;@sShtOo$N-&d2_=Bwfb0B~|<$INTPZThDq@V&A zQ%_Jk7nn+egH9@t!ihgY>L)=I#T7V?l9Vdy^A^zQD^0CxbyWkbB3e?iepnt3O0JVs z&|a^sQqlL&G{2;eipG|zTQ@TDf^m65rLYXz8DQ|o?kX_RIol|#76o~-TYB2BulfC- zZz6Sfqof*(K=4F7&m0e*|5MevQ3p-08EQ?}4PE~KA4aqB!twue6h;sAd+hn&H2x24 zb2$9?sn